You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/11/02 11:33:22 UTC
[01/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Repository: lucene-solr
Updated Branches:
refs/heads/jira/gradle 887b80895 -> d7c036847
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-schema.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-schema.xml b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-schema.xml
new file mode 100644
index 0000000..7187138
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-schema.xml
@@ -0,0 +1,70 @@
+<schema name="dih_test" version="4.0">
+
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
+ <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
+ <fieldType name="tint" class="${solr.tests.IntegerFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tfloat" class="${solr.tests.FloatFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tlong" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tdouble" class="${solr.tests.DoubleFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="date" class="${solr.tests.DateFieldType}" docValues="${solr.tests.numeric.dv}" sortMissingLast="true" omitNorms="true"/>
+ <fieldType name="text" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.MockTokenizerFactory"/>
+ <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1"
+ catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ <filter class="solr.FlattenGraphFilterFactory" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.MockTokenizerFactory"/>
+ <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0"
+ catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="textTight" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.MockTokenizerFactory"/>
+ <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1"
+ catenateNumbers="1" catenateAll="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ <filter class="solr.FlattenGraphFilterFactory" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.MockTokenizerFactory"/>
+ <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1"
+ catenateNumbers="1" catenateAll="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="ignored" stored="false" indexed="false" class="solr.StrField"/>
+
+ <field name="id" type="string" indexed="true" stored="true" required="true"/>
+ <field name="desc" type="string" indexed="true" stored="true" multiValued="true"/>
+ <field name="date" type="date" indexed="true" stored="true"/>
+ <field name="timestamp" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
+
+ <field name="NAME" type="text" indexed="true" stored="true" multiValued="false"/>
+ <field name="COUNTRY_NAME" type="text" indexed="true" stored="true" multiValued="true"/>
+ <field name="SPORT_NAME" type="text" indexed="true" stored="true" multiValued="true"/>
+ <field name="DO_NOT_INDEX" type="ignored"/>
+
+ <field name="_version_" type="tlong" indexed="true" stored="true" multiValued="false"/>
+ <field name="_root_" type="string" indexed="true" stored="true" multiValued="false"/>
+
+ <dynamicField name="*_i" type="tint" indexed="true" stored="true"/>
+ <dynamicField name="*_s" type="string" indexed="true" stored="true"/>
+ <dynamicField name="*_mult_s" type="string" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="*_l" type="tlong" indexed="true" stored="true"/>
+ <dynamicField name="*_t" type="text" indexed="true" stored="true"/>
+ <dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
+ <dynamicField name="*_f" type="tfloat" indexed="true" stored="true"/>
+ <dynamicField name="*_d" type="tdouble" indexed="true" stored="true"/>
+ <dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
+
+ <uniqueKey>id</uniqueKey>
+</schema>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-solr_id-schema.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-solr_id-schema.xml b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-solr_id-schema.xml
new file mode 100644
index 0000000..e8deee4
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-solr_id-schema.xml
@@ -0,0 +1,313 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ This is the Solr schema file. This file should be named "schema.xml" and
+ should be in the conf directory under the solr home
+ (i.e. ./solr/conf/schema.xml by default)
+ or located where the classloader for the Solr webapp can find it.
+
+ This example schema is the recommended starting point for users.
+ It should be kept correct and concise, usable out-of-the-box.
+
+ For more information, on how to customize this file, please see
+ http://wiki.apache.org/solr/SchemaXml
+-->
+
+<schema name="test" version="1.1">
+ <!-- attribute "name" is the name of this schema and is only used for display purposes.
+ Applications should change this to reflect the nature of the search collection.
+ version="1.1" is Solr's version number for the schema syntax and semantics. It should
+ not normally be changed by applications.
+ 1.0: multiValued attribute did not exist, all fields are multiValued by nature
+ 1.1: multiValued attribute introduced, false by default -->
+
+
+ <!-- field type definitions. The "name" attribute is
+ just a label to be used by field definitions. The "class"
+ attribute and any other attributes determine the real
+ behavior of the fieldType.
+ Class names starting with "solr" refer to java classes in the
+ org.apache.solr.analysis package.
+ -->
+
+ <!-- The StrField type is not analyzed, but indexed/stored verbatim.
+ - StrField and TextField support an optional compressThreshold which
+ limits compression (if enabled in the derived fields) to values which
+ exceed a certain size (in characters).
+ -->
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
+
+ <!-- boolean type: "true" or "false" -->
+ <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
+
+ <!-- The optional sortMissingLast and sortMissingFirst attributes are
+ currently supported on types that are sorted internally as strings.
+ - If sortMissingLast="true", then a sort on this field will cause documents
+ without the field to come after documents with the field,
+ regardless of the requested sort order (asc or desc).
+ - If sortMissingFirst="true", then a sort on this field will cause documents
+ without the field to come before documents with the field,
+ regardless of the requested sort order.
+ - If sortMissingLast="false" and sortMissingFirst="false" (the default),
+ then default lucene sorting will be used which places docs without the
+ field first in an ascending sort and last in a descending sort.
+ -->
+
+
+ <!--
+ Default numeric field types. For faster range queries, consider the tint/tfloat/tlong/tdouble types.
+ -->
+ <fieldType name="int" class="${solr.tests.IntegerFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="float" class="${solr.tests.FloatFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="long" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="double" class="${solr.tests.DoubleFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" positionIncrementGap="0"/>
+
+ <!--
+ Numeric field types that index each value at various levels of precision
+ to accelerate range queries when the number of values between the range
+ endpoints is large. See the javadoc for LegacyNumericRangeQuery for internal
+ implementation details.
+
+ Smaller precisionStep values (specified in bits) will lead to more tokens
+ indexed per value, slightly larger index size, and faster range queries.
+ A precisionStep of 0 disables indexing at different precision levels.
+ -->
+ <fieldType name="tint" class="${solr.tests.IntegerFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tfloat" class="${solr.tests.FloatFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tlong" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tdouble" class="${solr.tests.DoubleFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
+
+
+ <!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
+ is a more restricted form of the canonical representation of dateTime
+ http://www.w3.org/TR/xmlschema-2/#dateTime
+ The trailing "Z" designates UTC time and is mandatory.
+ Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
+ All other components are mandatory.
+
+ Expressions can also be used to denote calculations that should be
+ performed relative to "NOW" to determine the value, ie...
+
+ NOW/HOUR
+ ... Round to the start of the current hour
+ NOW-1DAY
+ ... Exactly 1 day prior to now
+ NOW/DAY+6MONTHS+3DAYS
+ ... 6 months and 3 days in the future from the start of
+ the current day
+
+ Consult the TrieDateField javadocs for more information.
+ -->
+ <fieldType name="date" class="${solr.tests.DateFieldType}" docValues="${solr.tests.numeric.dv}" sortMissingLast="true" omitNorms="true"/>
+
+
+ <!-- The "RandomSortField" is not used to store or search any
+ data. You can declare fields of this type it in your schema
+ to generate psuedo-random orderings of your docs for sorting
+ purposes. The ordering is generated based on the field name
+ and the version of the index, As long as the index version
+ remains unchanged, and the same field name is reused,
+ the ordering of the docs will be consistent.
+ If you want differend psuedo-random orderings of documents,
+ for the same version of the index, use a dynamicField and
+ change the name
+ -->
+ <fieldType name="random" class="solr.RandomSortField" indexed="true"/>
+
+ <!-- solr.TextField allows the specification of custom text analyzers
+ specified as a tokenizer and a list of token filters. Different
+ analyzers may be specified for indexing and querying.
+
+ The optional positionIncrementGap puts space between multiple fields of
+ this type on the same document, with the purpose of preventing false phrase
+ matching across fields.
+
+ For more info on customizing your analyzer chain, please see
+ http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters
+ -->
+
+ <!-- One can also specify an existing Analyzer class that has a
+ default constructor via the class attribute on the analyzer element
+ <fieldType name="text_greek" class="solr.TextField">
+ <analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/>
+ </fieldType>
+ -->
+
+ <!-- A text field that only splits on whitespace for exact matching of words -->
+ <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.MockTokenizerFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- A text field that uses WordDelimiterGraphFilter to enable splitting and matching of
+ words on case-change, alpha numeric boundaries, and non-alphanumeric chars,
+ so that a query of "wifi" or "wi fi" could match a document containing "Wi-Fi".
+ Synonyms and stopwords are customized by external files, and stemming is enabled.
+ Duplicate tokens at the same position (which may result from Stemmed Synonyms or
+ WordDelim parts) are removed.
+ -->
+ <fieldType name="text" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.MockTokenizerFactory"/>
+ <!-- in this example, we will only use synonyms at query time
+ <filter class="solr.SynonymGraphFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
+ -->
+ <!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
+ <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1"
+ catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <!--<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+ <filter class="solr.PorterStemFilterFactory"/>-->
+ <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ <filter class="solr.FlattenGraphFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.MockTokenizerFactory"/>
+ <!--<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>-->
+ <!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
+ <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0"
+ catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <!--<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+ <filter class="solr.PorterStemFilterFactory"/>-->
+ <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+
+ <!-- Less flexible matching, but less false matches. Probably not ideal for product names,
+ but may be good for SKUs. Can insert dashes in the wrong place and still match. -->
+ <fieldType name="textTight" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.MockTokenizerFactory"/>
+ <!--<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="false"/>-->
+ <!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
+ <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1"
+ catenateNumbers="1" catenateAll="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <!--<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+ <filter class="solr.EnglishMinimalStemFilterFactory"/>-->
+ <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ <filter class="solr.FlattenGraphFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.MockTokenizerFactory"/>
+ <!--<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="false"/>-->
+ <!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
+ <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1"
+ catenateNumbers="1" catenateAll="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <!--<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+ <filter class="solr.EnglishMinimalStemFilterFactory"/>-->
+ <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- This is an example of using the KeywordTokenizer along
+ With various TokenFilterFactories to produce a sortable field
+ that does not include some properties of the source text
+ -->
+ <fieldType name="alphaOnlySort" class="solr.TextField" sortMissingLast="true" omitNorms="true">
+ <analyzer>
+ <!-- KeywordTokenizer does no actual tokenizing, so the entire
+ input string is preserved as a single token
+ -->
+ <tokenizer class="solr.MockTokenizerFactory" pattern="keyword"/>
+ <!-- The LowerCase TokenFilter does what you expect, which can be
+ when you want your sorting to be case insensitive
+ -->
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <!-- The TrimFilter removes any leading or trailing whitespace -->
+ <filter class="solr.TrimFilterFactory"/>
+ <!-- The PatternReplaceFilter gives you the flexibility to use
+ Java Regular expression to replace any sequence of characters
+ matching a pattern with an arbitrary replacement string,
+ which may include back refrences to portions of the orriginal
+ string matched by the pattern.
+
+ See the Java Regular Expression documentation for more
+ infomation on pattern and replacement string syntax.
+
+ http://docs.oracle.com/javase/8/docs/api/java/util/regex/package-summary.html
+ -->
+ <filter class="solr.PatternReplaceFilterFactory"
+ pattern="([^a-z])" replacement="" replace="all"
+ />
+ </analyzer>
+ </fieldType>
+
+ <!-- since fields of this type are by default not stored or indexed, any data added to
+ them will be ignored outright
+ -->
+ <fieldType name="ignored" stored="false" indexed="false" class="solr.StrField"/>
+
+ <!-- Valid attributes for fields:
+ name: mandatory - the name for the field
+ type: mandatory - the name of a previously defined type from the <fieldType>s
+ indexed: true if this field should be indexed (searchable or sortable)
+ stored: true if this field should be retrievable
+ multiValued: true if this field may contain multiple values per document
+ omitNorms: (expert) set to true to omit the norms associated with
+ this field (this disables length normalization and index-time
+ boosting for the field, and saves some memory). Only full-text
+ fields or fields that need an index-time boost need norms.
+ termVectors: [false] set to true to store the term vector for a given field.
+ When using MoreLikeThis, fields used for similarity should be stored for
+ best performance.
+ -->
+
+ <field name="solr_id" type="string" indexed="true" stored="true" required="true"/>
+ <field name="desc" type="string" indexed="true" stored="true" multiValued="true"/>
+
+ <field name="date" type="date" indexed="true" stored="true"/>
+
+ <field name="timestamp" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
+
+
+ <!-- Dynamic field definitions. If a field name is not found, dynamicFields
+ will be used if the name matches any of the patterns.
+ RESTRICTION: the glob-like pattern in the name attribute must have
+ a "*" only at the start or the end.
+ EXAMPLE: name="*_i" will match any field ending in _i (like myid_i, z_i)
+ Longer patterns will be matched first. if equal size patterns
+ both match, the first appearing in the schema will be used. -->
+ <dynamicField name="*_i" type="int" indexed="true" stored="true"/>
+ <dynamicField name="*_s" type="string" indexed="true" stored="true"/>
+ <dynamicField name="*_l" type="long" indexed="true" stored="true"/>
+ <dynamicField name="*_t" type="text" indexed="true" stored="true"/>
+ <dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
+ <dynamicField name="*_f" type="float" indexed="true" stored="true"/>
+ <dynamicField name="*_d" type="double" indexed="true" stored="true"/>
+ <dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
+
+ <dynamicField name="random*" type="random"/>
+
+ <!-- uncomment the following to ignore any fields that don't already match an existing
+ field name or dynamic field, rather than reporting them as an error.
+ alternately, change the type="ignored" to some other type e.g. "text" if you want
+ unknown fields indexed and/or stored by default -->
+ <!--dynamicField name="*" type="ignored" /-->
+
+
+ <!-- Field to use to determine and enforce document uniqueness.
+ Unless this field is marked with required="false", it will be a required field
+ -->
+ <uniqueKey>solr_id</uniqueKey>
+</schema>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-solrconfig.xml b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-solrconfig.xml
new file mode 100644
index 0000000..a662fd9
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-solrconfig.xml
@@ -0,0 +1,293 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<config>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+ <indexConfig>
+ <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+ </indexConfig>
+
+ <!-- Used to specify an alternate directory to hold all index data
+ other than the default ./data under the Solr home.
+ If replication is in use, this should match the replication configuration. -->
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <!-- the default high-performance update handler -->
+ <updateHandler class="solr.DirectUpdateHandler2">
+
+ <!-- A prefix of "solr." for class names is an alias that
+ causes solr to search appropriate packages, including
+ org.apache.solr.(search|update|request|core|analysis)
+ -->
+
+ <!-- Limit the number of deletions Solr will buffer during doc updating.
+
+ Setting this lower can help bound memory use during indexing.
+ -->
+ <maxPendingDeletes>100000</maxPendingDeletes>
+
+ </updateHandler>
+
+
+ <query>
+ <!-- Maximum number of clauses in a boolean query... can affect
+ range or prefix queries that expand to big boolean
+ queries. An exception is thrown if exceeded. -->
+ <maxBooleanClauses>1024</maxBooleanClauses>
+
+
+ <!-- Cache used by SolrIndexSearcher for filters (DocSets),
+ unordered sets of *all* documents that match a query.
+ When a new searcher is opened, its caches may be prepopulated
+ or "autowarmed" using data from caches in the old searcher.
+ autowarmCount is the number of items to prepopulate. For LRUCache,
+ the autowarmed items will be the most recently accessed items.
+ Parameters:
+ class - the SolrCache implementation (currently only LRUCache)
+ size - the maximum number of entries in the cache
+ initialSize - the initial capacity (number of entries) of
+ the cache. (seel java.util.HashMap)
+ autowarmCount - the number of entries to prepopulate from
+ and old cache.
+ -->
+ <filterCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="256"/>
+
+ <!-- queryResultCache caches results of searches - ordered lists of
+ document ids (DocList) based on a query, a sort, and the range
+ of documents requested. -->
+ <queryResultCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="256"/>
+
+ <!-- documentCache caches Lucene Document objects (the stored fields for each document).
+ Since Lucene internal document ids are transient, this cache will not be autowarmed. -->
+ <documentCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+
+ <!-- If true, stored fields that are not requested will be loaded lazily.
+
+ This can result in a significant speed improvement if the usual case is to
+ not load all stored fields, especially if the skipped fields are large compressed
+ text fields.
+ -->
+ <enableLazyFieldLoading>true</enableLazyFieldLoading>
+
+ <!-- Example of a generic cache. These caches may be accessed by name
+ through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert().
+ The purpose is to enable easy caching of user/application level data.
+ The regenerator argument should be specified as an implementation
+ of solr.search.CacheRegenerator if autowarming is desired. -->
+ <!--
+ <cache name="myUserCache"
+ class="solr.LRUCache"
+ size="4096"
+ initialSize="1024"
+ autowarmCount="1024"
+ regenerator="org.mycompany.mypackage.MyRegenerator"
+ />
+ -->
+
+ <!-- An optimization that attempts to use a filter to satisfy a search.
+ If the requested sort does not include score, then the filterCache
+ will be checked for a filter matching the query. If found, the filter
+ will be used as the source of document ids, and then the sort will be
+ applied to that.
+ <useFilterForSortedQuery>true</useFilterForSortedQuery>
+ -->
+
+ <!-- An optimization for use with the queryResultCache. When a search
+ is requested, a superset of the requested number of document ids
+ are collected. For example, if a search for a particular query
+ requests matching documents 10 through 19, and queryWindowSize is 50,
+ then documents 0 through 49 will be collected and cached. Any further
+ requests in that range can be satisfied via the cache. -->
+ <queryResultWindowSize>50</queryResultWindowSize>
+
+ <!-- Maximum number of documents to cache for any entry in the
+ queryResultCache. -->
+ <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
+
+ <!-- This entry enables an int hash representation for filters (DocSets)
+ when the number of items in the set is less than maxSize. For smaller
+ sets, this representation is more memory efficient, more efficient to
+ iterate over, and faster to take intersections. -->
+ <HashDocSet maxSize="3000" loadFactor="0.75"/>
+
+ <!-- a newSearcher event is fired whenever a new searcher is being prepared
+ and there is a current searcher handling requests (aka registered). -->
+ <!-- QuerySenderListener takes an array of NamedList and executes a
+ local query request for each NamedList in sequence. -->
+ <!--<listener event="newSearcher" class="solr.QuerySenderListener">-->
+ <!--<arr name="queries">-->
+ <!--<lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
+ <!--<lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
+ <!--<lst><str name="q">static newSearcher warming query from solrconfig.xml</str></lst>-->
+ <!--</arr>-->
+ <!--</listener>-->
+
+ <!-- a firstSearcher event is fired whenever a new searcher is being
+ prepared but there is no current registered searcher to handle
+ requests or to gain autowarming data from. -->
+ <!--<listener event="firstSearcher" class="solr.QuerySenderListener">-->
+ <!--<arr name="queries">-->
+ <!--</arr>-->
+ <!--</listener>-->
+
+ <!-- If a search request comes in and there is no current registered searcher,
+ then immediately register the still warming searcher and use it. If
+ "false" then all requests will block until the first searcher is done
+ warming. -->
+ <useColdSearcher>false</useColdSearcher>
+
+ <!-- Maximum number of searchers that may be warming in the background
+ concurrently. An error is returned if this limit is exceeded. Recommend
+ 1-2 for read-only slaves, higher for masters w/o cache warming. -->
+ <maxWarmingSearchers>4</maxWarmingSearchers>
+
+ </query>
+
+ <requestDispatcher>
+ <!--Make sure your system has some authentication before enabling remote streaming!
+ <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="-1" />
+ -->
+
+ <!-- Set HTTP caching related parameters (for proxy caches and clients).
+
+ To get the behaviour of Solr 1.2 (ie: no caching related headers)
+ use the never304="true" option and do not specify a value for
+ <cacheControl>
+ -->
+ <httpCaching never304="true">
+ <!--httpCaching lastModifiedFrom="openTime"
+ etagSeed="Solr"-->
+ <!-- lastModFrom="openTime" is the default, the Last-Modified value
+ (and validation against If-Modified-Since requests) will all be
+ relative to when the current Searcher was opened.
+ You can change it to lastModFrom="dirLastMod" if you want the
+ value to exactly corrispond to when the physical index was last
+ modified.
+
+ etagSeed="..." is an option you can change to force the ETag
+ header (and validation against If-None-Match requests) to be
+ differnet even if the index has not changed (ie: when making
+ significant changes to your config file)
+
+ lastModifiedFrom and etagSeed are both ignored if you use the
+ never304="true" option.
+ -->
+ <!-- If you include a <cacheControl> directive, it will be used to
+ generate a Cache-Control header, as well as an Expires header
+ if the value contains "max-age="
+
+ By default, no Cache-Control header is generated.
+
+ You can use the <cacheControl> option even if you have set
+ never304="true"
+ -->
+ <!-- <cacheControl>max-age=30, public</cacheControl> -->
+ </httpCaching>
+ </requestDispatcher>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <!-- default values for query parameters -->
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="df">desc</str>
+ <!--
+ <int name="rows">10</int>
+ <str name="fl">*</str>
+ <str name="version">2.1</str>
+ -->
+ </lst>
+ </requestHandler>
+
+ <requestHandler name="/dataimport" class="org.apache.solr.handler.dataimport.DataImportHandler">
+ <lst name="defaults">
+ <str name="dots.in.hsqldb.driver">org.hsqldb.jdbcDriver</str>
+ </lst>
+ </requestHandler>
+
+ <!--
+
+ Search components are registered to SolrCore and used by Search Handlers
+
+ By default, the following components are avaliable:
+
+ <searchComponent name="query" class="org.apache.solr.handler.component.QueryComponent" />
+ <searchComponent name="facet" class="org.apache.solr.handler.component.FacetComponent" />
+ <searchComponent name="mlt" class="org.apache.solr.handler.component.MoreLikeThisComponent" />
+ <searchComponent name="highlight" class="org.apache.solr.handler.component.HighlightComponent" />
+ <searchComponent name="debug" class="org.apache.solr.handler.component.DebugComponent" />
+
+ If you register a searchComponent to one of the standard names, that will be used instead.
+
+ -->
+
+ <requestHandler name="/search" class="org.apache.solr.handler.component.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ </lst>
+ <!--
+ By default, this will register the following components:
+
+ <arr name="components">
+ <str>query</str>
+ <str>facet</str>
+ <str>mlt</str>
+ <str>highlight</str>
+ <str>debug</str>
+ </arr>
+
+ To insert handlers before or after the 'standard' components, use:
+
+ <arr name="first-components">
+ <str>first</str>
+ </arr>
+
+ <arr name="last-components">
+ <str>last</str>
+ </arr>
+
+ -->
+ </requestHandler>
+
+ <!-- config for the admin interface -->
+ <admin>
+ <defaultQuery>*:*</defaultQuery>
+ </admin>
+
+ <updateRequestProcessorChain key="dataimport" default="true">
+ <processor class="org.apache.solr.handler.dataimport.AbstractDataImportHandlerTestCase$TestUpdateRequestProcessorFactory"/>
+ <processor class="solr.RunUpdateProcessorFactory"/>
+ <processor class="solr.LogUpdateProcessorFactory"/>
+ </updateRequestProcessorChain>
+
+</config>
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/protwords.txt
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/protwords.txt b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/protwords.txt
new file mode 100644
index 0000000..7878147
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/protwords.txt
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#use a protected word file to avoid stemming two
+#unrelated words to the same base word.
+#to test, we will use words that would normally obviously be stemmed.
+cats
+ridding
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/single-entity-data-config.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/single-entity-data-config.xml b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/single-entity-data-config.xml
new file mode 100644
index 0000000..7375a2c
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/single-entity-data-config.xml
@@ -0,0 +1,9 @@
+<dataConfig>
+ <dataSource type="MockDataSource"/>
+ <document>
+ <entity name="x" query="select * from x">
+ <field column="id" />
+ <field column="desc" />
+ </entity>
+ </document>
+</dataConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/stopwords.txt
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/stopwords.txt b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/stopwords.txt
new file mode 100644
index 0000000..688e307
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/stopwords.txt
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+stopworda
+stopwordb
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/synonyms.txt
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/synonyms.txt b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/synonyms.txt
new file mode 100644
index 0000000..a7624f0
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/synonyms.txt
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+a => aa
+b => b1 b2
+c => c1,c2
+a\=>a => b\=>b
+a\,a => b\,b
+foo,bar,baz
+
+Television,TV,Televisions
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/solr.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/solr.xml b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/solr.xml
new file mode 100644
index 0000000..330eef1
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/solr.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ solr.xml mimicking the old default solr.xml
+-->
+
+<solr>
+ <shardHandlerFactory name="shardHandlerFactory" class="HttpShardHandlerFactory">
+ <str name="urlScheme">${urlScheme:}</str>
+ </shardHandlerFactory>
+</solr>
\ No newline at end of file
[20/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java
deleted file mode 100644
index 920472e..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.*;
-import java.lang.invoke.MethodHandles;
-import java.nio.charset.StandardCharsets;
-import java.util.Properties;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-
-/**
- * <p>
- * A {@link DataSource} which reads from local files
- * </p>
- * <p>
- * The file is read with the default platform encoding. It can be overriden by
- * specifying the encoding in solrconfig.xml
- * </p>
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 1.3
- */
-public class FileDataSource extends DataSource<Reader> {
- public static final String BASE_PATH = "basePath";
-
- /**
- * The basePath for this data source
- */
- protected String basePath;
-
- /**
- * The encoding using which the given file should be read
- */
- protected String encoding = null;
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- @Override
- public void init(Context context, Properties initProps) {
- basePath = initProps.getProperty(BASE_PATH);
- if (initProps.get(URLDataSource.ENCODING) != null)
- encoding = initProps.getProperty(URLDataSource.ENCODING);
- }
-
- /**
- * <p>
- * Returns a reader for the given file.
- * </p>
- * <p>
- * If the given file is not absolute, we try to construct an absolute path
- * using basePath configuration. If that fails, then the relative path is
- * tried. If file is not found a RuntimeException is thrown.
- * </p>
- * <p>
- * <b>It is the responsibility of the calling method to properly close the
- * returned Reader</b>
- * </p>
- */
- @Override
- public Reader getData(String query) {
- File f = getFile(basePath,query);
- try {
- return openStream(f);
- } catch (Exception e) {
- wrapAndThrow(SEVERE,e,"Unable to open File : "+f.getAbsolutePath());
- return null;
- }
- }
-
- static File getFile(String basePath, String query) {
- try {
- File file = new File(query);
-
- // If it's not an absolute path, try relative from basePath.
- if (!file.isAbsolute()) {
- // Resolve and correct basePath.
- File basePathFile;
- if (basePath == null) {
- basePathFile = new File(".").getAbsoluteFile();
- log.warn("FileDataSource.basePath is empty. " +
- "Resolving to: " + basePathFile.getAbsolutePath());
- } else {
- basePathFile = new File(basePath);
- if (!basePathFile.isAbsolute()) {
- basePathFile = basePathFile.getAbsoluteFile();
- log.warn("FileDataSource.basePath is not absolute. Resolving to: "
- + basePathFile.getAbsolutePath());
- }
- }
-
- file = new File(basePathFile, query).getAbsoluteFile();
- }
-
- if (file.isFile() && file.canRead()) {
- log.debug("Accessing File: " + file.getAbsolutePath());
- return file;
- } else {
- throw new FileNotFoundException("Could not find file: " + query +
- " (resolved to: " + file.getAbsolutePath());
- }
- } catch (FileNotFoundException e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Open a {@link java.io.Reader} for the given file name
- *
- * @param file a {@link java.io.File} instance
- * @return a Reader on the given file
- * @throws FileNotFoundException if the File does not exist
- * @throws UnsupportedEncodingException if the encoding is unsupported
- * @since solr 1.4
- */
- protected Reader openStream(File file) throws FileNotFoundException,
- UnsupportedEncodingException {
- if (encoding == null) {
- return new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8);
- } else {
- return new InputStreamReader(new FileInputStream(file), encoding);
- }
- }
-
- @Override
- public void close() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java
deleted file mode 100644
index a03354f..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java
+++ /dev/null
@@ -1,305 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.File;
-import java.io.FilenameFilter;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.solr.util.DateMathParser;
-
-/**
- * <p>
- * An {@link EntityProcessor} instance which can stream file names found in a given base
- * directory matching patterns and returning rows containing file information.
- * </p>
- * <p>
- * It supports querying a give base directory by matching:
- * <ul>
- * <li>regular expressions to file names</li>
- * <li>excluding certain files based on regular expression</li>
- * <li>last modification date (newer or older than a given date or time)</li>
- * <li>size (bigger or smaller than size given in bytes)</li>
- * <li>recursively iterating through sub-directories</li>
- * </ul>
- * Its output can be used along with {@link FileDataSource} to read from files in file
- * systems.
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 1.3
- * @see Pattern
- */
-public class FileListEntityProcessor extends EntityProcessorBase {
- /**
- * A regex pattern to identify files given in data-config.xml after resolving any variables
- */
- protected String fileName;
-
- /**
- * The baseDir given in data-config.xml after resolving any variables
- */
- protected String baseDir;
-
- /**
- * A Regex pattern of excluded file names as given in data-config.xml after resolving any variables
- */
- protected String excludes;
-
- /**
- * The newerThan given in data-config as a {@link java.util.Date}
- * <p>
- * <b>Note: </b> This variable is resolved just-in-time in the {@link #nextRow()} method.
- * </p>
- */
- protected Date newerThan;
-
- /**
- * The newerThan given in data-config as a {@link java.util.Date}
- */
- protected Date olderThan;
-
- /**
- * The biggerThan given in data-config as a long value
- * <p>
- * <b>Note: </b> This variable is resolved just-in-time in the {@link #nextRow()} method.
- * </p>
- */
- protected long biggerThan = -1;
-
- /**
- * The smallerThan given in data-config as a long value
- * <p>
- * <b>Note: </b> This variable is resolved just-in-time in the {@link #nextRow()} method.
- * </p>
- */
- protected long smallerThan = -1;
-
- /**
- * The recursive given in data-config. Default value is false.
- */
- protected boolean recursive = false;
-
- private Pattern fileNamePattern, excludesPattern;
-
- @Override
- public void init(Context context) {
- super.init(context);
- fileName = context.getEntityAttribute(FILE_NAME);
- if (fileName != null) {
- fileName = context.replaceTokens(fileName);
- fileNamePattern = Pattern.compile(fileName);
- }
- baseDir = context.getEntityAttribute(BASE_DIR);
- if (baseDir == null)
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "'baseDir' is a required attribute");
- baseDir = context.replaceTokens(baseDir);
- File dir = new File(baseDir);
- if (!dir.isDirectory())
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "'baseDir' value: " + baseDir + " is not a directory");
-
- String r = context.getEntityAttribute(RECURSIVE);
- if (r != null)
- recursive = Boolean.parseBoolean(r);
- excludes = context.getEntityAttribute(EXCLUDES);
- if (excludes != null) {
- excludes = context.replaceTokens(excludes);
- excludesPattern = Pattern.compile(excludes);
- }
- }
-
- /**
- * Get the Date object corresponding to the given string.
- *
- * @param dateStr the date string. It can be a DateMath string or it may have a evaluator function
- * @return a Date instance corresponding to the input string
- */
- private Date getDate(String dateStr) {
- if (dateStr == null)
- return null;
-
- Matcher m = PLACE_HOLDER_PATTERN.matcher(dateStr);
- if (m.find()) {
- Object o = context.resolve(m.group(1));
- if (o instanceof Date) return (Date)o;
- dateStr = (String) o;
- } else {
- dateStr = context.replaceTokens(dateStr);
- }
- m = Evaluator.IN_SINGLE_QUOTES.matcher(dateStr);
- if (m.find()) {
- String expr = m.group(1);
- //TODO refactor DateMathParser.parseMath a bit to have a static method for this logic.
- if (expr.startsWith("NOW")) {
- expr = expr.substring("NOW".length());
- }
- try {
- // DWS TODO: is this TimeZone the right default for us? Deserves explanation if so.
- return new DateMathParser(TimeZone.getDefault()).parseMath(expr);
- } catch (ParseException exp) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Invalid expression for date", exp);
- }
- }
- try {
- return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).parse(dateStr);
- } catch (ParseException exp) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Invalid expression for date", exp);
- }
- }
-
- /**
- * Get the Long value for the given string after resolving any evaluator or variable.
- *
- * @param sizeStr the size as a string
- * @return the Long value corresponding to the given string
- */
- private Long getSize(String sizeStr) {
- if (sizeStr == null)
- return null;
-
- Matcher m = PLACE_HOLDER_PATTERN.matcher(sizeStr);
- if (m.find()) {
- Object o = context.resolve(m.group(1));
- if (o instanceof Number) {
- Number number = (Number) o;
- return number.longValue();
- }
- sizeStr = (String) o;
- } else {
- sizeStr = context.replaceTokens(sizeStr);
- }
-
- return Long.parseLong(sizeStr);
- }
-
- @Override
- public Map<String, Object> nextRow() {
- if (rowIterator != null)
- return getNext();
- List<Map<String, Object>> fileDetails = new ArrayList<>();
- File dir = new File(baseDir);
-
- String dateStr = context.getEntityAttribute(NEWER_THAN);
- newerThan = getDate(dateStr);
- dateStr = context.getEntityAttribute(OLDER_THAN);
- olderThan = getDate(dateStr);
- String biggerThanStr = context.getEntityAttribute(BIGGER_THAN);
- if (biggerThanStr != null)
- biggerThan = getSize(biggerThanStr);
- String smallerThanStr = context.getEntityAttribute(SMALLER_THAN);
- if (smallerThanStr != null)
- smallerThan = getSize(smallerThanStr);
-
- getFolderFiles(dir, fileDetails);
- rowIterator = fileDetails.iterator();
- return getNext();
- }
-
- private void getFolderFiles(File dir, final List<Map<String, Object>> fileDetails) {
- // Fetch an array of file objects that pass the filter, however the
- // returned array is never populated; accept() always returns false.
- // Rather we make use of the fileDetails array which is populated as
- // a side affect of the accept method.
- dir.list(new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- File fileObj = new File(dir, name);
- if (fileObj.isDirectory()) {
- if (recursive) getFolderFiles(fileObj, fileDetails);
- } else if (fileNamePattern == null) {
- addDetails(fileDetails, dir, name);
- } else if (fileNamePattern.matcher(name).find()) {
- if (excludesPattern != null && excludesPattern.matcher(name).find())
- return false;
- addDetails(fileDetails, dir, name);
- }
- return false;
- }
- });
- }
-
- private void addDetails(List<Map<String, Object>> files, File dir, String name) {
- Map<String, Object> details = new HashMap<>();
- File aFile = new File(dir, name);
- if (aFile.isDirectory()) return;
- long sz = aFile.length();
- Date lastModified = new Date(aFile.lastModified());
- if (biggerThan != -1 && sz <= biggerThan)
- return;
- if (smallerThan != -1 && sz >= smallerThan)
- return;
- if (olderThan != null && lastModified.after(olderThan))
- return;
- if (newerThan != null && lastModified.before(newerThan))
- return;
- details.put(DIR, dir.getAbsolutePath());
- details.put(FILE, name);
- details.put(ABSOLUTE_FILE, aFile.getAbsolutePath());
- details.put(SIZE, sz);
- details.put(LAST_MODIFIED, lastModified);
- files.add(details);
- }
-
- public static final Pattern PLACE_HOLDER_PATTERN = Pattern
- .compile("\\$\\{(.*?)\\}");
-
- public static final String DIR = "fileDir";
-
- public static final String FILE = "file";
-
- public static final String ABSOLUTE_FILE = "fileAbsolutePath";
-
- public static final String SIZE = "fileSize";
-
- public static final String LAST_MODIFIED = "fileLastModified";
-
- public static final String FILE_NAME = "fileName";
-
- public static final String BASE_DIR = "baseDir";
-
- public static final String EXCLUDES = "excludes";
-
- public static final String NEWER_THAN = "newerThan";
-
- public static final String OLDER_THAN = "olderThan";
-
- public static final String BIGGER_THAN = "biggerThan";
-
- public static final String SMALLER_THAN = "smallerThan";
-
- public static final String RECURSIVE = "recursive";
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java
deleted file mode 100644
index e62c329..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.lucene.analysis.charfilter.HTMLStripCharFilter;
-
-import java.io.IOException;
-import java.io.StringReader;
-import java.io.BufferedReader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A {@link Transformer} implementation which strip off HTML tags using {@link HTMLStripCharFilter} This is useful
- * in case you don't need this HTML anyway.
- *
- * @see HTMLStripCharFilter
- * @since solr 1.4
- */
-public class HTMLStripTransformer extends Transformer {
-
- @Override
- @SuppressWarnings("unchecked")
- public Object transformRow(Map<String, Object> row, Context context) {
- List<Map<String, String>> fields = context.getAllEntityFields();
- for (Map<String, String> field : fields) {
- String col = field.get(DataImporter.COLUMN);
- String splitHTML = context.replaceTokens(field.get(STRIP_HTML));
- if (!TRUE.equals(splitHTML))
- continue;
- Object tmpVal = row.get(col);
- if (tmpVal == null)
- continue;
-
- if (tmpVal instanceof List) {
- List<String> inputs = (List<String>) tmpVal;
- List results = new ArrayList();
- for (String input : inputs) {
- if (input == null)
- continue;
- Object o = stripHTML(input, col);
- if (o != null)
- results.add(o);
- }
- row.put(col, results);
- } else {
- String value = tmpVal.toString();
- Object o = stripHTML(value, col);
- if (o != null)
- row.put(col, o);
- }
- }
- return row;
- }
-
- private Object stripHTML(String value, String column) {
- StringBuilder out = new StringBuilder();
- StringReader strReader = new StringReader(value);
- try {
- HTMLStripCharFilter html = new HTMLStripCharFilter(strReader.markSupported() ? strReader : new BufferedReader(strReader));
- char[] cbuf = new char[1024 * 10];
- while (true) {
- int count = html.read(cbuf);
- if (count == -1)
- break; // end of stream mark is -1
- if (count > 0)
- out.append(cbuf, 0, count);
- }
- html.close();
- } catch (IOException e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Failed stripping HTML for column: " + column, e);
- }
- return out.toString();
- }
-
- public static final String STRIP_HTML = "stripHTML";
-
- public static final String TRUE = "true";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java
deleted file mode 100644
index a8eed55..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java
+++ /dev/null
@@ -1,593 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-
-import org.apache.solr.common.SolrException;
-import org.apache.solr.util.CryptoKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.naming.InitialContext;
-import javax.naming.NamingException;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.lang.invoke.MethodHandles;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.sql.*;
-import java.util.*;
-import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
-
-/**
- * <p> A DataSource implementation which can fetch data using JDBC. </p> <p> Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a> for more
- * details. </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 1.3
- */
-public class JdbcDataSource extends
- DataSource<Iterator<Map<String, Object>>> {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- protected Callable<Connection> factory;
-
- private long connLastUsed = 0;
-
- private Connection conn;
-
- private ResultSetIterator resultSetIterator;
-
- private Map<String, Integer> fieldNameVsType = new HashMap<>();
-
- private boolean convertType = false;
-
- private int batchSize = FETCH_SIZE;
-
- private int maxRows = 0;
-
- @Override
- public void init(Context context, Properties initProps) {
- resolveVariables(context, initProps);
- initProps = decryptPwd(context, initProps);
- Object o = initProps.get(CONVERT_TYPE);
- if (o != null)
- convertType = Boolean.parseBoolean(o.toString());
-
- factory = createConnectionFactory(context, initProps);
-
- String bsz = initProps.getProperty("batchSize");
- if (bsz != null) {
- bsz = context.replaceTokens(bsz);
- try {
- batchSize = Integer.parseInt(bsz);
- if (batchSize == -1)
- batchSize = Integer.MIN_VALUE;
- } catch (NumberFormatException e) {
- log.warn("Invalid batch size: " + bsz);
- }
- }
-
- for (Map<String, String> map : context.getAllEntityFields()) {
- String n = map.get(DataImporter.COLUMN);
- String t = map.get(DataImporter.TYPE);
- if ("sint".equals(t) || "integer".equals(t))
- fieldNameVsType.put(n, Types.INTEGER);
- else if ("slong".equals(t) || "long".equals(t))
- fieldNameVsType.put(n, Types.BIGINT);
- else if ("float".equals(t) || "sfloat".equals(t))
- fieldNameVsType.put(n, Types.FLOAT);
- else if ("double".equals(t) || "sdouble".equals(t))
- fieldNameVsType.put(n, Types.DOUBLE);
- else if ("date".equals(t))
- fieldNameVsType.put(n, Types.DATE);
- else if ("boolean".equals(t))
- fieldNameVsType.put(n, Types.BOOLEAN);
- else if ("binary".equals(t))
- fieldNameVsType.put(n, Types.BLOB);
- else
- fieldNameVsType.put(n, Types.VARCHAR);
- }
- }
-
- private Properties decryptPwd(Context context, Properties initProps) {
- String encryptionKey = initProps.getProperty("encryptKeyFile");
- if (initProps.getProperty("password") != null && encryptionKey != null) {
- // this means the password is encrypted and use the file to decode it
- try {
- try (Reader fr = new InputStreamReader(new FileInputStream(encryptionKey), UTF_8)) {
- char[] chars = new char[100];//max 100 char password
- int len = fr.read(chars);
- if (len < 6)
- throw new DataImportHandlerException(SEVERE, "There should be a password of length 6 atleast " + encryptionKey);
- Properties props = new Properties();
- props.putAll(initProps);
- String password = null;
- try {
- password = CryptoKeys.decodeAES(initProps.getProperty("password"), new String(chars, 0, len)).trim();
- } catch (SolrException se) {
- throw new DataImportHandlerException(SEVERE, "Error decoding password", se.getCause());
- }
- props.put("password", password);
- initProps = props;
- }
- } catch (IOException e) {
- throw new DataImportHandlerException(SEVERE, "Could not load encryptKeyFile " + encryptionKey);
- }
- }
- return initProps;
- }
-
- protected Callable<Connection> createConnectionFactory(final Context context,
- final Properties initProps) {
-// final VariableResolver resolver = context.getVariableResolver();
- final String jndiName = initProps.getProperty(JNDI_NAME);
- final String url = initProps.getProperty(URL);
- final String driver = initProps.getProperty(DRIVER);
-
- if (url == null && jndiName == null)
- throw new DataImportHandlerException(SEVERE,
- "JDBC URL or JNDI name has to be specified");
-
- if (driver != null) {
- try {
- DocBuilder.loadClass(driver, context.getSolrCore());
- } catch (ClassNotFoundException e) {
- wrapAndThrow(SEVERE, e, "Could not load driver: " + driver);
- }
- } else {
- if(jndiName == null){
- throw new DataImportHandlerException(SEVERE, "One of driver or jndiName must be specified in the data source");
- }
- }
-
- String s = initProps.getProperty("maxRows");
- if (s != null) {
- maxRows = Integer.parseInt(s);
- }
-
- return factory = new Callable<Connection>() {
- @Override
- public Connection call() throws Exception {
- log.info("Creating a connection for entity "
- + context.getEntityAttribute(DataImporter.NAME) + " with URL: "
- + url);
- long start = System.nanoTime();
- Connection c = null;
-
- if (jndiName != null) {
- c = getFromJndi(initProps, jndiName);
- } else if (url != null) {
- try {
- c = DriverManager.getConnection(url, initProps);
- } catch (SQLException e) {
- // DriverManager does not allow you to use a driver which is not loaded through
- // the class loader of the class which is trying to make the connection.
- // This is a workaround for cases where the user puts the driver jar in the
- // solr.home/lib or solr.home/core/lib directories.
- Driver d = (Driver) DocBuilder.loadClass(driver, context.getSolrCore()).newInstance();
- c = d.connect(url, initProps);
- }
- }
- if (c != null) {
- try {
- initializeConnection(c, initProps);
- } catch (SQLException e) {
- try {
- c.close();
- } catch (SQLException e2) {
- log.warn("Exception closing connection during cleanup", e2);
- }
-
- throw new DataImportHandlerException(SEVERE, "Exception initializing SQL connection", e);
- }
- }
- log.info("Time taken for getConnection(): "
- + TimeUnit.MILLISECONDS.convert(System.nanoTime() - start, TimeUnit.NANOSECONDS));
- return c;
- }
-
- private void initializeConnection(Connection c, final Properties initProps)
- throws SQLException {
- if (Boolean.parseBoolean(initProps.getProperty("readOnly"))) {
- c.setReadOnly(true);
- // Add other sane defaults
- c.setAutoCommit(true);
- c.setTransactionIsolation(Connection.TRANSACTION_READ_UNCOMMITTED);
- c.setHoldability(ResultSet.CLOSE_CURSORS_AT_COMMIT);
- }
- if (!Boolean.parseBoolean(initProps.getProperty("autoCommit"))) {
- c.setAutoCommit(false);
- }
- String transactionIsolation = initProps.getProperty("transactionIsolation");
- if ("TRANSACTION_READ_UNCOMMITTED".equals(transactionIsolation)) {
- c.setTransactionIsolation(Connection.TRANSACTION_READ_UNCOMMITTED);
- } else if ("TRANSACTION_READ_COMMITTED".equals(transactionIsolation)) {
- c.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED);
- } else if ("TRANSACTION_REPEATABLE_READ".equals(transactionIsolation)) {
- c.setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ);
- } else if ("TRANSACTION_SERIALIZABLE".equals(transactionIsolation)) {
- c.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE);
- } else if ("TRANSACTION_NONE".equals(transactionIsolation)) {
- c.setTransactionIsolation(Connection.TRANSACTION_NONE);
- }
- String holdability = initProps.getProperty("holdability");
- if ("CLOSE_CURSORS_AT_COMMIT".equals(holdability)) {
- c.setHoldability(ResultSet.CLOSE_CURSORS_AT_COMMIT);
- } else if ("HOLD_CURSORS_OVER_COMMIT".equals(holdability)) {
- c.setHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT);
- }
- }
-
- private Connection getFromJndi(final Properties initProps, final String jndiName) throws NamingException,
- SQLException {
-
- Connection c = null;
- InitialContext ctx = new InitialContext();
- Object jndival = ctx.lookup(jndiName);
- if (jndival instanceof javax.sql.DataSource) {
- javax.sql.DataSource dataSource = (javax.sql.DataSource) jndival;
- String user = (String) initProps.get("user");
- String pass = (String) initProps.get("password");
- if(user == null || user.trim().equals("")){
- c = dataSource.getConnection();
- } else {
- c = dataSource.getConnection(user, pass);
- }
- } else {
- throw new DataImportHandlerException(SEVERE,
- "the jndi name : '"+jndiName +"' is not a valid javax.sql.DataSource");
- }
- return c;
- }
- };
- }
-
- private void resolveVariables(Context ctx, Properties initProps) {
- for (Map.Entry<Object, Object> entry : initProps.entrySet()) {
- if (entry.getValue() != null) {
- entry.setValue(ctx.replaceTokens((String) entry.getValue()));
- }
- }
- }
-
- @Override
- public Iterator<Map<String, Object>> getData(String query) {
- if (resultSetIterator != null) {
- resultSetIterator.close();
- resultSetIterator = null;
- }
- resultSetIterator = createResultSetIterator(query);
- return resultSetIterator.getIterator();
- }
-
- protected ResultSetIterator createResultSetIterator(String query) {
- return new ResultSetIterator(query);
- }
-
- private void logError(String msg, Exception e) {
- log.warn(msg, e);
- }
-
- protected List<String> readFieldNames(ResultSetMetaData metaData)
- throws SQLException {
- List<String> colNames = new ArrayList<>();
- int count = metaData.getColumnCount();
- for (int i = 0; i < count; i++) {
- colNames.add(metaData.getColumnLabel(i + 1));
- }
- return colNames;
- }
-
- protected class ResultSetIterator {
- private ResultSet resultSet;
-
- private Statement stmt = null;
-
- private List<String> colNames;
-
- private Iterator<Map<String, Object>> rSetIterator;
-
- public ResultSetIterator(String query) {
-
- try {
- Connection c = getConnection();
- stmt = createStatement(c, batchSize, maxRows);
- log.debug("Executing SQL: " + query);
- long start = System.nanoTime();
- resultSet = executeStatement(stmt, query);
- log.trace("Time taken for sql :"
- + TimeUnit.MILLISECONDS.convert(System.nanoTime() - start, TimeUnit.NANOSECONDS));
- setColNames(resultSet);
- } catch (Exception e) {
- close();
- wrapAndThrow(SEVERE, e, "Unable to execute query: " + query);
- return;
- }
- if (resultSet == null) {
- close();
- rSetIterator = new ArrayList<Map<String, Object>>().iterator();
- return;
- }
-
- rSetIterator = createIterator(convertType, fieldNameVsType);
- }
-
-
- protected Statement createStatement(final Connection c, final int batchSize, final int maxRows)
- throws SQLException {
- Statement statement = c.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- statement.setFetchSize(batchSize);
- statement.setMaxRows(maxRows);
- return statement;
- }
-
- protected ResultSet executeStatement(Statement statement, String query) throws SQLException {
- boolean resultSetReturned = statement.execute(query);
- return getNextResultSet(resultSetReturned, statement);
- }
-
- protected ResultSet getNextResultSet(final boolean initialResultSetAvailable, final Statement statement) throws SQLException {
- boolean resultSetAvailable = initialResultSetAvailable;
- while (!resultSetAvailable && statement.getUpdateCount() != -1) {
- resultSetAvailable = statement.getMoreResults();
- }
- if (resultSetAvailable) {
- return statement.getResultSet();
- }
- return null;
- }
-
- protected void setColNames(final ResultSet resultSet) throws SQLException {
- if (resultSet != null) {
- colNames = readFieldNames(resultSet.getMetaData());
- } else {
- colNames = Collections.emptyList();
- }
- }
-
- protected Iterator<Map<String,Object>> createIterator(final boolean convertType,
- final Map<String,Integer> fieldNameVsType) {
- return new Iterator<Map<String,Object>>() {
- @Override
- public boolean hasNext() {
- return hasnext();
- }
-
- @Override
- public Map<String,Object> next() {
- return getARow(convertType, fieldNameVsType);
- }
-
- @Override
- public void remove() {/* do nothing */
- }
- };
- }
-
-
-
- protected Map<String,Object> getARow(boolean convertType, Map<String,Integer> fieldNameVsType) {
- if (getResultSet() == null)
- return null;
- Map<String, Object> result = new HashMap<>();
- for (String colName : getColNames()) {
- try {
- if (!convertType) {
- // Use underlying database's type information except for BigDecimal and BigInteger
- // which cannot be serialized by JavaBin/XML. See SOLR-6165
- Object value = getResultSet().getObject(colName);
- if (value instanceof BigDecimal || value instanceof BigInteger) {
- result.put(colName, value.toString());
- } else {
- result.put(colName, value);
- }
- continue;
- }
-
- Integer type = fieldNameVsType.get(colName);
- if (type == null)
- type = Types.VARCHAR;
- switch (type) {
- case Types.INTEGER:
- result.put(colName, getResultSet().getInt(colName));
- break;
- case Types.FLOAT:
- result.put(colName, getResultSet().getFloat(colName));
- break;
- case Types.BIGINT:
- result.put(colName, getResultSet().getLong(colName));
- break;
- case Types.DOUBLE:
- result.put(colName, getResultSet().getDouble(colName));
- break;
- case Types.DATE:
- result.put(colName, getResultSet().getTimestamp(colName));
- break;
- case Types.BOOLEAN:
- result.put(colName, getResultSet().getBoolean(colName));
- break;
- case Types.BLOB:
- result.put(colName, getResultSet().getBytes(colName));
- break;
- default:
- result.put(colName, getResultSet().getString(colName));
- break;
- }
- } catch (SQLException e) {
- logError("Error reading data ", e);
- wrapAndThrow(SEVERE, e, "Error reading data from database");
- }
- }
- return result;
- }
-
- protected boolean hasnext() {
- if (getResultSet() == null) {
- close();
- return false;
- }
- try {
- if (getResultSet().next()) {
- return true;
- } else {
- closeResultSet();
- setResultSet(getNextResultSet(getStatement().getMoreResults(), getStatement()));
- setColNames(getResultSet());
- return hasnext();
- }
- } catch (SQLException e) {
- close();
- wrapAndThrow(SEVERE,e);
- return false;
- }
- }
-
- protected void close() {
- closeResultSet();
- try {
- if (getStatement() != null)
- getStatement().close();
- } catch (Exception e) {
- logError("Exception while closing statement", e);
- } finally {
- setStatement(null);
- }
- }
-
- protected void closeResultSet() {
- try {
- if (getResultSet() != null) {
- getResultSet().close();
- }
- } catch (Exception e) {
- logError("Exception while closing result set", e);
- } finally {
- setResultSet(null);
- }
- }
-
- protected final Iterator<Map<String,Object>> getIterator() {
- return rSetIterator;
- }
-
-
- protected final Statement getStatement() {
- return stmt;
- }
-
- protected final void setStatement(Statement stmt) {
- this.stmt = stmt;
- }
-
- protected final ResultSet getResultSet() {
- return resultSet;
- }
-
- protected final void setResultSet(ResultSet resultSet) {
- this.resultSet = resultSet;
- }
-
- protected final List<String> getColNames() {
- return colNames;
- }
-
- protected final void setColNames(List<String> colNames) {
- this.colNames = colNames;
- }
-
- }
-
- protected Connection getConnection() throws Exception {
- long currTime = System.nanoTime();
- if (currTime - connLastUsed > CONN_TIME_OUT) {
- synchronized (this) {
- Connection tmpConn = factory.call();
- closeConnection();
- connLastUsed = System.nanoTime();
- return conn = tmpConn;
- }
-
- } else {
- connLastUsed = currTime;
- return conn;
- }
- }
-
- @Override
- protected void finalize() throws Throwable {
- try {
- if(!isClosed){
- log.error("JdbcDataSource was not closed prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!");
- close();
- }
- } finally {
- super.finalize();
- }
- }
-
- private boolean isClosed = false;
-
- @Override
- public void close() {
- if (resultSetIterator != null) {
- resultSetIterator.close();
- }
- try {
- closeConnection();
- } finally {
- isClosed = true;
- }
- }
-
- private void closeConnection() {
- try {
- if (conn != null) {
- try {
- //SOLR-2045
- conn.commit();
- } catch(Exception ex) {
- //ignore.
- }
- conn.close();
- }
- } catch (Exception e) {
- log.error("Ignoring Error when closing connection", e);
- }
- }
-
- private static final long CONN_TIME_OUT = TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS);
-
- private static final int FETCH_SIZE = 500;
-
- public static final String URL = "url";
-
- public static final String JNDI_NAME = "jndiName";
-
- public static final String DRIVER = "driver";
-
- public static final String CONVERT_TYPE = "convertType";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/LineEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/LineEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/LineEntityProcessor.java
deleted file mode 100644
index 0940cbd..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/LineEntityProcessor.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.*;
-import java.util.*;
-import java.util.regex.Pattern;
-
-import org.apache.commons.io.IOUtils;
-
-
-/**
- * <p>
- * An {@link EntityProcessor} instance which can stream lines of text read from a
- * datasource. Options allow lines to be explicitly skipped or included in the index.
- * </p>
- * <p>
- * Attribute summary
- * <ul>
- * <li>url is the required location of the input file. If this value is
- * relative, it assumed to be relative to baseLoc.</li>
- * <li>acceptLineRegex is an optional attribute that if present discards any
- * line which does not match the regExp.</li>
- * <li>skipLineRegex is an optional attribute that is applied after any
- * acceptLineRegex and discards any line which matches this regExp.</li>
- * </ul>
- * <p>
- * Although envisioned for reading lines from a file or url, LineEntityProcessor may also be useful
- * for dealing with change lists, where each line contains filenames which can be used by subsequent entities
- * to parse content from those files.
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 1.4
- * @see Pattern
- */
-public class LineEntityProcessor extends EntityProcessorBase {
- private Pattern acceptLineRegex, skipLineRegex;
- private String url;
- private BufferedReader reader;
-
- /**
- * Parses each of the entity attributes.
- */
- @Override
- public void init(Context context) {
- super.init(context);
- String s;
-
- // init a regex to locate files from the input we want to index
- s = context.getResolvedEntityAttribute(ACCEPT_LINE_REGEX);
- if (s != null) {
- acceptLineRegex = Pattern.compile(s);
- }
-
- // init a regex to locate files from the input to be skipped
- s = context.getResolvedEntityAttribute(SKIP_LINE_REGEX);
- if (s != null) {
- skipLineRegex = Pattern.compile(s);
- }
-
- // the FileName is required.
- url = context.getResolvedEntityAttribute(URL);
- if (url == null) throw
- new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "'"+ URL +"' is a required attribute");
- }
-
-
- /**
- * Reads lines from the url till it finds a lines that matches the
- * optional acceptLineRegex and does not match the optional skipLineRegex.
- *
- * @return A row containing a minimum of one field "rawLine" or null to signal
- * end of file. The rawLine is the as line as returned by readLine()
- * from the url. However transformers can be used to create as
- * many other fields as required.
- */
- @Override
- public Map<String, Object> nextRow() {
- if (reader == null) {
- reader = new BufferedReader((Reader) context.getDataSource().getData(url));
- }
-
- String line;
-
- while ( true ) {
- // read a line from the input file
- try {
- line = reader.readLine();
- }
- catch (IOException exp) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Problem reading from input", exp);
- }
-
- // end of input
- if (line == null) {
- closeResources();
- return null;
- }
-
- // First scan whole line to see if we want it
- if (acceptLineRegex != null && ! acceptLineRegex.matcher(line).find()) continue;
- if (skipLineRegex != null && skipLineRegex.matcher(line).find()) continue;
- // Contruct the 'row' of fields
- Map<String, Object> row = new HashMap<>();
- row.put("rawLine", line);
- return row;
- }
- }
-
- public void closeResources() {
- if (reader != null) {
- IOUtils.closeQuietly(reader);
- }
- reader= null;
- }
-
- @Override
- public void destroy() {
- closeResources();
- super.destroy();
- }
-
- /**
- * Holds the name of entity attribute that will be parsed to obtain
- * the filename containing the changelist.
- */
- public static final String URL = "url";
-
- /**
- * Holds the name of entity attribute that will be parsed to obtain
- * the pattern to be used when checking to see if a line should
- * be returned.
- */
- public static final String ACCEPT_LINE_REGEX = "acceptLineRegex";
-
- /**
- * Holds the name of entity attribute that will be parsed to obtain
- * the pattern to be used when checking to see if a line should
- * be ignored.
- */
- public static final String SKIP_LINE_REGEX = "skipLineRegex";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/LogTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/LogTransformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/LogTransformer.java
deleted file mode 100644
index 66c525e..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/LogTransformer.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.invoke.MethodHandles;
-import java.util.Map;
-
-/**
- * A {@link Transformer} implementation which logs messages in a given template format.
- * <p>
- * Refer to <a href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 1.4
- */
-public class LogTransformer extends Transformer {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- @Override
- public Object transformRow(Map<String, Object> row, Context ctx) {
- String expr = ctx.getEntityAttribute(LOG_TEMPLATE);
- String level = ctx.replaceTokens(ctx.getEntityAttribute(LOG_LEVEL));
-
- if (expr == null || level == null) return row;
-
- if ("info".equals(level)) {
- if (log.isInfoEnabled())
- log.info(ctx.replaceTokens(expr));
- } else if ("trace".equals(level)) {
- if (log.isTraceEnabled())
- log.trace(ctx.replaceTokens(expr));
- } else if ("warn".equals(level)) {
- if (log.isWarnEnabled())
- log.warn(ctx.replaceTokens(expr));
- } else if ("error".equals(level)) {
- if (log.isErrorEnabled())
- log.error(ctx.replaceTokens(expr));
- } else if ("debug".equals(level)) {
- if (log.isDebugEnabled())
- log.debug(ctx.replaceTokens(expr));
- }
-
- return row;
- }
-
- public static final String LOG_TEMPLATE = "logTemplate";
- public static final String LOG_LEVEL = "logLevel";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/MockDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/MockDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/MockDataSource.java
deleted file mode 100644
index 8989ea2..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/MockDataSource.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Properties;
-
-/**
- * <p>
- * A mock DataSource implementation which can be used for testing.
- * </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 1.3
- */
-public class MockDataSource extends
- DataSource<Iterator<Map<String, Object>>> {
-
- private static Map<String, Iterator<Map<String, Object>>> cache = new HashMap<>();
-
- public static void setIterator(String query,
- Iterator<Map<String, Object>> iter) {
- cache.put(query, iter);
- }
-
- public static void clearCache() {
- cache.clear();
- }
-
- @Override
- public void init(Context context, Properties initProps) {
- }
-
- @Override
- public Iterator<Map<String, Object>> getData(String query) {
- return cache.get(query);
- }
-
- @Override
- public void close() {
- cache.clear();
-
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java
deleted file mode 100644
index 349b14e..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-import java.text.NumberFormat;
-import java.text.ParseException;
-import java.text.ParsePosition;
-import java.util.ArrayList;
-import java.util.IllformedLocaleException;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-/**
- * <p>
- * A {@link Transformer} instance which can extract numbers out of strings. It uses
- * {@link NumberFormat} class to parse strings and supports
- * Number, Integer, Currency and Percent styles as supported by
- * {@link NumberFormat} with configurable locales.
- * </p>
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 1.3
- */
-public class NumberFormatTransformer extends Transformer {
-
- @Override
- @SuppressWarnings("unchecked")
- public Object transformRow(Map<String, Object> row, Context context) {
- for (Map<String, String> fld : context.getAllEntityFields()) {
- String style = context.replaceTokens(fld.get(FORMAT_STYLE));
- if (style != null) {
- String column = fld.get(DataImporter.COLUMN);
- String srcCol = fld.get(RegexTransformer.SRC_COL_NAME);
- String localeStr = context.replaceTokens(fld.get(LOCALE));
- if (srcCol == null)
- srcCol = column;
- Locale locale = Locale.ROOT;
- if (localeStr != null) {
- try {
- locale = new Locale.Builder().setLanguageTag(localeStr).build();
- } catch (IllformedLocaleException e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Invalid Locale '" + localeStr + "' specified for field: " + fld, e);
- }
- }
-
- Object val = row.get(srcCol);
- String styleSmall = style.toLowerCase(Locale.ROOT);
-
- if (val instanceof List) {
- List<String> inputs = (List) val;
- List results = new ArrayList();
- for (String input : inputs) {
- try {
- results.add(process(input, styleSmall, locale));
- } catch (ParseException e) {
- throw new DataImportHandlerException(
- DataImportHandlerException.SEVERE,
- "Failed to apply NumberFormat on column: " + column, e);
- }
- }
- row.put(column, results);
- } else {
- if (val == null || val.toString().trim().equals(""))
- continue;
- try {
- row.put(column, process(val.toString(), styleSmall, locale));
- } catch (ParseException e) {
- throw new DataImportHandlerException(
- DataImportHandlerException.SEVERE,
- "Failed to apply NumberFormat on column: " + column, e);
- }
- }
- }
- }
- return row;
- }
-
- private Number process(String val, String style, Locale locale) throws ParseException {
- if (INTEGER.equals(style)) {
- return parseNumber(val, NumberFormat.getIntegerInstance(locale));
- } else if (NUMBER.equals(style)) {
- return parseNumber(val, NumberFormat.getNumberInstance(locale));
- } else if (CURRENCY.equals(style)) {
- return parseNumber(val, NumberFormat.getCurrencyInstance(locale));
- } else if (PERCENT.equals(style)) {
- return parseNumber(val, NumberFormat.getPercentInstance(locale));
- }
-
- return null;
- }
-
- private Number parseNumber(String val, NumberFormat numFormat) throws ParseException {
- ParsePosition parsePos = new ParsePosition(0);
- Number num = numFormat.parse(val, parsePos);
- if (parsePos.getIndex() != val.length()) {
- throw new ParseException("illegal number format", parsePos.getIndex());
- }
- return num;
- }
-
- public static final String FORMAT_STYLE = "formatStyle";
-
- public static final String LOCALE = "locale";
-
- public static final String NUMBER = "number";
-
- public static final String PERCENT = "percent";
-
- public static final String INTEGER = "integer";
-
- public static final String CURRENCY = "currency";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/PlainTextEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/PlainTextEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/PlainTextEntityProcessor.java
deleted file mode 100644
index c75608c..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/PlainTextEntityProcessor.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-import static org.apache.solr.handler.dataimport.XPathEntityProcessor.URL;
-import org.apache.commons.io.IOUtils;
-
-import java.io.IOException;
-import java.io.Reader;
-import java.io.StringWriter;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * <p>An implementation of {@link EntityProcessor} which reads data from a url/file and give out a row which contains one String
- * value. The name of the field is 'plainText'.
- *
- * @since solr 1.4
- */
-public class PlainTextEntityProcessor extends EntityProcessorBase {
- private boolean ended = false;
-
- @Override
- public void init(Context context) {
- super.init(context);
- ended = false;
- }
-
- @Override
- public Map<String, Object> nextRow() {
- if (ended) return null;
- DataSource<Reader> ds = context.getDataSource();
- String url = context.replaceTokens(context.getEntityAttribute(URL));
- Reader r = null;
- try {
- r = ds.getData(url);
- } catch (Exception e) {
- wrapAndThrow(SEVERE, e, "Exception reading url : " + url);
- }
- StringWriter sw = new StringWriter();
- char[] buf = new char[1024];
- while (true) {
- int len = 0;
- try {
- len = r.read(buf);
- } catch (IOException e) {
- IOUtils.closeQuietly(r);
- wrapAndThrow(SEVERE, e, "Exception reading url : " + url);
- }
- if (len <= 0) break;
- sw.append(new String(buf, 0, len));
- }
- Map<String, Object> row = new HashMap<>();
- row.put(PLAIN_TEXT, sw.toString());
- ended = true;
- IOUtils.closeQuietly(r);
- return row;
- }
-
- public static final String PLAIN_TEXT = "plainText";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RegexTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RegexTransformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RegexTransformer.java
deleted file mode 100644
index 7a919de..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RegexTransformer.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.lang.invoke.MethodHandles;
-import java.util.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * <p>
- * A {@link Transformer} implementation which uses Regular Expressions to extract, split
- * and replace data in fields.
- * </p>
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 1.3
- * @see Pattern
- */
-public class RegexTransformer extends Transformer {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- @Override
- @SuppressWarnings("unchecked")
- public Map<String, Object> transformRow(Map<String, Object> row,
- Context ctx) {
- List<Map<String, String>> fields = ctx.getAllEntityFields();
- for (Map<String, String> field : fields) {
- String col = field.get(DataImporter.COLUMN);
- String reStr = ctx.replaceTokens(field.get(REGEX));
- String splitBy = ctx.replaceTokens(field.get(SPLIT_BY));
- String replaceWith = ctx.replaceTokens(field.get(REPLACE_WITH));
- String groupNames = ctx.replaceTokens(field.get(GROUP_NAMES));
- if (reStr != null || splitBy != null) {
- String srcColName = field.get(SRC_COL_NAME);
- if (srcColName == null) {
- srcColName = col;
- }
- Object tmpVal = row.get(srcColName);
- if (tmpVal == null)
- continue;
-
- if (tmpVal instanceof List) {
- List<String> inputs = (List<String>) tmpVal;
- List results = new ArrayList();
- Map<String,List> otherVars= null;
- for (String input : inputs) {
- Object o = process(col, reStr, splitBy, replaceWith, input, groupNames);
- if (o != null){
- if (o instanceof Map) {
- Map map = (Map) o;
- for (Object e : map.entrySet()) {
- Map.Entry<String ,Object> entry = (Map.Entry<String, Object>) e;
- List l = results;
- if(!col.equals(entry.getKey())){
- if(otherVars == null) otherVars = new HashMap<>();
- l = otherVars.get(entry.getKey());
- if(l == null){
- l = new ArrayList();
- otherVars.put(entry.getKey(), l);
- }
- }
- if (entry.getValue() instanceof Collection) {
- l.addAll((Collection) entry.getValue());
- } else {
- l.add(entry.getValue());
- }
- }
- } else {
- if (o instanceof Collection) {
- results.addAll((Collection) o);
- } else {
- results.add(o);
- }
- }
- }
- }
- row.put(col, results);
- if(otherVars != null) row.putAll(otherVars);
- } else {
- String value = tmpVal.toString();
- Object o = process(col, reStr, splitBy, replaceWith, value, groupNames);
- if (o != null){
- if (o instanceof Map) {
- row.putAll((Map) o);
- } else{
- row.put(col, o);
- }
- }
- }
- }
- }
- return row;
- }
-
- private Object process(String col, String reStr, String splitBy,
- String replaceWith, String value, String groupNames) {
- if (splitBy != null) {
- return readBySplit(splitBy, value);
- } else if (replaceWith != null) {
- Pattern p = getPattern(reStr);
- Matcher m = p.matcher(value);
- return m.find() ? m.replaceAll(replaceWith) : value;
- } else {
- return readfromRegExp(reStr, value, col, groupNames);
- }
- }
-
- @SuppressWarnings("unchecked")
- private List<String> readBySplit(String splitBy, String value) {
- String[] vals = value.split(splitBy);
- List<String> l = new ArrayList<>();
- l.addAll(Arrays.asList(vals));
- return l;
- }
-
- @SuppressWarnings("unchecked")
- private Object readfromRegExp(String reStr, String value, String columnName, String gNames) {
- String[] groupNames = null;
- if(gNames != null && gNames.trim().length() >0){
- groupNames = gNames.split(",");
- }
- Pattern regexp = getPattern(reStr);
- Matcher m = regexp.matcher(value);
- if (m.find() && m.groupCount() > 0) {
- if (m.groupCount() > 1) {
- List l = null;
- Map<String ,String > map = null;
- if(groupNames == null){
- l = new ArrayList();
- } else {
- map = new HashMap<>();
- }
- for (int i = 1; i <= m.groupCount(); i++) {
- try {
- if(l != null){
- l.add(m.group(i));
- } else if (map != null ){
- if(i <= groupNames.length){
- String nameOfGroup = groupNames[i-1];
- if(nameOfGroup != null && nameOfGroup.trim().length() >0){
- map.put(nameOfGroup, m.group(i));
- }
- }
- }
- } catch (Exception e) {
- log.warn("Parsing failed for field : " + columnName, e);
- }
- }
- return l == null ? map: l;
- } else {
- return m.group(1);
- }
- }
-
- return null;
- }
-
- private Pattern getPattern(String reStr) {
- Pattern result = PATTERN_CACHE.get(reStr);
- if (result == null) {
- PATTERN_CACHE.put(reStr, result = Pattern.compile(reStr));
- }
- return result;
- }
-
- private HashMap<String, Pattern> PATTERN_CACHE = new HashMap<>();
-
- public static final String REGEX = "regex";
-
- public static final String REPLACE_WITH = "replaceWith";
-
- public static final String SPLIT_BY = "splitBy";
-
- public static final String SRC_COL_NAME = "sourceColName";
-
- public static final String GROUP_NAMES = "groupNames";
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RequestInfo.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RequestInfo.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RequestInfo.java
deleted file mode 100644
index d3f1a56..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RequestInfo.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.solr.common.util.ContentStream;
-import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.request.SolrQueryRequest;
-
-public class RequestInfo {
- private final String command;
- private final boolean debug;
- private final boolean syncMode;
- private final boolean commit;
- private final boolean optimize;
- private final int start;
- private final long rows;
- private final boolean clean;
- private final List<String> entitiesToRun;
- private final Map<String,Object> rawParams;
- private final String configFile;
- private final String dataConfig;
- private final SolrQueryRequest request;
-
- //TODO: find a different home for these two...
- private final ContentStream contentStream;
- private final DebugInfo debugInfo;
-
- public RequestInfo(SolrQueryRequest request, Map<String,Object> requestParams, ContentStream stream) {
- this.request = request;
- this.contentStream = stream;
- if (requestParams.containsKey("command")) {
- command = (String) requestParams.get("command");
- } else {
- command = null;
- }
- boolean debugMode = StrUtils.parseBool((String) requestParams.get("debug"), false);
- if (debugMode) {
- debug = true;
- debugInfo = new DebugInfo(requestParams);
- } else {
- debug = false;
- debugInfo = null;
- }
- if (requestParams.containsKey("clean")) {
- clean = StrUtils.parseBool( (String) requestParams.get("clean"), true);
- } else if (DataImporter.DELTA_IMPORT_CMD.equals(command) || DataImporter.IMPORT_CMD.equals(command)) {
- clean = false;
- } else {
- clean = debug ? false : true;
- }
- optimize = StrUtils.parseBool((String) requestParams.get("optimize"), false);
- if(optimize) {
- commit = true;
- } else {
- commit = StrUtils.parseBool( (String) requestParams.get("commit"), (debug ? false : true));
- }
- if (requestParams.containsKey("rows")) {
- rows = Integer.parseInt((String) requestParams.get("rows"));
- } else {
- rows = debug ? 10 : Long.MAX_VALUE;
- }
-
- if (requestParams.containsKey("start")) {
- start = Integer.parseInt((String) requestParams.get("start"));
- } else {
- start = 0;
- }
- syncMode = StrUtils.parseBool((String) requestParams.get("synchronous"), false);
-
- Object o = requestParams.get("entity");
- List<String> modifiableEntities = null;
- if(o != null) {
- if (o instanceof String) {
- modifiableEntities = new ArrayList<>();
- modifiableEntities.add((String) o);
- } else if (o instanceof List<?>) {
- @SuppressWarnings("unchecked")
- List<String> modifiableEntities1 = new ArrayList<>((List<String>) o);
- modifiableEntities = modifiableEntities1;
- }
- entitiesToRun = Collections.unmodifiableList(modifiableEntities);
- } else {
- entitiesToRun = null;
- }
- String configFileParam = (String) requestParams.get("config");
- configFile = configFileParam;
- String dataConfigParam = (String) requestParams.get("dataConfig");
- if (dataConfigParam != null && dataConfigParam.trim().length() == 0) {
- // Empty data-config param is not valid, change it to null
- dataConfigParam = null;
- }
- dataConfig = dataConfigParam;
- this.rawParams = Collections.unmodifiableMap(new HashMap<>(requestParams));
- }
-
- public String getCommand() {
- return command;
- }
-
- public boolean isDebug() {
- return debug;
- }
-
- public boolean isSyncMode() {
- return syncMode;
- }
-
- public boolean isCommit() {
- return commit;
- }
-
- public boolean isOptimize() {
- return optimize;
- }
-
- public int getStart() {
- return start;
- }
-
- public long getRows() {
- return rows;
- }
-
- public boolean isClean() {
- return clean;
- }
- /**
- * Returns null if we are to run all entities, otherwise just run the entities named in the list.
- */
- public List<String> getEntitiesToRun() {
- return entitiesToRun;
- }
-
- public String getDataConfig() {
- return dataConfig;
- }
-
- public Map<String,Object> getRawParams() {
- return rawParams;
- }
-
- public ContentStream getContentStream() {
- return contentStream;
- }
-
- public DebugInfo getDebugInfo() {
- return debugInfo;
- }
-
- public String getConfigFile() {
- return configFile;
- }
-
- public SolrQueryRequest getRequest() {
- return request;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ScriptTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ScriptTransformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ScriptTransformer.java
deleted file mode 100644
index 165d76d..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ScriptTransformer.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-
-import java.util.Map;
-
-import javax.script.Invocable;
-import javax.script.ScriptEngine;
-import javax.script.ScriptEngineManager;
-import javax.script.ScriptException;
-
-/**
- * <p>
- * A {@link Transformer} instance capable of executing functions written in scripting
- * languages as a {@link Transformer} instance.
- * </p>
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 1.3
- */
-public class ScriptTransformer extends Transformer {
- private Invocable engine;
- private String functionName;
-
- @Override
- public Object transformRow(Map<String, Object> row, Context context) {
- try {
- if (engine == null)
- initEngine(context);
- if (engine == null)
- return row;
- return engine.invokeFunction(functionName, new Object[]{row, context});
- } catch (DataImportHandlerException e) {
- throw e;
- } catch (Exception e) {
- wrapAndThrow(SEVERE,e, "Error invoking script for entity " + context.getEntityAttribute("name"));
- }
- //will not reach here
- return null;
- }
-
- private void initEngine(Context context) {
- String scriptText = context.getScript();
- String scriptLang = context.getScriptLanguage();
- if (scriptText == null) {
- throw new DataImportHandlerException(SEVERE,
- "<script> tag is not present under <dataConfig>");
- }
- ScriptEngineManager scriptEngineMgr = new ScriptEngineManager();
- ScriptEngine scriptEngine = scriptEngineMgr.getEngineByName(scriptLang);
- if (scriptEngine == null) {
- throw new DataImportHandlerException(SEVERE,
- "Cannot load Script Engine for language: " + scriptLang);
- }
- if (scriptEngine instanceof Invocable) {
- engine = (Invocable) scriptEngine;
- } else {
- throw new DataImportHandlerException(SEVERE,
- "The installed ScriptEngine for: " + scriptLang
- + " does not implement Invocable. Class is "
- + scriptEngine.getClass().getName());
- }
- try {
- scriptEngine.eval(scriptText);
- } catch (ScriptException e) {
- wrapAndThrow(SEVERE, e, "'eval' failed with language: " + scriptLang
- + " and script: \n" + scriptText);
- }
- }
-
- public void setFunctionName(String methodName) {
- this.functionName = methodName;
- }
-
- public String getFunctionName() {
- return functionName;
- }
-
-}
[05/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDataConfig.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDataConfig.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDataConfig.java
deleted file mode 100644
index 7a56fd2..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDataConfig.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.handler.dataimport.config.DIHConfiguration;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.w3c.dom.Document;
-import org.xml.sax.InputSource;
-
-import javax.xml.parsers.DocumentBuilderFactory;
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * <p>
- * Test for DataConfig
- * </p>
- *
- *
- * @since solr 1.3
- */
-public class TestDataConfig extends AbstractDataImportHandlerTestCase {
- @BeforeClass
- public static void beforeClass() throws Exception {
- initCore("dataimport-nodatasource-solrconfig.xml", "dataimport-schema.xml");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testDataConfigWithDataSource() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(loadDataConfig("data-config-with-datasource.xml"));
-
- assertQ(req("id:1"), "//*[@numFound='1']");
- }
-
- @Test
- public void testBasic() throws Exception {
- javax.xml.parsers.DocumentBuilder builder = DocumentBuilderFactory
- .newInstance().newDocumentBuilder();
- Document doc = builder.parse(new InputSource(new StringReader(xml)));
- DataImporter di = new DataImporter();
- DIHConfiguration dc = di.readFromXml(doc);
- assertEquals("atrimlisting", dc.getEntities().get(0).getName());
- }
-
- private static final String xml = "<dataConfig>\n"
- + "\t<document name=\"autos\" >\n"
- + "\t\t<entity name=\"atrimlisting\" pk=\"acode\"\n"
- + "\t\t\tquery=\"select acode,make,model,year,msrp,category,image,izmo_image_url,price_range_low,price_range_high,invoice_range_low,invoice_range_high from atrimlisting\"\n"
- + "\t\t\tdeltaQuery=\"select acode from atrimlisting where last_modified > '${indexer.last_index_time}'\">\n"
- +
-
- "\t\t</entity>\n" +
-
- "\t</document>\n" + "</dataConfig>";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java
deleted file mode 100644
index a1e85d7..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.junit.Test;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-
-/**
- * <p>
- * Test for DateFormatTransformer
- * </p>
- *
- *
- * @since solr 1.3
- */
-public class TestDateFormatTransformer extends AbstractDataImportHandlerTestCase {
-
- @Test
- @SuppressWarnings("unchecked")
- public void testTransformRow_SingleRow() throws Exception {
- List<Map<String, String>> fields = new ArrayList<>();
- fields.add(createMap(DataImporter.COLUMN, "lastModified"));
- fields.add(createMap(DataImporter.COLUMN,
- "dateAdded", RegexTransformer.SRC_COL_NAME, "lastModified",
- DateFormatTransformer.DATE_TIME_FMT, "${xyz.myDateFormat}"));
-
- SimpleDateFormat format = new SimpleDateFormat("MM/dd/yyyy", Locale.ROOT);
- Date now = format.parse(format.format(new Date()));
-
- Map<String,Object> row = createMap("lastModified", format.format(now));
-
- VariableResolver resolver = new VariableResolver();
- resolver.addNamespace("e", row);
- resolver.addNamespace("xyz", createMap("myDateFormat", "MM/dd/yyyy"));
-
- Context context = getContext(null, resolver,
- null, Context.FULL_DUMP, fields, null);
- new DateFormatTransformer().transformRow(row, context);
- assertEquals(now, row.get("dateAdded"));
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testTransformRow_MultipleRows() throws Exception {
- List<Map<String, String>> fields = new ArrayList<>();
- fields.add(createMap(DataImporter.COLUMN, "lastModified"));
- fields.add(createMap(DataImporter.COLUMN,
- "dateAdded", RegexTransformer.SRC_COL_NAME, "lastModified",
- DateFormatTransformer.DATE_TIME_FMT, "MM/dd/yyyy hh:mm:ss.SSS"));
-
- SimpleDateFormat format = new SimpleDateFormat("MM/dd/yyyy hh:mm:ss.SSS", Locale.ROOT);
- Date now1 = format.parse(format.format(new Date()));
- Date now2 = format.parse(format.format(new Date()));
-
- Map<String,Object> row = new HashMap<>();
- List<String> list = new ArrayList<>();
- list.add(format.format(now1));
- list.add(format.format(now2));
- row.put("lastModified", list);
-
- VariableResolver resolver = new VariableResolver();
- resolver.addNamespace("e", row);
-
- Context context = getContext(null, resolver,
- null, Context.FULL_DUMP, fields, null);
- new DateFormatTransformer().transformRow(row, context);
- List<Object> output = new ArrayList<>();
- output.add(now1);
- output.add(now2);
- assertEquals(output, row.get("dateAdded"));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java
deleted file mode 100644
index 7f7278e..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java
+++ /dev/null
@@ -1,335 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.handler.dataimport.config.DIHConfiguration;
-import org.apache.solr.handler.dataimport.config.Entity;
-
-import org.junit.After;
-import org.junit.Test;
-
-import java.util.*;
-
-/**
- * <p>
- * Test for DocBuilder
- * </p>
- *
- *
- * @since solr 1.3
- */
-// See: https://issues.apache.org/jira/browse/SOLR-12028 Tests cannot remove files on Windows machines occasionally
-public class TestDocBuilder extends AbstractDataImportHandlerTestCase {
-
- @Override
- @After
- public void tearDown() throws Exception {
- MockDataSource.clearCache();
- MockStringDataSource.clearCache();
- super.tearDown();
- }
-
- @Test
- public void loadClass() throws Exception {
- @SuppressWarnings("unchecked")
- Class<Transformer> clz = DocBuilder.loadClass("RegexTransformer", null);
- assertNotNull(clz);
- }
-
- @Test
- public void singleEntityNoRows() {
- DataImporter di = new DataImporter();
- di.loadAndInit(dc_singleEntity);
- DIHConfiguration cfg = di.getConfig();
- Entity ent = cfg.getEntities().get(0);
- MockDataSource.setIterator("select * from x", new ArrayList<Map<String, Object>>().iterator());
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals(Boolean.TRUE, swi.deleteAllCalled);
- assertEquals(Boolean.TRUE, swi.commitCalled);
- assertEquals(Boolean.TRUE, swi.finishCalled);
- assertEquals(0, swi.docs.size());
- assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
- assertEquals(0, di.getDocBuilder().importStatistics.docCount.get());
- assertEquals(0, di.getDocBuilder().importStatistics.rowsCount.get());
- }
-
- @Test
- public void testDeltaImportNoRows_MustNotCommit() {
- DataImporter di = new DataImporter();
- di.loadAndInit(dc_deltaConfig);
- redirectTempProperties(di);
-
- DIHConfiguration cfg = di.getConfig();
- Entity ent = cfg.getEntities().get(0);
- MockDataSource.setIterator("select * from x", new ArrayList<Map<String, Object>>().iterator());
- MockDataSource.setIterator("select id from x", new ArrayList<Map<String, Object>>().iterator());
- RequestInfo rp = new RequestInfo(null, createMap("command", "delta-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals(Boolean.FALSE, swi.deleteAllCalled);
- assertEquals(Boolean.FALSE, swi.commitCalled);
- assertEquals(Boolean.TRUE, swi.finishCalled);
- assertEquals(0, swi.docs.size());
- assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
- assertEquals(0, di.getDocBuilder().importStatistics.docCount.get());
- assertEquals(0, di.getDocBuilder().importStatistics.rowsCount.get());
- }
-
- @Test
- public void singleEntityOneRow() {
- DataImporter di = new DataImporter();
- di.loadAndInit(dc_singleEntity);
- DIHConfiguration cfg = di.getConfig();
- Entity ent = cfg.getEntities().get(0);
- List<Map<String, Object>> l = new ArrayList<>();
- l.add(createMap("id", 1, "desc", "one"));
- MockDataSource.setIterator("select * from x", l.iterator());
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals(Boolean.TRUE, swi.deleteAllCalled);
- assertEquals(Boolean.TRUE, swi.commitCalled);
- assertEquals(Boolean.TRUE, swi.finishCalled);
- assertEquals(1, swi.docs.size());
- assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
- assertEquals(1, di.getDocBuilder().importStatistics.docCount.get());
- assertEquals(1, di.getDocBuilder().importStatistics.rowsCount.get());
-
- for (int i = 0; i < l.size(); i++) {
- Map<String, Object> map = l.get(i);
- SolrInputDocument doc = swi.docs.get(i);
- for (Map.Entry<String, Object> entry : map.entrySet()) {
- assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
- }
- }
- }
-
- @Test
- public void testImportCommand() {
- DataImporter di = new DataImporter();
- di.loadAndInit(dc_singleEntity);
- DIHConfiguration cfg = di.getConfig();
- Entity ent = cfg.getEntities().get(0);
- List<Map<String, Object>> l = new ArrayList<>();
- l.add(createMap("id", 1, "desc", "one"));
- MockDataSource.setIterator("select * from x", l.iterator());
- RequestInfo rp = new RequestInfo(null, createMap("command", "import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals(Boolean.FALSE, swi.deleteAllCalled);
- assertEquals(Boolean.TRUE, swi.commitCalled);
- assertEquals(Boolean.TRUE, swi.finishCalled);
- assertEquals(1, swi.docs.size());
- assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
- assertEquals(1, di.getDocBuilder().importStatistics.docCount.get());
- assertEquals(1, di.getDocBuilder().importStatistics.rowsCount.get());
-
- for (int i = 0; i < l.size(); i++) {
- Map<String, Object> map = (Map<String, Object>) l.get(i);
- SolrInputDocument doc = swi.docs.get(i);
- for (Map.Entry<String, Object> entry : map.entrySet()) {
- assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
- }
- }
- }
-
- @Test
- public void singleEntityMultipleRows() {
- DataImporter di = new DataImporter();
- di.loadAndInit(dc_singleEntity);
- DIHConfiguration cfg = di.getConfig();
- Entity ent = cfg.getEntities().get(0);
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- List<Map<String, Object>> l = new ArrayList<>();
- l.add(createMap("id", 1, "desc", "one"));
- l.add(createMap("id", 2, "desc", "two"));
- l.add(createMap("id", 3, "desc", "three"));
-
- MockDataSource.setIterator("select * from x", l.iterator());
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals(Boolean.TRUE, swi.deleteAllCalled);
- assertEquals(Boolean.TRUE, swi.commitCalled);
- assertEquals(Boolean.TRUE, swi.finishCalled);
- assertEquals(3, swi.docs.size());
- for (int i = 0; i < l.size(); i++) {
- Map<String, Object> map = (Map<String, Object>) l.get(i);
- SolrInputDocument doc = swi.docs.get(i);
- for (Map.Entry<String, Object> entry : map.entrySet()) {
- assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
- }
- assertEquals(map.get("desc"), doc.getFieldValue("desc_s"));
- }
- assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
- assertEquals(3, di.getDocBuilder().importStatistics.docCount.get());
- assertEquals(3, di.getDocBuilder().importStatistics.rowsCount.get());
- }
-
- @Test
- public void templateXPath() {
- DataImporter di = new DataImporter();
- di.loadAndInit(dc_variableXpath);
- DIHConfiguration cfg = di.getConfig();
-
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- List<Map<String, Object>> l = new ArrayList<>();
- l.add(createMap("id", 1, "name", "iphone", "manufacturer", "Apple"));
- l.add(createMap("id", 2, "name", "ipad", "manufacturer", "Apple"));
- l.add(createMap("id", 3, "name", "pixel", "manufacturer", "Google"));
-
- MockDataSource.setIterator("select * from x", l.iterator());
-
- List<Map<String,Object>> nestedData = new ArrayList<>();
- nestedData.add(createMap("founded", "Cupertino, California, U.S", "year", "1976", "year2", "1976"));
- nestedData.add(createMap("founded", "Cupertino, California, U.S", "year", "1976", "year2", "1976"));
- nestedData.add(createMap("founded", "Menlo Park, California, U.S", "year", "1998", "year2", "1998"));
-
- MockStringDataSource.setData("companies.xml", xml_attrVariableXpath);
- MockStringDataSource.setData("companies2.xml", xml_variableXpath);
- MockStringDataSource.setData("companies3.xml", xml_variableForEach);
-
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals(Boolean.TRUE, swi.deleteAllCalled);
- assertEquals(Boolean.TRUE, swi.commitCalled);
- assertEquals(Boolean.TRUE, swi.finishCalled);
- assertEquals(3, swi.docs.size());
- for (int i = 0; i < l.size(); i++) {
- SolrInputDocument doc = swi.docs.get(i);
-
- Map<String, Object> map = l.get(i);
- for (Map.Entry<String, Object> entry : map.entrySet()) {
- assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
- }
-
- map = nestedData.get(i);
- for (Map.Entry<String, Object> entry : map.entrySet()) {
- assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
- }
- }
- assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
- assertEquals(3, di.getDocBuilder().importStatistics.docCount.get());
- }
-
- static class SolrWriterImpl extends SolrWriter {
- List<SolrInputDocument> docs = new ArrayList<>();
-
- Boolean deleteAllCalled = Boolean.FALSE;
-
- Boolean commitCalled = Boolean.FALSE;
-
- Boolean finishCalled = Boolean.FALSE;
-
- public SolrWriterImpl() {
- super(null, null);
- }
-
- @Override
- public boolean upload(SolrInputDocument doc) {
- return docs.add(doc);
- }
-
- @Override
- public void doDeleteAll() {
- deleteAllCalled = Boolean.TRUE;
- }
-
- @Override
- public void commit(boolean b) {
- commitCalled = Boolean.TRUE;
- }
-
- @Override
- public void close() {
- finishCalled = Boolean.TRUE;
- }
- }
-
- public static final String dc_singleEntity = "<dataConfig>\n"
- + "<dataSource type=\"MockDataSource\"/>\n"
- + " <document name=\"X\" >\n"
- + " <entity name=\"x\" query=\"select * from x\">\n"
- + " <field column=\"id\"/>\n"
- + " <field column=\"desc\"/>\n"
- + " <field column=\"desc\" name=\"desc_s\" />" + " </entity>\n"
- + " </document>\n" + "</dataConfig>";
-
- public static final String dc_deltaConfig = "<dataConfig>\n"
- + "<dataSource type=\"MockDataSource\"/>\n"
- + " <document name=\"X\" >\n"
- + " <entity name=\"x\" query=\"select * from x\" deltaQuery=\"select id from x\">\n"
- + " <field column=\"id\"/>\n"
- + " <field column=\"desc\"/>\n"
- + " <field column=\"desc\" name=\"desc_s\" />" + " </entity>\n"
- + " </document>\n" + "</dataConfig>";
-
- public static final String dc_variableXpath = "<dataConfig>\n"
- + "<dataSource type=\"MockDataSource\"/>\n"
- + "<dataSource name=\"xml\" type=\"MockStringDataSource\"/>\n"
- + " <document name=\"X\" >\n"
- + " <entity name=\"x\" query=\"select * from x\">\n"
- + " <field column=\"id\"/>\n"
- + " <field column=\"name\"/>\n"
- + " <field column=\"manufacturer\"/>"
- + " <entity name=\"c1\" url=\"companies.xml\" dataSource=\"xml\" forEach=\"/companies/company\" processor=\"XPathEntityProcessor\">"
- + " <field column=\"year\" xpath=\"/companies/company/year[@name='p_${x.manufacturer}_s']\" />"
- + " </entity>"
- + " <entity name=\"c2\" url=\"companies2.xml\" dataSource=\"xml\" forEach=\"/companies/company\" processor=\"XPathEntityProcessor\">"
- + " <field column=\"founded\" xpath=\"/companies/company/p_${x.manufacturer}_s/founded\" />"
- + " </entity>"
- + " <entity name=\"c3\" url=\"companies3.xml\" dataSource=\"xml\" forEach=\"/companies/${x.manufacturer}\" processor=\"XPathEntityProcessor\">"
- + " <field column=\"year2\" xpath=\"/companies/${x.manufacturer}/year\" />"
- + " </entity>"
- + " </entity>\n"
- + " </document>\n" + "</dataConfig>";
-
-
- public static final String xml_variableForEach = "<companies>\n" +
- "\t<Apple>\n" +
- "\t\t<year>1976</year>\n" +
- "\t</Apple>\n" +
- "\t<Google>\n" +
- "\t\t<year>1998</year>\n" +
- "\t</Google>\n" +
- "</companies>";
-
- public static final String xml_variableXpath = "<companies>\n" +
- "\t<company>\n" +
- "\t\t<p_Apple_s>\n" +
- "\t\t\t<founded>Cupertino, California, U.S</founded>\n" +
- "\t\t</p_Apple_s>\t\t\n" +
- "\t</company>\n" +
- "\t<company>\n" +
- "\t\t<p_Google_s>\n" +
- "\t\t\t<founded>Menlo Park, California, U.S</founded>\n" +
- "\t\t</p_Google_s>\n" +
- "\t</company>\n" +
- "</companies>";
-
- public static final String xml_attrVariableXpath = "<companies>\n" +
- "\t<company>\n" +
- "\t\t<year name='p_Apple_s'>1976</year>\n" +
- "\t</company>\n" +
- "\t<company>\n" +
- "\t\t<year name='p_Google_s'>1998</year>\t\t\n" +
- "\t</company>\n" +
- "</companies>";
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java
deleted file mode 100644
index 27865cd..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java
+++ /dev/null
@@ -1,433 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.solr.request.LocalSolrQueryRequest;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.nio.charset.StandardCharsets;
-
-/**
- * <p>
- * Test for DocBuilder using the test harness
- * </p>
- *
- *
- * @since solr 1.3
- */
-public class TestDocBuilder2 extends AbstractDataImportHandlerTestCase {
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testSingleEntity() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(loadDataConfig("single-entity-data-config.xml"));
-
- assertQ(req("id:1"), "//*[@numFound='1']");
-
- assertTrue("Update request processor processAdd was not called", TestUpdateRequestProcessor.processAddCalled);
- assertTrue("Update request processor processCommit was not callled", TestUpdateRequestProcessor.processCommitCalled);
- assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testSingleEntity_CaseInsensitive() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desC", "one"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(DATA_CONFIG_WITH_CASE_INSENSITIVE_FIELDS);
-
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertTrue("Start event listener was not called", StartEventListener.executed);
- assertTrue("End event listener was not called", EndEventListener.executed);
- assertTrue("Update request processor processAdd was not called", TestUpdateRequestProcessor.processAddCalled);
- assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
- }
-
- @Test
- public void testErrorHandler() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "FORCE_ERROR", "true"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(DATA_CONFIG_WITH_ERROR_HANDLER);
-
- assertTrue("Error event listener was not called", ErrorEventListener.executed);
- assertTrue(ErrorEventListener.lastException.getMessage().contains("ForcedException"));
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testDynamicFields() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(DATA_CONFIG_WITH_DYNAMIC_TRANSFORMER);
-
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("dynamic_s:test"), "//*[@numFound='1']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testRequestParamsAsVariable() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "101", "desc", "ApacheSolr"));
- MockDataSource.setIterator("select * from books where category='search'", rows.iterator());
-
- LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import",
- "debug", "on", "clean", "true", "commit", "true",
- "category", "search",
- "dataConfig", REQUEST_PARAM_AS_VARIABLE);
- h.query("/dataimport", request);
- assertQ(req("desc:ApacheSolr"), "//*[@numFound='1']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testDynamicFieldNames() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("mypk", "101", "text", "ApacheSolr"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import",
- "debug", "on", "clean", "true", "commit", "true",
- "dataConfig", DATA_CONFIG_WITH_DYNAMIC_FIELD_NAMES);
- h.query("/dataimport", request);
- assertQ(req("id:101"), "//*[@numFound='1']", "//*[@name='101_s']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testRequestParamsAsFieldName() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("mypk", "101", "text", "ApacheSolr"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import",
- "debug", "on", "clean", "true", "commit", "true",
- "mypk", "id", "text", "desc",
- "dataConfig", DATA_CONFIG_WITH_TEMPLATIZED_FIELD_NAMES);
- h.query("/dataimport", request);
- assertQ(req("id:101"), "//*[@numFound='1']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testContext() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(loadDataConfig("data-config-with-transformer.xml"));
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testSkipDoc() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- rows.add(createMap("id", "2", "desc", "two", DocBuilder.SKIP_DOC, "true"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(DATA_CONFIG_WITH_DYNAMIC_TRANSFORMER);
-
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("id:2"), "//*[@numFound='0']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testSkipRow() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- rows.add(createMap("id", "2", "desc", "two", DocBuilder.SKIP_ROW, "true"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(DATA_CONFIG_WITH_DYNAMIC_TRANSFORMER);
-
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("id:2"), "//*[@numFound='0']");
-
- MockDataSource.clearCache();
-
- rows = new ArrayList();
- rows.add(createMap("id", "3", "desc", "one"));
- rows.add(createMap("id", "4", "desc", "two"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- rows = new ArrayList();
- rows.add(createMap("name_s", "abcd"));
- MockDataSource.setIterator("3", rows.iterator());
-
- rows = new ArrayList();
- rows.add(createMap("name_s", "xyz", DocBuilder.SKIP_ROW, "true"));
- MockDataSource.setIterator("4", rows.iterator());
-
- runFullImport(DATA_CONFIG_WITH_TWO_ENTITIES);
- assertQ(req("id:3"), "//*[@numFound='1']");
- assertQ(req("id:4"), "//*[@numFound='1']");
- assertQ(req("name_s:abcd"), "//*[@numFound='1']");
- assertQ(req("name_s:xyz"), "//*[@numFound='0']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testStopTransform() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- rows.add(createMap("id", "2", "desc", "two", "$stopTransform", "true"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(DATA_CONFIG_FOR_SKIP_TRANSFORM);
-
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("id:2"), "//*[@numFound='1']");
- assertQ(req("name_s:xyz"), "//*[@numFound='1']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testDeleteDocs() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- rows.add(createMap("id", "2", "desc", "two"));
- rows.add(createMap("id", "3", "desc", "two", DocBuilder.DELETE_DOC_BY_ID, "2"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(DATA_CONFIG_FOR_SKIP_TRANSFORM);
-
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("id:2"), "//*[@numFound='0']");
- assertQ(req("id:3"), "//*[@numFound='1']");
-
- assertTrue("Update request processor processDelete was not called", TestUpdateRequestProcessor.processDeleteCalled);
- assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
-
- MockDataSource.clearCache();
- rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- rows.add(createMap("id", "2", "desc", "one"));
- rows.add(createMap("id", "3", "desc", "two", DocBuilder.DELETE_DOC_BY_QUERY, "desc:one"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(DATA_CONFIG_FOR_SKIP_TRANSFORM);
-
- assertQ(req("id:1"), "//*[@numFound='0']");
- assertQ(req("id:2"), "//*[@numFound='0']");
- assertQ(req("id:3"), "//*[@numFound='1']");
-
- assertTrue("Update request processor processDelete was not called", TestUpdateRequestProcessor.processDeleteCalled);
- assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
-
- MockDataSource.clearCache();
- rows = new ArrayList();
- rows.add(createMap(DocBuilder.DELETE_DOC_BY_ID, "3"));
- MockDataSource.setIterator("select * from x", rows.iterator());
- runFullImport(DATA_CONFIG_FOR_SKIP_TRANSFORM, createMap("clean","false"));
- assertQ(req("id:3"), "//*[@numFound='0']");
-
- assertTrue("Update request processor processDelete was not called", TestUpdateRequestProcessor.processDeleteCalled);
- assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
-
- }
-
- @Test
- @Ignore("Fix Me. See SOLR-4103.")
- public void testFileListEntityProcessor_lastIndexTime() throws Exception {
- File tmpdir = createTempDir().toFile();
-
- Map<String, String> params = createMap("baseDir", tmpdir.getAbsolutePath());
-
- createFile(tmpdir, "a.xml", "a.xml".getBytes(StandardCharsets.UTF_8), true);
- createFile(tmpdir, "b.xml", "b.xml".getBytes(StandardCharsets.UTF_8), true);
- createFile(tmpdir, "c.props", "c.props".getBytes(StandardCharsets.UTF_8), true);
- runFullImport(DATA_CONFIG_FILE_LIST, params);
- assertQ(req("*:*"), "//*[@numFound='3']");
-
- // Add a new file after a full index is done
- createFile(tmpdir, "t.xml", "t.xml".getBytes(StandardCharsets.UTF_8), false);
- runFullImport(DATA_CONFIG_FILE_LIST, params);
- // we should find only 1 because by default clean=true is passed
- // and this particular import should find only one file t.xml
- assertQ(req("*:*"), "//*[@numFound='1']");
- }
-
- public static class MockTransformer extends Transformer {
- @Override
- public Object transformRow(Map<String, Object> row, Context context) {
- assertTrue("Context gave incorrect data source", context.getDataSource("mockDs") instanceof MockDataSource2);
- return row;
- }
- }
-
- public static class AddDynamicFieldTransformer extends Transformer {
- @Override
- public Object transformRow(Map<String, Object> row, Context context) {
- // Add a dynamic field
- row.put("dynamic_s", "test");
- return row;
- }
- }
-
- public static class ForcedExceptionTransformer extends Transformer {
- @Override
- public Object transformRow(Map<String, Object> row, Context context) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "ForcedException");
- }
- }
-
- public static class MockDataSource2 extends MockDataSource {
-
- }
-
- public static class StartEventListener implements EventListener {
- public static boolean executed = false;
-
- @Override
- public void onEvent(Context ctx) {
- executed = true;
- }
- }
-
- public static class EndEventListener implements EventListener {
- public static boolean executed = false;
-
- @Override
- public void onEvent(Context ctx) {
- executed = true;
- }
- }
-
- public static class ErrorEventListener implements EventListener {
- public static boolean executed = false;
- public static Exception lastException = null;
-
- @Override
- public void onEvent(Context ctx) {
- executed = true;
- lastException = ((ContextImpl) ctx).getLastException();
- }
- }
-
- private static final String REQUEST_PARAM_AS_VARIABLE = "<dataConfig>\n" +
- " <dataSource type=\"MockDataSource\" />\n" +
- " <document>\n" +
- " <entity name=\"books\" query=\"select * from books where category='${dataimporter.request.category}'\">\n" +
- " <field column=\"id\" />\n" +
- " <field column=\"desc\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private static final String DATA_CONFIG_WITH_DYNAMIC_TRANSFORMER = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
- " <document>\n" +
- " <entity name=\"books\" query=\"select * from x\"" +
- " transformer=\"TestDocBuilder2$AddDynamicFieldTransformer\">\n" +
- " <field column=\"id\" />\n" +
- " <field column=\"desc\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private static final String DATA_CONFIG_FOR_SKIP_TRANSFORM = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
- " <document>\n" +
- " <entity name=\"books\" query=\"select * from x\"" +
- " transformer=\"TemplateTransformer\">\n" +
- " <field column=\"id\" />\n" +
- " <field column=\"desc\" />\n" +
- " <field column=\"name_s\" template=\"xyz\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private static final String DATA_CONFIG_WITH_TWO_ENTITIES = "<dataConfig><dataSource type=\"MockDataSource\"/>\n" +
- " <document>\n" +
- " <entity name=\"books\" query=\"select * from x\">" +
- " <field column=\"id\" />\n" +
- " <field column=\"desc\" />\n" +
- " <entity name=\"authors\" query=\"${books.id}\">" +
- " <field column=\"name_s\" />" +
- " </entity>" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private static final String DATA_CONFIG_WITH_CASE_INSENSITIVE_FIELDS = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
- " <document onImportStart=\"TestDocBuilder2$StartEventListener\" onImportEnd=\"TestDocBuilder2$EndEventListener\">\n" +
- " <entity name=\"books\" query=\"select * from x\">\n" +
- " <field column=\"ID\" />\n" +
- " <field column=\"Desc\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private static final String DATA_CONFIG_WITH_ERROR_HANDLER = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
- " <document onError=\"TestDocBuilder2$ErrorEventListener\">\n" +
- " <entity name=\"books\" query=\"select * from x\" transformer=\"TestDocBuilder2$ForcedExceptionTransformer\">\n" +
- " <field column=\"id\" />\n" +
- " <field column=\"FORCE_ERROR\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private static final String DATA_CONFIG_WITH_TEMPLATIZED_FIELD_NAMES = "<dataConfig><dataSource type=\"MockDataSource\"/>\n" +
- " <document>\n" +
- " <entity name=\"books\" query=\"select * from x\">\n" +
- " <field column=\"mypk\" name=\"${dih.request.mypk}\" />\n" +
- " <field column=\"text\" name=\"${dih.request.text}\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private static final String DATA_CONFIG_WITH_DYNAMIC_FIELD_NAMES = "<dataConfig><dataSource type=\"MockDataSource\"/>\n" +
- " <document>\n" +
- " <entity name=\"books\" query=\"select * from x\">\n" +
- " <field column=\"mypk\" name=\"id\" />\n" +
- " <field column=\"text\" name=\"${books.mypk}_s\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private static final String DATA_CONFIG_FILE_LIST = "<dataConfig>\n" +
- "\t<document>\n" +
- "\t\t<entity name=\"x\" processor=\"FileListEntityProcessor\" \n" +
- "\t\t\t\tfileName=\".*\" newerThan=\"${dih.last_index_time}\" \n" +
- "\t\t\t\tbaseDir=\"${dih.request.baseDir}\" transformer=\"TemplateTransformer\">\n" +
- "\t\t\t<field column=\"id\" template=\"${x.file}\" />\n" +
- "\t\t</entity>\n" +
- "\t</document>\n" +
- "</dataConfig>";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java
deleted file mode 100644
index 623e49e..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * <p>
- * Test for EntityProcessorBase
- * </p>
- *
- *
- * @since solr 1.3
- */
-public class TestEntityProcessorBase extends AbstractDataImportHandlerTestCase {
-
- @Test
- public void multiTransformer() {
- List<Map<String, String>> fields = new ArrayList<>();
- Map<String, String> entity = new HashMap<>();
- entity.put("transformer", T1.class.getName() + "," + T2.class.getName()
- + "," + T3.class.getName());
- fields.add(getField("A", null, null, null, null));
- fields.add(getField("B", null, null, null, null));
-
- Context context = getContext(null, null, new MockDataSource(), Context.FULL_DUMP,
- fields, entity);
- Map<String, Object> src = new HashMap<>();
- src.put("A", "NA");
- src.put("B", "NA");
- EntityProcessorWrapper sep = new EntityProcessorWrapper(new SqlEntityProcessor(), null, null);
- sep.init(context);
- Map<String, Object> res = sep.applyTransformer(src);
- assertNotNull(res.get("T1"));
- assertNotNull(res.get("T2"));
- assertNotNull(res.get("T3"));
- }
-
- static class T1 extends Transformer {
-
- @Override
- public Object transformRow(Map<String, Object> aRow, Context context) {
- aRow.put("T1", "T1 called");
- return aRow;
-
- }
- }
-
- static class T2 extends Transformer {
-
- @Override
- public Object transformRow(Map<String, Object> aRow, Context context) {
- aRow.put("T2", "T2 called");
- return aRow;
- }
- }
-
- static class T3 {
-
- public Object transformRow(Map<String, Object> aRow) {
- aRow.put("T3", "T3 called");
- return aRow;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEphemeralCache.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEphemeralCache.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEphemeralCache.java
deleted file mode 100644
index ddccafe..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEphemeralCache.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.hamcrest.CoreMatchers.*;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class TestEphemeralCache extends AbstractDataImportHandlerTestCase {
-
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
- }
-
- @Before
- public void reset() {
- DestroyCountCache.destroyed.clear();
- setupMockData();
- }
-
- @Test
- public void test() throws Exception {
- assertFullImport(getDataConfigDotXml());
- }
-
- @SuppressWarnings("unchecked")
- private void setupMockData() {
- List parentRows = new ArrayList();
- parentRows.add(createMap("id", new BigDecimal("1"), "parent_s", "one"));
- parentRows.add(createMap("id", new BigDecimal("2"), "parent_s", "two"));
- parentRows.add(createMap("id", new BigDecimal("3"), "parent_s", "three"));
- parentRows.add(createMap("id", new BigDecimal("4"), "parent_s", "four"));
- parentRows.add(createMap("id", new BigDecimal("5"), "parent_s", "five"));
-
- List child1Rows = new ArrayList();
- child1Rows.add(createMap("id", new BigDecimal("6"), "child1a_mult_s", "this is the number six."));
- child1Rows.add(createMap("id", new BigDecimal("5"), "child1a_mult_s", "this is the number five."));
- child1Rows.add(createMap("id", new BigDecimal("6"), "child1a_mult_s", "let's sing a song of six."));
- child1Rows.add(createMap("id", new BigDecimal("3"), "child1a_mult_s", "three"));
- child1Rows.add(createMap("id", new BigDecimal("3"), "child1a_mult_s", "III"));
- child1Rows.add(createMap("id", new BigDecimal("3"), "child1a_mult_s", "3"));
- child1Rows.add(createMap("id", new BigDecimal("3"), "child1a_mult_s", "|||"));
- child1Rows.add(createMap("id", new BigDecimal("1"), "child1a_mult_s", "one"));
- child1Rows.add(createMap("id", new BigDecimal("1"), "child1a_mult_s", "uno"));
- child1Rows.add(createMap("id", new BigDecimal("2"), "child1b_s", "CHILD1B", "child1a_mult_s", "this is the number two."));
-
- List child2Rows = new ArrayList();
- child2Rows.add(createMap("id", new BigDecimal("6"), "child2a_mult_s", "Child 2 says, 'this is the number six.'"));
- child2Rows.add(createMap("id", new BigDecimal("5"), "child2a_mult_s", "Child 2 says, 'this is the number five.'"));
- child2Rows.add(createMap("id", new BigDecimal("6"), "child2a_mult_s", "Child 2 says, 'let's sing a song of six.'"));
- child2Rows.add(createMap("id", new BigDecimal("3"), "child2a_mult_s", "Child 2 says, 'three'"));
- child2Rows.add(createMap("id", new BigDecimal("3"), "child2a_mult_s", "Child 2 says, 'III'"));
- child2Rows.add(createMap("id", new BigDecimal("3"), "child2b_s", "CHILD2B", "child2a_mult_s", "Child 2 says, '3'"));
- child2Rows.add(createMap("id", new BigDecimal("3"), "child2a_mult_s", "Child 2 says, '|||'"));
- child2Rows.add(createMap("id", new BigDecimal("1"), "child2a_mult_s", "Child 2 says, 'one'"));
- child2Rows.add(createMap("id", new BigDecimal("1"), "child2a_mult_s", "Child 2 says, 'uno'"));
- child2Rows.add(createMap("id", new BigDecimal("2"), "child2a_mult_s", "Child 2 says, 'this is the number two.'"));
-
- MockDataSource.setIterator("SELECT * FROM PARENT", parentRows.iterator());
- MockDataSource.setIterator("SELECT * FROM CHILD_1", child1Rows.iterator());
- MockDataSource.setIterator("SELECT * FROM CHILD_2", child2Rows.iterator());
-
- }
- private String getDataConfigDotXml() {
- return
- "<dataConfig>" +
- " <dataSource type=\"MockDataSource\" />" +
- " <document>" +
- " <entity " +
- " name=\"PARENT\"" +
- " processor=\"SqlEntityProcessor\"" +
- " cacheImpl=\"org.apache.solr.handler.dataimport.DestroyCountCache\"" +
- " cacheName=\"PARENT\"" +
- " query=\"SELECT * FROM PARENT\" " +
- " >" +
- " <entity" +
- " name=\"CHILD_1\"" +
- " processor=\"SqlEntityProcessor\"" +
- " cacheImpl=\"org.apache.solr.handler.dataimport.DestroyCountCache\"" +
- " cacheName=\"CHILD\"" +
- " cacheKey=\"id\"" +
- " cacheLookup=\"PARENT.id\"" +
- " fieldNames=\"id, child1a_mult_s, child1b_s\"" +
- " fieldTypes=\"BIGDECIMAL, STRING, STRING\"" +
- " query=\"SELECT * FROM CHILD_1\" " +
- " />" +
- " <entity" +
- " name=\"CHILD_2\"" +
- " processor=\"SqlEntityProcessor\"" +
- " cacheImpl=\"org.apache.solr.handler.dataimport.DestroyCountCache\"" +
- " cacheKey=\"id\"" +
- " cacheLookup=\"PARENT.id\"" +
- " query=\"SELECT * FROM CHILD_2\" " +
- " />" +
- " </entity>" +
- " </document>" +
- "</dataConfig>"
- ;
- }
-
- private void assertFullImport(String dataConfig) throws Exception {
- runFullImport(dataConfig);
-
- assertQ(req("*:*"), "//*[@numFound='5']");
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("id:6"), "//*[@numFound='0']");
- assertQ(req("parent_s:four"), "//*[@numFound='1']");
- assertQ(req("child1a_mult_s:this\\ is\\ the\\ numbe*"), "//*[@numFound='2']");
- assertQ(req("child2a_mult_s:Child\\ 2\\ say*"), "//*[@numFound='4']");
- assertQ(req("child1b_s:CHILD1B"), "//*[@numFound='1']");
- assertQ(req("child2b_s:CHILD2B"), "//*[@numFound='1']");
- assertQ(req("child1a_mult_s:one"), "//*[@numFound='1']");
- assertQ(req("child1a_mult_s:uno"), "//*[@numFound='1']");
- assertQ(req("child1a_mult_s:(uno OR one)"), "//*[@numFound='1']");
-
- assertThat(DestroyCountCache.destroyed.size(), is(3));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestErrorHandling.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestErrorHandling.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestErrorHandling.java
deleted file mode 100644
index 1ea1ad4..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestErrorHandling.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-import java.io.Reader;
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-
-/**
- * Tests exception handling during imports in DataImportHandler
- *
- *
- * @since solr 1.4
- */
-public class TestErrorHandling extends AbstractDataImportHandlerTestCase {
-
- //TODO: fix this test to not require FSDirectory.
- static String savedFactory;
- @BeforeClass
- public static void beforeClass() throws Exception {
- savedFactory = System.getProperty("solr.DirectoryFactory");
- System.setProperty("solr.directoryFactory", "solr.MockFSDirectoryFactory");
- initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
- ignoreException("Unexpected close tag");
- }
-
- @AfterClass
- public static void afterClass() {
- if (savedFactory == null) {
- System.clearProperty("solr.directoryFactory");
- } else {
- System.setProperty("solr.directoryFactory", savedFactory);
- }
- }
-
- @Before @Override
- public void setUp() throws Exception {
- super.setUp();
- clearIndex();
- assertU(commit());
- }
-
- public void testMalformedStreamingXml() throws Exception {
- StringDataSource.xml = malformedXml;
- runFullImport(dataConfigWithStreaming);
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("id:2"), "//*[@numFound='1']");
- }
-
- public void testMalformedNonStreamingXml() throws Exception {
- StringDataSource.xml = malformedXml;
- runFullImport(dataConfigWithoutStreaming);
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("id:2"), "//*[@numFound='1']");
- }
-
- public void testAbortOnError() throws Exception {
- StringDataSource.xml = malformedXml;
- runFullImport(dataConfigAbortOnError);
- assertQ(req("*:*"), "//*[@numFound='0']");
- }
-
- public void testTransformerErrorContinue() throws Exception {
- StringDataSource.xml = wellformedXml;
- List<Map<String, Object>> rows = new ArrayList<>();
- rows.add(createMap("id", "3", "desc", "exception-transformer"));
- MockDataSource.setIterator("select * from foo", rows.iterator());
- runFullImport(dataConfigWithTransformer);
- assertQ(req("*:*"), "//*[@numFound='3']");
- }
-
- public void testExternalEntity() throws Exception {
- StringDataSource.xml = wellformedXml;
- // This should not fail as external entities are replaced by an empty string during parsing:
- runFullImport(dataConfigWithEntity);
- assertQ(req("*:*"), "//*[@numFound='3']");
- }
-
- public static class StringDataSource extends DataSource<Reader> {
- public static String xml = "";
-
- @Override
- public void init(Context context, Properties initProps) {
- }
-
- @Override
- public Reader getData(String query) {
- return new StringReader(xml);
- }
-
- @Override
- public void close() {
-
- }
- }
-
- public static class ExceptionTransformer extends Transformer {
- @Override
- public Object transformRow(Map<String, Object> row, Context context) {
- throw new RuntimeException("Test exception");
- }
- }
-
- private String dataConfigWithStreaming = "<dataConfig>\n" +
- " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
- " <document>\n" +
- " <entity name=\"node\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" stream=\"true\" forEach=\"/root/node\" onError=\"skip\">\n" +
- " <field column=\"id\" xpath=\"/root/node/id\" />\n" +
- " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private String dataConfigWithoutStreaming = "<dataConfig>\n" +
- " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
- " <document>\n" +
- " <entity name=\"node\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" forEach=\"/root/node\" onError=\"skip\">\n" +
- " <field column=\"id\" xpath=\"/root/node/id\" />\n" +
- " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private String dataConfigAbortOnError = "<dataConfig>\n" +
- " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
- " <document>\n" +
- " <entity name=\"node\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" forEach=\"/root/node\" onError=\"abort\">\n" +
- " <field column=\"id\" xpath=\"/root/node/id\" />\n" +
- " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private String dataConfigWithTransformer = "<dataConfig>\n" +
- " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
- "<dataSource type=\"MockDataSource\"/>" +
- " <document>\n" +
- " <entity name=\"node\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" forEach=\"/root/node\">\n" +
- " <field column=\"id\" xpath=\"/root/node/id\" />\n" +
- " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
- " <entity name=\"child\" query=\"select * from foo\" transformer=\"TestErrorHandling$ExceptionTransformer\" onError=\"continue\">\n" +
- " </entity>" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private String dataConfigWithEntity = "<!DOCTYPE dataConfig [\n" +
- " <!ENTITY internalTerm \"node\">\n" +
- " <!ENTITY externalTerm SYSTEM \"foo://bar.xyz/external\">\n" +
- "]><dataConfig>\n" +
- " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
- " <document>\n" +
- " <entity name=\"&internalTerm;\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" forEach=\"/root/node\" onError=\"skip\">\n" +
- " <field column=\"id\" xpath=\"/root/node/id\">&externalTerm;</field>\n" +
- " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private String malformedXml = "<root>\n" +
- " <node>\n" +
- " <id>1</id>\n" +
- " <desc>test1</desc>\n" +
- " </node>\n" +
- " <node>\n" +
- " <id>2</id>\n" +
- " <desc>test2</desc>\n" +
- " </node>\n" +
- " <node>\n" +
- " <id/>3</id>\n" +
- " <desc>test3</desc>\n" +
- " </node>\n" +
- "</root>";
-
- private String wellformedXml = "<root>\n" +
- " <node>\n" +
- " <id>1</id>\n" +
- " <desc>test1</desc>\n" +
- " </node>\n" +
- " <node>\n" +
- " <id>2</id>\n" +
- " <desc>test2</desc>\n" +
- " </node>\n" +
- " <node>\n" +
- " <id>3</id>\n" +
- " <desc>test3</desc>\n" +
- " </node>\n" +
- "</root>";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java
deleted file mode 100644
index 347a40b..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Test for FieldReaderDataSource
- *
- *
- * @see org.apache.solr.handler.dataimport.FieldReaderDataSource
- * @since 1.4
- */
-public class TestFieldReader extends AbstractDataImportHandlerTestCase {
-
- @Test
- public void simple() {
- DataImporter di = new DataImporter();
- di.loadAndInit(config);
- redirectTempProperties(di);
-
- TestDocBuilder.SolrWriterImpl sw = new TestDocBuilder.SolrWriterImpl();
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- List<Map<String, Object>> l = new ArrayList<>();
- l.add(createMap("xml", xml));
- MockDataSource.setIterator("select * from a", l.iterator());
- di.runCmd(rp, sw);
- assertEquals(sw.docs.get(0).getFieldValue("y"), "Hello");
- MockDataSource.clearCache();
- }
-
- String config = "<dataConfig>\n" +
- " <dataSource type=\"FieldReaderDataSource\" name=\"f\"/>\n" +
- " <dataSource type=\"MockDataSource\"/>\n" +
- " <document>\n" +
- " <entity name=\"a\" query=\"select * from a\" >\n" +
- " <entity name=\"b\" dataSource=\"f\" processor=\"XPathEntityProcessor\" forEach=\"/x\" dataField=\"a.xml\">\n" +
- " <field column=\"y\" xpath=\"/x/y\"/>\n" +
- " </entity>\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- String xml = "<x>\n" +
- " <y>Hello</y>\n" +
- "</x>";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
deleted file mode 100644
index dd2cf72..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.solr.common.util.SuppressForbidden;
-import org.junit.Test;
-
-/**
- * <p>
- * Test for FileListEntityProcessor
- * </p>
- *
- *
- * @since solr 1.3
- */
-public class TestFileListEntityProcessor extends AbstractDataImportHandlerTestCase {
-
- @Test
- @SuppressWarnings("unchecked")
- public void testSimple() throws IOException {
- File tmpdir = createTempDir().toFile();
-
- createFile(tmpdir, "a.xml", "a.xml".getBytes(StandardCharsets.UTF_8), false);
- createFile(tmpdir, "b.xml", "b.xml".getBytes(StandardCharsets.UTF_8), false);
- createFile(tmpdir, "c.props", "c.props".getBytes(StandardCharsets.UTF_8), false);
- Map attrs = createMap(
- FileListEntityProcessor.FILE_NAME, "xml$",
- FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath());
- Context c = getContext(null,
- new VariableResolver(), null, Context.FULL_DUMP, Collections.EMPTY_LIST, attrs);
- FileListEntityProcessor fileListEntityProcessor = new FileListEntityProcessor();
- fileListEntityProcessor.init(c);
- List<String> fList = new ArrayList<>();
- while (true) {
- Map<String, Object> f = fileListEntityProcessor.nextRow();
- if (f == null)
- break;
- fList.add((String) f.get(FileListEntityProcessor.ABSOLUTE_FILE));
- }
- assertEquals(2, fList.size());
- }
-
- @Test
- public void testBiggerSmallerFiles() throws IOException {
- File tmpdir = createTempDir().toFile();
-
- long minLength = Long.MAX_VALUE;
- String smallestFile = "";
- byte[] content = "abcdefgij".getBytes(StandardCharsets.UTF_8);
- createFile(tmpdir, "a.xml", content, false);
- if (minLength > content.length) {
- minLength = content.length;
- smallestFile = "a.xml";
- }
- content = "abcdefgij".getBytes(StandardCharsets.UTF_8);
- createFile(tmpdir, "b.xml", content, false);
- if (minLength > content.length) {
- minLength = content.length;
- smallestFile = "b.xml";
- }
- content = "abc".getBytes(StandardCharsets.UTF_8);
- createFile(tmpdir, "c.props", content, false);
- if (minLength > content.length) {
- minLength = content.length;
- smallestFile = "c.props";
- }
- Map attrs = createMap(
- FileListEntityProcessor.FILE_NAME, ".*",
- FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
- FileListEntityProcessor.BIGGER_THAN, String.valueOf(minLength));
- List<String> fList = getFiles(null, attrs);
- assertEquals(2, fList.size());
- Set<String> l = new HashSet<>();
- l.add(new File(tmpdir, "a.xml").getAbsolutePath());
- l.add(new File(tmpdir, "b.xml").getAbsolutePath());
- assertEquals(l, new HashSet<>(fList));
- attrs = createMap(
- FileListEntityProcessor.FILE_NAME, ".*",
- FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
- FileListEntityProcessor.SMALLER_THAN, String.valueOf(minLength+1));
- fList = getFiles(null, attrs);
- l.clear();
- l.add(new File(tmpdir, smallestFile).getAbsolutePath());
- assertEquals(l, new HashSet<>(fList));
- attrs = createMap(
- FileListEntityProcessor.FILE_NAME, ".*",
- FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
- FileListEntityProcessor.SMALLER_THAN, "${a.x}");
- VariableResolver resolver = new VariableResolver();
- resolver.addNamespace("a", createMap("x", "4"));
- fList = getFiles(resolver, attrs);
- assertEquals(l, new HashSet<>(fList));
- }
-
- @SuppressWarnings("unchecked")
- static List<String> getFiles(VariableResolver resolver, Map attrs) {
- Context c = getContext(null,
- resolver, null, Context.FULL_DUMP, Collections.EMPTY_LIST, attrs);
- FileListEntityProcessor fileListEntityProcessor = new FileListEntityProcessor();
- fileListEntityProcessor.init(c);
- List<String> fList = new ArrayList<>();
- while (true) {
- Map<String, Object> f = fileListEntityProcessor.nextRow();
- if (f == null)
- break;
- fList.add((String) f.get(FileListEntityProcessor.ABSOLUTE_FILE));
- }
- return fList;
- }
-
- @SuppressForbidden(reason = "Needs currentTimeMillis to set last modified time")
- @Test
- public void testNTOT() throws IOException {
- File tmpdir = createTempDir().toFile();
-
- createFile(tmpdir, "a.xml", "a.xml".getBytes(StandardCharsets.UTF_8), true);
- createFile(tmpdir, "b.xml", "b.xml".getBytes(StandardCharsets.UTF_8), true);
- createFile(tmpdir, "c.props", "c.props".getBytes(StandardCharsets.UTF_8), true);
- Map attrs = createMap(
- FileListEntityProcessor.FILE_NAME, "xml$",
- FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
- FileListEntityProcessor.OLDER_THAN, "'NOW'");
- List<String> fList = getFiles(null, attrs);
- assertEquals(2, fList.size());
- attrs = createMap(
- FileListEntityProcessor.FILE_NAME, ".xml$",
- FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
- FileListEntityProcessor.NEWER_THAN, "'NOW-2HOURS'");
- fList = getFiles(null, attrs);
- assertEquals(2, fList.size());
-
- // Use a variable for newerThan
- attrs = createMap(
- FileListEntityProcessor.FILE_NAME, ".xml$",
- FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
- FileListEntityProcessor.NEWER_THAN, "${a.x}");
- VariableResolver resolver = new VariableResolver();
- String lastMod = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).format(new Date(System.currentTimeMillis() - 50000));
- resolver.addNamespace("a", createMap("x", lastMod));
- createFile(tmpdir, "t.xml", "t.xml".getBytes(StandardCharsets.UTF_8), false);
- fList = getFiles(resolver, attrs);
- assertEquals(1, fList.size());
- assertEquals("File name must be t.xml", new File(tmpdir, "t.xml").getAbsolutePath(), fList.get(0));
- }
-
- @Test
- public void testRECURSION() throws IOException {
- File tmpdir = createTempDir().toFile();
- File childdir = new File(tmpdir + "/child" );
- childdir.mkdir();
- createFile(childdir, "a.xml", "a.xml".getBytes(StandardCharsets.UTF_8), true);
- createFile(childdir, "b.xml", "b.xml".getBytes(StandardCharsets.UTF_8), true);
- createFile(childdir, "c.props", "c.props".getBytes(StandardCharsets.UTF_8), true);
- Map attrs = createMap(
- FileListEntityProcessor.FILE_NAME, "^.*\\.xml$",
- FileListEntityProcessor.BASE_DIR, childdir.getAbsolutePath(),
- FileListEntityProcessor.RECURSIVE, "true");
- List<String> fList = getFiles(null, attrs);
- assertEquals(2, fList.size());
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListWithLineEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListWithLineEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListWithLineEntityProcessor.java
deleted file mode 100644
index aad8e30..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListWithLineEntityProcessor.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.File;
-import java.nio.charset.StandardCharsets;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.solr.request.LocalSolrQueryRequest;
-import org.junit.BeforeClass;
-
-public class TestFileListWithLineEntityProcessor extends AbstractDataImportHandlerTestCase {
- @BeforeClass
- public static void beforeClass() throws Exception {
- initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
- }
-
- public void test() throws Exception {
- File tmpdir = createTempDir(LuceneTestCase.getTestClass().getSimpleName()).toFile();
- createFile(tmpdir, "a.txt", "a line one\na line two\na line three".getBytes(StandardCharsets.UTF_8), false);
- createFile(tmpdir, "b.txt", "b line one\nb line two".getBytes(StandardCharsets.UTF_8), false);
- createFile(tmpdir, "c.txt", "c line one\nc line two\nc line three\nc line four".getBytes(StandardCharsets.UTF_8), false);
-
- String config = generateConfig(tmpdir);
- LocalSolrQueryRequest request = lrf.makeRequest(
- "command", "full-import", "dataConfig", config,
- "clean", "true", "commit", "true", "synchronous", "true", "indent", "true");
- h.query("/dataimport", request);
-
- assertQ(req("*:*"), "//*[@numFound='9']");
- assertQ(req("id:?\\ line\\ one"), "//*[@numFound='3']");
- assertQ(req("id:a\\ line*"), "//*[@numFound='3']");
- assertQ(req("id:b\\ line*"), "//*[@numFound='2']");
- assertQ(req("id:c\\ line*"), "//*[@numFound='4']");
- }
-
- private String generateConfig(File dir) {
- return
- "<dataConfig> \n"+
- "<dataSource type=\"FileDataSource\" encoding=\"UTF-8\" name=\"fds\"/> \n"+
- " <document> \n"+
- " <entity name=\"f\" processor=\"FileListEntityProcessor\" fileName=\".*[.]txt\" baseDir=\"" + dir.getAbsolutePath() + "\" recursive=\"false\" rootEntity=\"false\" transformer=\"TemplateTransformer\"> \n" +
- " <entity name=\"jc\" processor=\"LineEntityProcessor\" url=\"${f.fileAbsolutePath}\" dataSource=\"fds\" rootEntity=\"true\" transformer=\"TemplateTransformer\"> \n" +
- " <field column=\"rawLine\" name=\"id\" /> \n" +
- " </entity> \n"+
- " </entity> \n"+
- " </document> \n"+
- "</dataConfig> \n";
- }
-}
[09/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFieldReader.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFieldReader.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFieldReader.java
new file mode 100644
index 0000000..347a40b
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFieldReader.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Test for FieldReaderDataSource
+ *
+ *
+ * @see org.apache.solr.handler.dataimport.FieldReaderDataSource
+ * @since 1.4
+ */
+public class TestFieldReader extends AbstractDataImportHandlerTestCase {
+
+ @Test
+ public void simple() {
+ DataImporter di = new DataImporter();
+ di.loadAndInit(config);
+ redirectTempProperties(di);
+
+ TestDocBuilder.SolrWriterImpl sw = new TestDocBuilder.SolrWriterImpl();
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
+ List<Map<String, Object>> l = new ArrayList<>();
+ l.add(createMap("xml", xml));
+ MockDataSource.setIterator("select * from a", l.iterator());
+ di.runCmd(rp, sw);
+ assertEquals(sw.docs.get(0).getFieldValue("y"), "Hello");
+ MockDataSource.clearCache();
+ }
+
+ String config = "<dataConfig>\n" +
+ " <dataSource type=\"FieldReaderDataSource\" name=\"f\"/>\n" +
+ " <dataSource type=\"MockDataSource\"/>\n" +
+ " <document>\n" +
+ " <entity name=\"a\" query=\"select * from a\" >\n" +
+ " <entity name=\"b\" dataSource=\"f\" processor=\"XPathEntityProcessor\" forEach=\"/x\" dataField=\"a.xml\">\n" +
+ " <field column=\"y\" xpath=\"/x/y\"/>\n" +
+ " </entity>\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ String xml = "<x>\n" +
+ " <y>Hello</y>\n" +
+ "</x>";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
new file mode 100644
index 0000000..dd2cf72
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.solr.common.util.SuppressForbidden;
+import org.junit.Test;
+
+/**
+ * <p>
+ * Test for FileListEntityProcessor
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestFileListEntityProcessor extends AbstractDataImportHandlerTestCase {
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testSimple() throws IOException {
+ File tmpdir = createTempDir().toFile();
+
+ createFile(tmpdir, "a.xml", "a.xml".getBytes(StandardCharsets.UTF_8), false);
+ createFile(tmpdir, "b.xml", "b.xml".getBytes(StandardCharsets.UTF_8), false);
+ createFile(tmpdir, "c.props", "c.props".getBytes(StandardCharsets.UTF_8), false);
+ Map attrs = createMap(
+ FileListEntityProcessor.FILE_NAME, "xml$",
+ FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath());
+ Context c = getContext(null,
+ new VariableResolver(), null, Context.FULL_DUMP, Collections.EMPTY_LIST, attrs);
+ FileListEntityProcessor fileListEntityProcessor = new FileListEntityProcessor();
+ fileListEntityProcessor.init(c);
+ List<String> fList = new ArrayList<>();
+ while (true) {
+ Map<String, Object> f = fileListEntityProcessor.nextRow();
+ if (f == null)
+ break;
+ fList.add((String) f.get(FileListEntityProcessor.ABSOLUTE_FILE));
+ }
+ assertEquals(2, fList.size());
+ }
+
+ @Test
+ public void testBiggerSmallerFiles() throws IOException {
+ File tmpdir = createTempDir().toFile();
+
+ long minLength = Long.MAX_VALUE;
+ String smallestFile = "";
+ byte[] content = "abcdefgij".getBytes(StandardCharsets.UTF_8);
+ createFile(tmpdir, "a.xml", content, false);
+ if (minLength > content.length) {
+ minLength = content.length;
+ smallestFile = "a.xml";
+ }
+ content = "abcdefgij".getBytes(StandardCharsets.UTF_8);
+ createFile(tmpdir, "b.xml", content, false);
+ if (minLength > content.length) {
+ minLength = content.length;
+ smallestFile = "b.xml";
+ }
+ content = "abc".getBytes(StandardCharsets.UTF_8);
+ createFile(tmpdir, "c.props", content, false);
+ if (minLength > content.length) {
+ minLength = content.length;
+ smallestFile = "c.props";
+ }
+ Map attrs = createMap(
+ FileListEntityProcessor.FILE_NAME, ".*",
+ FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
+ FileListEntityProcessor.BIGGER_THAN, String.valueOf(minLength));
+ List<String> fList = getFiles(null, attrs);
+ assertEquals(2, fList.size());
+ Set<String> l = new HashSet<>();
+ l.add(new File(tmpdir, "a.xml").getAbsolutePath());
+ l.add(new File(tmpdir, "b.xml").getAbsolutePath());
+ assertEquals(l, new HashSet<>(fList));
+ attrs = createMap(
+ FileListEntityProcessor.FILE_NAME, ".*",
+ FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
+ FileListEntityProcessor.SMALLER_THAN, String.valueOf(minLength+1));
+ fList = getFiles(null, attrs);
+ l.clear();
+ l.add(new File(tmpdir, smallestFile).getAbsolutePath());
+ assertEquals(l, new HashSet<>(fList));
+ attrs = createMap(
+ FileListEntityProcessor.FILE_NAME, ".*",
+ FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
+ FileListEntityProcessor.SMALLER_THAN, "${a.x}");
+ VariableResolver resolver = new VariableResolver();
+ resolver.addNamespace("a", createMap("x", "4"));
+ fList = getFiles(resolver, attrs);
+ assertEquals(l, new HashSet<>(fList));
+ }
+
+ @SuppressWarnings("unchecked")
+ static List<String> getFiles(VariableResolver resolver, Map attrs) {
+ Context c = getContext(null,
+ resolver, null, Context.FULL_DUMP, Collections.EMPTY_LIST, attrs);
+ FileListEntityProcessor fileListEntityProcessor = new FileListEntityProcessor();
+ fileListEntityProcessor.init(c);
+ List<String> fList = new ArrayList<>();
+ while (true) {
+ Map<String, Object> f = fileListEntityProcessor.nextRow();
+ if (f == null)
+ break;
+ fList.add((String) f.get(FileListEntityProcessor.ABSOLUTE_FILE));
+ }
+ return fList;
+ }
+
+ @SuppressForbidden(reason = "Needs currentTimeMillis to set last modified time")
+ @Test
+ public void testNTOT() throws IOException {
+ File tmpdir = createTempDir().toFile();
+
+ createFile(tmpdir, "a.xml", "a.xml".getBytes(StandardCharsets.UTF_8), true);
+ createFile(tmpdir, "b.xml", "b.xml".getBytes(StandardCharsets.UTF_8), true);
+ createFile(tmpdir, "c.props", "c.props".getBytes(StandardCharsets.UTF_8), true);
+ Map attrs = createMap(
+ FileListEntityProcessor.FILE_NAME, "xml$",
+ FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
+ FileListEntityProcessor.OLDER_THAN, "'NOW'");
+ List<String> fList = getFiles(null, attrs);
+ assertEquals(2, fList.size());
+ attrs = createMap(
+ FileListEntityProcessor.FILE_NAME, ".xml$",
+ FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
+ FileListEntityProcessor.NEWER_THAN, "'NOW-2HOURS'");
+ fList = getFiles(null, attrs);
+ assertEquals(2, fList.size());
+
+ // Use a variable for newerThan
+ attrs = createMap(
+ FileListEntityProcessor.FILE_NAME, ".xml$",
+ FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
+ FileListEntityProcessor.NEWER_THAN, "${a.x}");
+ VariableResolver resolver = new VariableResolver();
+ String lastMod = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).format(new Date(System.currentTimeMillis() - 50000));
+ resolver.addNamespace("a", createMap("x", lastMod));
+ createFile(tmpdir, "t.xml", "t.xml".getBytes(StandardCharsets.UTF_8), false);
+ fList = getFiles(resolver, attrs);
+ assertEquals(1, fList.size());
+ assertEquals("File name must be t.xml", new File(tmpdir, "t.xml").getAbsolutePath(), fList.get(0));
+ }
+
+ @Test
+ public void testRECURSION() throws IOException {
+ File tmpdir = createTempDir().toFile();
+ File childdir = new File(tmpdir + "/child" );
+ childdir.mkdir();
+ createFile(childdir, "a.xml", "a.xml".getBytes(StandardCharsets.UTF_8), true);
+ createFile(childdir, "b.xml", "b.xml".getBytes(StandardCharsets.UTF_8), true);
+ createFile(childdir, "c.props", "c.props".getBytes(StandardCharsets.UTF_8), true);
+ Map attrs = createMap(
+ FileListEntityProcessor.FILE_NAME, "^.*\\.xml$",
+ FileListEntityProcessor.BASE_DIR, childdir.getAbsolutePath(),
+ FileListEntityProcessor.RECURSIVE, "true");
+ List<String> fList = getFiles(null, attrs);
+ assertEquals(2, fList.size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListWithLineEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListWithLineEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListWithLineEntityProcessor.java
new file mode 100644
index 0000000..aad8e30
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListWithLineEntityProcessor.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.junit.BeforeClass;
+
+public class TestFileListWithLineEntityProcessor extends AbstractDataImportHandlerTestCase {
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
+ }
+
+ public void test() throws Exception {
+ File tmpdir = createTempDir(LuceneTestCase.getTestClass().getSimpleName()).toFile();
+ createFile(tmpdir, "a.txt", "a line one\na line two\na line three".getBytes(StandardCharsets.UTF_8), false);
+ createFile(tmpdir, "b.txt", "b line one\nb line two".getBytes(StandardCharsets.UTF_8), false);
+ createFile(tmpdir, "c.txt", "c line one\nc line two\nc line three\nc line four".getBytes(StandardCharsets.UTF_8), false);
+
+ String config = generateConfig(tmpdir);
+ LocalSolrQueryRequest request = lrf.makeRequest(
+ "command", "full-import", "dataConfig", config,
+ "clean", "true", "commit", "true", "synchronous", "true", "indent", "true");
+ h.query("/dataimport", request);
+
+ assertQ(req("*:*"), "//*[@numFound='9']");
+ assertQ(req("id:?\\ line\\ one"), "//*[@numFound='3']");
+ assertQ(req("id:a\\ line*"), "//*[@numFound='3']");
+ assertQ(req("id:b\\ line*"), "//*[@numFound='2']");
+ assertQ(req("id:c\\ line*"), "//*[@numFound='4']");
+ }
+
+ private String generateConfig(File dir) {
+ return
+ "<dataConfig> \n"+
+ "<dataSource type=\"FileDataSource\" encoding=\"UTF-8\" name=\"fds\"/> \n"+
+ " <document> \n"+
+ " <entity name=\"f\" processor=\"FileListEntityProcessor\" fileName=\".*[.]txt\" baseDir=\"" + dir.getAbsolutePath() + "\" recursive=\"false\" rootEntity=\"false\" transformer=\"TemplateTransformer\"> \n" +
+ " <entity name=\"jc\" processor=\"LineEntityProcessor\" url=\"${f.fileAbsolutePath}\" dataSource=\"fds\" rootEntity=\"true\" transformer=\"TemplateTransformer\"> \n" +
+ " <field column=\"rawLine\" name=\"id\" /> \n" +
+ " </entity> \n"+
+ " </entity> \n"+
+ " </document> \n"+
+ "</dataConfig> \n";
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestHierarchicalDocBuilder.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestHierarchicalDocBuilder.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestHierarchicalDocBuilder.java
new file mode 100644
index 0000000..4a3ba72
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestHierarchicalDocBuilder.java
@@ -0,0 +1,479 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.join.BitSetProducer;
+import org.apache.lucene.search.join.QueryBitSetProducer;
+import org.apache.lucene.search.join.ScoreMode;
+import org.apache.lucene.search.join.ToParentBlockJoinQuery;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.handler.dataimport.config.ConfigNameConstants;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.util.TestHarness;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test for DocBuilder using the test harness.
+ * <b> Documents are hierarchical in this test, i.e. each document have nested children documents.</b>
+ */
+public class TestHierarchicalDocBuilder extends AbstractDataImportHandlerTestCase {
+
+ private static final String FIELD_ID = "id";
+ private int id = 0; //unique id
+ private SolrQueryRequest req;
+
+ /**
+ * Holds the data related to randomly created index.
+ * It is used for making assertions.
+ */
+ private static class ContextHolder {
+ /** Overall documents number **/
+ int counter = 0;
+
+ /**
+ * Each Hierarchy object represents nested documents with a parent at the root of hierarchy
+ */
+ List<Hierarchy> hierarchies = new ArrayList<Hierarchy>();
+ }
+
+ /**
+ * Represents a hierarchical document structure
+ */
+ private static class Hierarchy {
+
+ /**
+ * Type of element, i.e. parent, child, grandchild, etc..
+ */
+ String elementType;
+
+ /**
+ * Fields of a current element
+ */
+ Map<String, Object> elementData = new HashMap<String,Object>();
+
+ /**
+ * Nested elements/documents hierarchies.
+ */
+ List<Hierarchy> elements = new ArrayList<Hierarchy>();
+ }
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
+ }
+
+ @Before
+ public void before() {
+ req = req("*:*"); // don't really care about query
+ MockDataSource.clearCache();
+ }
+
+ @After
+ public void after() {
+ req.close();
+ MockDataSource.clearCache();
+ }
+
+ @Test
+ public void testThreeLevelHierarchy() throws Exception {
+ int parentsNum = 3; //fixed for simplicity of test
+ int childrenNum = 0;
+ int grandChildrenNum = 0;
+
+ final String parentType = "parent";
+ final String childType = "child";
+ final String grandChildType = "grand_child";
+
+ List<String> parentIds = createDataIterator("select * from PARENT", parentType, parentType, parentsNum);
+ Collections.shuffle(parentIds, random());
+ final String parentId1 = parentIds.get(0);
+ String parentId2 = parentIds.get(1);
+
+ //parent 1 children
+ int firstParentChildrenNum = 3; //fixed for simplicity of test
+ String select = "select * from CHILD where parent_id='" + parentId1 + "'";
+ List<String> childrenIds = createDataIterator(select, childType, "child of first parent", firstParentChildrenNum);
+ List<String> firstParentChildrenIds = new ArrayList<String>(childrenIds);
+ childrenNum += childrenIds.size();
+
+ // grand children of first parent first child
+ final String childId = childrenIds.get(0);
+ String description = "grandchild of first parent, child of " + childId + " child";
+ select = "select * from GRANDCHILD where parent_id='" + childId + "'";
+ List<String> grandChildrenIds = createDataIterator(select, grandChildType, description, atLeast(2));
+ grandChildrenNum += grandChildrenIds.size();
+
+ // grand children of first parent second child
+ {
+ String childId2 = childrenIds.get(1);
+ description = "grandchild of first parent, child of " + childId2 + " child";
+ select = "select * from GRANDCHILD where parent_id='" + childId2 + "'";
+ }
+ final List<String> grandChildrenIds2 = createDataIterator(select, grandChildType, description, atLeast(2));
+ grandChildrenNum += grandChildrenIds2.size();
+
+ List<String> allGrandChildrenIds = new ArrayList<>(grandChildrenIds);
+ allGrandChildrenIds.addAll(grandChildrenIds2);
+
+ // third children of first parent has no grand children
+
+ // parent 2 children (no grand children)
+ select = "select * from CHILD where parent_id='" + parentId2 + "'";
+ childrenIds = createDataIterator(select, childType, "child of second parent", atLeast(2));
+ childrenNum += childrenIds.size();
+
+ // parent 3 has no children and grand children
+
+ int totalDocsNum = parentsNum + childrenNum + grandChildrenNum;
+
+ String resp = runFullImport(THREE_LEVEL_HIERARCHY_CONFIG);
+ String xpath = "//arr[@name='documents']/lst[arr[@name='id']/str='"+parentId1+"']/"+
+ "arr[@name='_childDocuments_']/lst[arr[@name='id']/str='"+childId+"']/"+
+ "arr[@name='_childDocuments_']/lst[arr[@name='id']/str='"+grandChildrenIds.get(0)+"']";
+ String results = TestHarness.validateXPath(resp,
+ xpath);
+ assertTrue("Debug documents does not contain child documents\n"+resp+"\n"+ xpath+
+ "\n"+results, results == null);
+
+ assertTrue("Update request processor processAdd was not called", TestUpdateRequestProcessor.processAddCalled);
+ assertTrue("Update request processor processCommit was not callled", TestUpdateRequestProcessor.processCommitCalled);
+ assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
+
+ // very simple asserts to check that we at least have correct num of docs indexed
+ assertQ(req("*:*"), "//*[@numFound='" + totalDocsNum + "']");
+ assertQ(req("type_s:parent"), "//*[@numFound='" + parentsNum + "']");
+ assertQ(req("type_s:child"), "//*[@numFound='" + childrenNum + "']");
+ assertQ(req("type_s:grand_child"), "//*[@numFound='" + grandChildrenNum + "']");
+
+ // let's check BlockJoin
+ // get first parent by any grand children
+ String randomGrandChildId = allGrandChildrenIds.get(random().nextInt(allGrandChildrenIds.size()));
+ Query query = createToParentQuery(parentType, FIELD_ID, randomGrandChildId);
+ assertSearch(query, FIELD_ID, parentId1);
+
+ // get first parent by any children
+ String randomChildId = firstParentChildrenIds.get(random().nextInt(firstParentChildrenIds.size()));
+ query = createToParentQuery(parentType, FIELD_ID, randomChildId);
+ assertSearch(query, FIELD_ID, parentId1);
+
+ // get parent by children by grand children
+ randomGrandChildId = grandChildrenIds.get(random().nextInt(grandChildrenIds.size()));
+ ToParentBlockJoinQuery childBlockJoinQuery = createToParentQuery(childType, FIELD_ID, randomGrandChildId);
+ ToParentBlockJoinQuery blockJoinQuery = new ToParentBlockJoinQuery(childBlockJoinQuery, createParentFilter(parentType), ScoreMode.Avg);
+ assertSearch(blockJoinQuery, FIELD_ID, parentId1);
+ }
+
+ @Test
+ public void testRandomDepthHierarchy() throws Exception {
+ final String parentType = "parent";
+
+ // Be aware that hierarchies grows exponentially, thus
+ // numbers bigger than 6 may lead to significant memory usage
+ // and cause OOME
+ int parentsNum = 2 + random().nextInt(3);
+ int depth = 2 + random().nextInt(3);
+
+ ContextHolder holder = new ContextHolder();
+
+ String config = createRandomizedConfig(depth, parentType, parentsNum, holder);
+ runFullImport(config);
+
+ assertTrue("Update request processor processAdd was not called", TestUpdateRequestProcessor.processAddCalled);
+ assertTrue("Update request processor processCommit was not callled", TestUpdateRequestProcessor.processCommitCalled);
+ assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
+
+ assertQ(req("type_s:" + parentType), "//*[@numFound='" + parentsNum + "']");
+ assertQ(req("-type_s:"+ parentType), "//*[@numFound='" + (holder.counter - parentsNum) + "']");
+
+ // let's check BlockJoin
+ Hierarchy randomHierarchy = holder.hierarchies.get(random().nextInt(holder.hierarchies.size()));
+
+ Query deepestQuery = createBlockJoinQuery(randomHierarchy);
+ assertSearch(deepestQuery, FIELD_ID, (String) randomHierarchy.elementData.get(FIELD_ID));
+ }
+
+ private Query createBlockJoinQuery(Hierarchy hierarchy) {
+ List<Hierarchy> elements = hierarchy.elements;
+ if (elements.isEmpty()) {
+ BooleanQuery.Builder childQuery = new BooleanQuery.Builder();
+ childQuery.add(new TermQuery(new Term(FIELD_ID, (String) hierarchy.elementData.get(FIELD_ID))), Occur.MUST);
+ return childQuery.build();
+ }
+
+ Query childQuery = createBlockJoinQuery(elements.get(random().nextInt(elements.size())));
+ return createToParentQuery(hierarchy.elementType, childQuery);
+ }
+
+ private ToParentBlockJoinQuery createToParentQuery(String parentType, String childField, String childFieldValue) {
+ BooleanQuery.Builder childQuery = new BooleanQuery.Builder();
+ childQuery.add(new TermQuery(new Term(childField, childFieldValue)), Occur.MUST);
+ ToParentBlockJoinQuery result = createToParentQuery(parentType, childQuery.build());
+
+ return result;
+ }
+
+ private ToParentBlockJoinQuery createToParentQuery(String parentType, Query childQuery) {
+ ToParentBlockJoinQuery blockJoinQuery = new ToParentBlockJoinQuery(childQuery, createParentFilter(parentType), ScoreMode.Avg);
+
+ return blockJoinQuery;
+ }
+
+ private void assertSearch(Query query, String field, String... values) throws IOException {
+ /* The limit of search queue is doubled to catch the error in case when for some reason there are more docs than expected */
+ SolrIndexSearcher searcher = req.getSearcher();
+ TopDocs result = searcher.search(query, values.length * 2);
+ assertEquals(values.length, result.totalHits.value);
+ List<String> actualValues = new ArrayList<String>();
+ for (int index = 0; index < values.length; ++index) {
+ Document doc = searcher.doc(result.scoreDocs[index].doc);
+ actualValues.add(doc.get(field));
+ }
+
+ for (String expectedValue: values) {
+ boolean removed = actualValues.remove(expectedValue);
+ if (!removed) {
+ fail("Search result does not contain expected values");
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<String> createDataIterator(String query, String type, String description, int count) {
+ List<Map<String, Object>> data = new ArrayList<Map<String, Object>>();
+ List<String> ids = new ArrayList<String>(count);
+ for (int index = 0; index < count; ++index) {
+ String docId = nextId();
+ ids.add(docId);
+ Map<String, Object> doc = createMap(FIELD_ID, docId, "desc", docId + " " + description, "type_s", type);
+ data.add(doc);
+ }
+ Collections.shuffle(data, random());
+ MockDataSource.setIterator(query, data.iterator());
+
+ return ids;
+ }
+
+ /**
+ * Creates randomized configuration of a specified depth. Simple configuration example:
+ *
+ * <pre>
+ *
+ * <dataConfig>
+ * <dataSource type="MockDataSource" />
+ * <document>
+ * <entity name="parent" query="SELECT * FROM parent">
+ * <field column="id" />
+ * <field column="desc" />
+ * <field column="type_s" />
+ * <entity child="true" name="parentChild0" query="select * from parentChild0 where parentChild0_parent_id='${parent.id}'">
+ * <field column="id" />
+ * <field column="desc" />
+ * <field column="type_s" />
+ * <entity child="true" name="parentChild0Child0" query="select * from parentChild0Child0 where parentChild0Child0_parent_id='${parentChild0.id}'">
+ * <field column="id" />
+ * <field column="desc" />
+ * <field column="type_s" />
+ * </entity>
+ * <entity child="true" name="parentChild0Child1" query="select * from parentChild0Child1 where parentChild0Child1_parent_id='${parentChild0.id}'">
+ * <field column="id" />
+ * <field column="desc" />
+ * <field column="type_s" />
+ * </entity>
+ * </entity>
+ * <entity child="true" name="parentChild1" query="select * from parentChild1 where parentChild1_parent_id='${parent.id}'">
+ * <field column="id" />
+ * <field column="desc" />
+ * <field column="type_s" />
+ * <entity child="true" name="parentChild1Child0" query="select * from parentChild1Child0 where parentChild1Child0_parent_id='${parentChild1.id}'">
+ * <field column="id" />
+ * <field column="desc" />
+ * <field column="type_s" />
+ * </entity>
+ * <entity child="true" name="parentChild1Child1" query="select * from parentChild1Child1 where parentChild1Child1_parent_id='${parentChild1.id}'">
+ * <field column="id" />
+ * <field column="desc" />
+ * <field column="type_s" />
+ * </entity>
+ * </entity>
+ * </entity>
+ * </document>
+ * </dataConfig>
+ *
+ * </pre>
+ *
+ * Internally configures MockDataSource.
+ **/
+ private String createRandomizedConfig(int depth, String parentType, int parentsNum, ContextHolder holder) {
+ List<Hierarchy> parentData = createMockedIterator(parentType, "SELECT * FROM " + parentType, parentsNum, holder);
+
+ holder.hierarchies = parentData;
+
+ String children = createChildren(parentType, 0, depth, parentData, holder);
+
+ String rootFields = createFieldsList(FIELD_ID, "desc", "type_s");
+ String rootEntity = StrUtils.formatString(ROOT_ENTITY_TEMPLATE, parentType, "SELECT * FROM " + parentType, rootFields, children);
+
+ String config = StrUtils.formatString(DATA_CONFIG_TEMPLATE, rootEntity);
+ return config;
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<Hierarchy> createMockedIterator(String type, String query, int amount, ContextHolder holder) {
+ List<Hierarchy> hierarchies = new ArrayList<Hierarchy>();
+ List<Map<String, Object>> data = new ArrayList<Map<String, Object>>();
+ for (int index = 0; index < amount; ++index) {
+ holder.counter++;
+ String idStr = String.valueOf(holder.counter);
+ Map<String, Object> element = createMap(FIELD_ID, idStr, "desc", type + "_" + holder.counter, "type_s", type);
+ data.add(element);
+
+ Hierarchy hierarchy = new Hierarchy();
+ hierarchy.elementType = type;
+ hierarchy.elementData = element;
+ hierarchies.add(hierarchy);
+ }
+
+ MockDataSource.setIterator(query, data.iterator());
+
+ return hierarchies;
+ }
+
+ private List<Hierarchy> createMockedIterator(String type, List<Hierarchy> parentData, ContextHolder holder) {
+ List<Hierarchy> result = new ArrayList<Hierarchy>();
+ for (Hierarchy parentHierarchy: parentData) {
+ Map<String, Object> data = parentHierarchy.elementData;
+ String id = (String) data.get(FIELD_ID);
+ String select = String.format(Locale.ROOT, "select * from %s where %s='%s'", type, type + "_parent_id", id);
+
+ // Number of actual children documents
+ int childrenNum = 1 + random().nextInt(3);
+ List<Hierarchy> childHierarchies = createMockedIterator(type, select, childrenNum, holder);
+ parentHierarchy.elements.addAll(childHierarchies);
+ result.addAll(childHierarchies);
+ }
+ return result;
+ }
+
+ private String createChildren(String parentName, int currentLevel, int maxLevel,
+ List<Hierarchy> parentData, ContextHolder holder) {
+
+ if (currentLevel == maxLevel) { //recursion base
+ return "";
+ }
+
+ // number of different children <b>types</b> of parent, i.e. parentChild0, parentChild1
+ // @see #createMockedIterator for the actual number of each children type
+ int childrenNumber = 2 + random().nextInt(3);
+ StringBuilder builder = new StringBuilder();
+ for (int childIndex = 0; childIndex < childrenNumber; ++childIndex) {
+ String childName = parentName + "Child" + childIndex;
+ String fields = createFieldsList(FIELD_ID, "desc", "type_s");
+ String select = String.format(Locale.ROOT, "select * from %s where %s='%s'", childName, childName + "_parent_id", "${" + parentName + ".id}");
+
+ //for each child entity create several iterators
+ List<Hierarchy> childData = createMockedIterator(childName, parentData, holder);
+
+ String subChildren = createChildren(childName, currentLevel + 1, maxLevel, childData, holder);
+ String child = StrUtils.formatString(CHILD_ENTITY_TEMPLATE, childName, select, fields, subChildren);
+ builder.append(child);
+ builder.append('\n');
+ }
+
+ return builder.toString();
+ }
+
+ private String createFieldsList(String... fields) {
+ StringBuilder builder = new StringBuilder();
+ for (String field: fields) {
+ String text = String.format(Locale.ROOT, "<field column='%s' />", field);
+ builder.append(text);
+ builder.append('\n');
+ }
+ return builder.toString();
+ }
+
+ private static final String THREE_LEVEL_HIERARCHY_CONFIG = "<dataConfig>\n" +
+ " <dataSource type='MockDataSource' />\n" +
+ " <document>\n" +
+ " <entity name='PARENT' query='select * from PARENT'>\n" +
+ " <field column='id' />\n" +
+ " <field column='desc' />\n" +
+ " <field column='type_s' />\n" +
+ " <entity child='true' name='CHILD' query=\"select * from CHILD where parent_id='${PARENT.id}'\">\n" +
+ " <field column='id' />\n" +
+ " <field column='desc' />\n" +
+ " <field column='type_s' />\n" +
+ " <entity child='true' name='GRANDCHILD' query=\"select * from GRANDCHILD where parent_id='${CHILD.id}'\">\n" +
+ " <field column='id' />\n" +
+ " <field column='desc' />\n" +
+ " <field column='type_s' />\n" +
+ " </entity>\n" +
+ " </entity>\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ /** {0} is rootEntity block **/
+ private static final String DATA_CONFIG_TEMPLATE = "<dataConfig><dataSource type=\"MockDataSource\" />\n<document>\n {0}</document></dataConfig>";
+
+ /**
+ * {0} - entityName,
+ * {1} - select query
+ * {2} - fieldsList
+ * {3} - childEntitiesList
+ **/
+ private static final String ROOT_ENTITY_TEMPLATE = "<entity name=\"{0}\" query=\"{1}\">\n{2} {3}\n</entity>\n";
+
+ /**
+ * {0} - entityName,
+ * {1} - select query
+ * {2} - fieldsList
+ * {3} - childEntitiesList
+ **/
+ private static final String CHILD_ENTITY_TEMPLATE = "<entity " + ConfigNameConstants.CHILD + "=\"true\" name=\"{0}\" query=\"{1}\">\n {2} {3} </entity>\n";
+
+ private BitSetProducer createParentFilter(String type) {
+ BooleanQuery.Builder parentQuery = new BooleanQuery.Builder();
+ parentQuery.add(new TermQuery(new Term("type_s", type)), Occur.MUST);
+ return new QueryBitSetProducer(parentQuery.build());
+ }
+
+ private String nextId() {
+ ++id;
+ return String.valueOf(id);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestJdbcDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestJdbcDataSource.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestJdbcDataSource.java
new file mode 100644
index 0000000..e44e7f9
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestJdbcDataSource.java
@@ -0,0 +1,662 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.sql.Connection;
+import java.sql.Driver;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import javax.sql.DataSource;
+
+import org.apache.solr.common.util.SuppressForbidden;
+import org.apache.solr.handler.dataimport.JdbcDataSource.ResultSetIterator;
+import static org.mockito.Mockito.*;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * <p>
+ * Test for JdbcDataSource
+ * </p>
+ * <p>
+ * Note: The tests are ignored for the lack of DB support for testing
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestJdbcDataSource extends AbstractDataImportHandlerTestCase {
+ private Driver driver;
+ private DataSource dataSource;
+ private Connection connection;
+ private JdbcDataSource jdbcDataSource = new JdbcDataSource();
+ List<Map<String, String>> fields = new ArrayList<>();
+
+ Context context = AbstractDataImportHandlerTestCase.getContext(null, null,
+ jdbcDataSource, Context.FULL_DUMP, fields, null);
+
+ Properties props = new Properties();
+
+ String sysProp = System.getProperty("java.naming.factory.initial");
+
+ @BeforeClass
+ public static void beforeClass() {
+ assumeWorkingMockito();
+ }
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ System.setProperty("java.naming.factory.initial",
+ MockInitialContextFactory.class.getName());
+
+ driver = mock(Driver.class);
+ dataSource = mock(DataSource.class);
+ connection = mock(Connection.class);
+ props.clear();
+ }
+
+ @Override
+ @After
+ public void tearDown() throws Exception {
+ if (sysProp == null) {
+ System.getProperties().remove("java.naming.factory.initial");
+ } else {
+ System.setProperty("java.naming.factory.initial", sysProp);
+ }
+ super.tearDown();
+ reset(driver, dataSource, connection);
+ }
+
+ @Test
+ public void testRetrieveFromJndi() throws Exception {
+ MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
+
+ props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
+
+ when(dataSource.getConnection()).thenReturn(connection);
+
+ Connection conn = jdbcDataSource.createConnectionFactory(context, props)
+ .call();
+
+ verify(connection).setAutoCommit(false);
+ verify(dataSource).getConnection();
+
+ assertSame("connection", conn, connection);
+ }
+
+ @Test
+ public void testRetrieveFromJndiWithCredentials() throws Exception {
+ MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
+
+ props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
+ props.put("user", "Fred");
+ props.put("password", "4r3d");
+ props.put("holdability", "HOLD_CURSORS_OVER_COMMIT");
+
+ when(dataSource.getConnection("Fred", "4r3d")).thenReturn(
+ connection);
+
+ Connection conn = jdbcDataSource.createConnectionFactory(context, props)
+ .call();
+
+ verify(connection).setAutoCommit(false);
+ verify(connection).setHoldability(1);
+ verify(dataSource).getConnection("Fred", "4r3d");
+
+ assertSame("connection", conn, connection);
+ }
+
+ @Test
+ public void testRetrieveFromJndiWithCredentialsEncryptedAndResolved() throws Exception {
+ MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
+
+ String user = "Fred";
+ String plainPassword = "MyPassword";
+ String encryptedPassword = "U2FsdGVkX18QMjY0yfCqlfBMvAB4d3XkwY96L7gfO2o=";
+ String propsNamespace = "exampleNamespace";
+
+ props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
+
+ props.put("user", "${" +propsNamespace +".user}");
+ props.put("encryptKeyFile", "${" +propsNamespace +".encryptKeyFile}");
+ props.put("password", "${" +propsNamespace +".password}");
+
+ when(dataSource.getConnection(user, plainPassword)).thenReturn(
+ connection);
+
+ Map<String,Object> values = new HashMap<>();
+ values.put("user", user);
+ values.put("encryptKeyFile", createEncryptionKeyFile());
+ values.put("password", encryptedPassword);
+ context.getVariableResolver().addNamespace(propsNamespace, values);
+
+ jdbcDataSource.init(context, props);
+ Connection conn = jdbcDataSource.getConnection();
+
+ verify(connection).setAutoCommit(false);
+ verify(dataSource).getConnection(user, plainPassword);
+
+ assertSame("connection", conn, connection);
+ }
+
+ @Test
+ public void testRetrieveFromJndiWithCredentialsWithEncryptedAndResolvedPwd() throws Exception {
+ MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
+
+ Properties properties = new Properties();
+ properties.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
+ properties.put("user", "Fred");
+ properties.put("encryptKeyFile", "${foo.bar}");
+ properties.put("password", "U2FsdGVkX18QMjY0yfCqlfBMvAB4d3XkwY96L7gfO2o=");
+ when(dataSource.getConnection("Fred", "MyPassword")).thenReturn(
+ connection);
+
+ Map<String,Object> values = new HashMap<>();
+ values.put("bar", createEncryptionKeyFile());
+ context.getVariableResolver().addNamespace("foo", values);
+
+ jdbcDataSource.init(context, properties);
+ jdbcDataSource.getConnection();
+
+ verify(connection).setAutoCommit(false);
+ verify(dataSource).getConnection("Fred", "MyPassword");
+ }
+
+ @Test
+ public void testRetrieveFromJndiFailureNotHidden() throws Exception {
+ MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
+
+ props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
+
+ SQLException sqlException = new SQLException("fake");
+ when(dataSource.getConnection()).thenThrow(sqlException);
+
+ try {
+ jdbcDataSource.createConnectionFactory(context, props).call();
+ } catch (SQLException ex) {
+ assertSame(sqlException, ex);
+ }
+
+ verify(dataSource).getConnection();
+ }
+
+ @Test
+ public void testClosesConnectionWhenExceptionThrownOnSetAutocommit() throws Exception {
+ MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
+
+ props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
+
+ SQLException sqlException = new SQLException("fake");
+ when(dataSource.getConnection()).thenReturn(connection);
+ doThrow(sqlException).when(connection).setAutoCommit(false);
+
+ try {
+ jdbcDataSource.createConnectionFactory(context, props).call();
+ } catch (DataImportHandlerException ex) {
+ assertSame(sqlException, ex.getCause());
+ }
+ verify(dataSource).getConnection();
+ verify(connection).setAutoCommit(false);
+ verify(connection).close();
+ }
+
+ @Test
+ public void testClosesStatementWhenExceptionThrownOnExecuteQuery() throws Exception {
+ MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
+
+ props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
+ when(dataSource.getConnection()).thenReturn(connection);
+
+ jdbcDataSource.init(context, props);
+
+ SQLException sqlException = new SQLException("fake");
+ Statement statement = mock(Statement.class);
+ when(connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY))
+ .thenReturn(statement);
+ when(statement.execute("query")).thenThrow(sqlException);
+
+ try {
+ jdbcDataSource.getData("query");
+ fail("exception expected");
+ } catch (DataImportHandlerException ex) {
+ assertSame(sqlException, ex.getCause());
+ }
+
+ verify(dataSource).getConnection();
+ verify(connection).setAutoCommit(false);
+ verify(connection).createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ verify(statement).setFetchSize(500);
+ verify(statement).setMaxRows(0);
+ verify(statement).execute("query");
+ verify(statement).close();
+ }
+
+ @Test
+ public void testClosesStatementWhenResultSetNull() throws Exception {
+ MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
+
+ props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
+ when(dataSource.getConnection()).thenReturn(connection);
+
+ jdbcDataSource.init(context, props);
+
+ Statement statement = mock(Statement.class);
+ when(connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY))
+ .thenReturn(statement);
+ when(statement.execute("query")).thenReturn(false);
+ when(statement.getUpdateCount()).thenReturn(-1);
+
+ jdbcDataSource.getData("query");
+
+ verify(dataSource).getConnection();
+ verify(connection).setAutoCommit(false);
+ verify(connection).createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ verify(statement).setFetchSize(500);
+ verify(statement).setMaxRows(0);
+ verify(statement).execute("query");
+ verify(statement).getUpdateCount();
+ verify(statement).close();
+ }
+
+ @Test
+ public void testClosesStatementWhenHasNextCalledAndResultSetNull() throws Exception {
+
+ MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
+
+ props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
+ when(dataSource.getConnection()).thenReturn(connection);
+
+ jdbcDataSource.init(context, props);
+
+ Statement statement = mock(Statement.class);
+ when(connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY))
+ .thenReturn(statement);
+ when(statement.execute("query")).thenReturn(true);
+ ResultSet resultSet = mock(ResultSet.class);
+ when(statement.getResultSet()).thenReturn(resultSet);
+ ResultSetMetaData metaData = mock(ResultSetMetaData.class);
+ when(resultSet.getMetaData()).thenReturn(metaData);
+ when(metaData.getColumnCount()).thenReturn(0);
+
+ Iterator<Map<String,Object>> data = jdbcDataSource.getData("query");
+
+ ResultSetIterator resultSetIterator = (ResultSetIterator) data.getClass().getDeclaredField("this$1").get(data);
+ resultSetIterator.setResultSet(null);
+
+ data.hasNext();
+
+ verify(dataSource).getConnection();
+ verify(connection).setAutoCommit(false);
+ verify(connection).createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ verify(statement).setFetchSize(500);
+ verify(statement).setMaxRows(0);
+ verify(statement).execute("query");
+ verify(statement).getResultSet();
+ verify(statement).close();
+ verify(resultSet).getMetaData();
+ verify(metaData).getColumnCount();
+ }
+
+ @Test
+ public void testClosesResultSetAndStatementWhenDataSourceIsClosed() throws Exception {
+
+ MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
+
+ props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
+ when(dataSource.getConnection()).thenReturn(connection);
+
+ jdbcDataSource.init(context, props);
+
+ Statement statement = mock(Statement.class);
+ when(connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY))
+ .thenReturn(statement);
+ when(statement.execute("query")).thenReturn(true);
+ ResultSet resultSet = mock(ResultSet.class);
+ when(statement.getResultSet()).thenReturn(resultSet);
+ ResultSetMetaData metaData = mock(ResultSetMetaData.class);
+ when(resultSet.getMetaData()).thenReturn(metaData);
+ when(metaData.getColumnCount()).thenReturn(0);
+
+ jdbcDataSource.getData("query");
+ jdbcDataSource.close();
+
+ verify(dataSource).getConnection();
+ verify(connection).setAutoCommit(false);
+ verify(connection).createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ verify(statement).setFetchSize(500);
+ verify(statement).setMaxRows(0);
+ verify(statement).execute("query");
+ verify(statement).getResultSet();
+ verify(resultSet).getMetaData();
+ verify(metaData).getColumnCount();
+ verify(resultSet).close();
+ verify(statement).close();
+ verify(connection).commit();
+ verify(connection).close();
+ }
+
+ @Test
+ public void testClosesCurrentResultSetIteratorWhenNewOneIsCreated() throws Exception {
+
+ MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
+
+ props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
+ when(dataSource.getConnection()).thenReturn(connection);
+
+ jdbcDataSource.init(context, props);
+
+ Statement statement = mock(Statement.class);
+ when(connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY))
+ .thenReturn(statement);
+ when(statement.execute("query")).thenReturn(true);
+ ResultSet resultSet = mock(ResultSet.class);
+ when(statement.getResultSet()).thenReturn(resultSet);
+ ResultSetMetaData metaData = mock(ResultSetMetaData.class);
+ when(resultSet.getMetaData()).thenReturn(metaData);
+ when(metaData.getColumnCount()).thenReturn(0);
+ when(statement.execute("other query")).thenReturn(false);
+ when(statement.getUpdateCount()).thenReturn(-1);
+
+ jdbcDataSource.getData("query");
+ jdbcDataSource.getData("other query");
+
+ verify(dataSource).getConnection();
+ verify(connection).setAutoCommit(false);
+ verify(connection, times(2)).createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ verify(statement, times(2)).setFetchSize(500);
+ verify(statement, times(2)).setMaxRows(0);
+ verify(statement).execute("query");
+ verify(statement).getResultSet();
+ verify(resultSet).getMetaData();
+ verify(metaData).getColumnCount();
+ verify(resultSet).close();
+ verify(statement, times(2)).close();
+ verify(statement).execute("other query");
+ }
+
+ @Test
+ public void testMultipleResultsSets_UpdateCountUpdateCountResultSet() throws Exception {
+ MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
+ props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
+ when(dataSource.getConnection()).thenReturn(connection);
+ jdbcDataSource.init(context, props);
+
+ Statement statement = mock(Statement.class);
+ when(connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY))
+ .thenReturn(statement);
+ when(statement.execute("query")).thenReturn(false);
+ when(statement.getUpdateCount()).thenReturn(1);
+ when(statement.getMoreResults()).thenReturn(false).thenReturn(true);
+ ResultSet resultSet = mock(ResultSet.class);
+ when(statement.getResultSet()).thenReturn(resultSet);
+ ResultSetMetaData metaData = mock(ResultSetMetaData.class);
+ when(resultSet.getMetaData()).thenReturn(metaData);
+ when(metaData.getColumnCount()).thenReturn(0);
+
+ final ResultSetIterator resultSetIterator = jdbcDataSource.new ResultSetIterator("query");
+ assertSame(resultSet, resultSetIterator.getResultSet());
+
+ verify(connection).setAutoCommit(false);
+ verify(connection).createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ verify(statement).setFetchSize(500);
+ verify(statement).setMaxRows(0);
+ verify(statement).execute("query");
+ verify(statement, times(2)).getUpdateCount();
+ verify(statement, times(2)).getMoreResults();
+ }
+
+ @Test
+ public void testMultipleResultsSets_ResultSetResultSet() throws Exception {
+ MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
+ props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
+ when(dataSource.getConnection()).thenReturn(connection);
+ jdbcDataSource.init(context, props);
+ connection.setAutoCommit(false);
+
+ Statement statement = mock(Statement.class);
+ when(connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY))
+ .thenReturn(statement);
+ when(statement.execute("query")).thenReturn(true);
+ ResultSet resultSet1 = mock(ResultSet.class);
+ ResultSet resultSet2 = mock(ResultSet.class);
+ when(statement.getResultSet()).thenReturn(resultSet1).thenReturn(resultSet2).thenReturn(null);
+ when(statement.getMoreResults()).thenReturn(true).thenReturn(false);
+ ResultSetMetaData metaData1 = mock(ResultSetMetaData.class);
+ when(resultSet1.getMetaData()).thenReturn(metaData1);
+ when(metaData1.getColumnCount()).thenReturn(0);
+ when(resultSet1.next()).thenReturn(false);
+ ResultSetMetaData metaData2 = mock(ResultSetMetaData.class);
+ when(resultSet2.getMetaData()).thenReturn(metaData2);
+ when(metaData2.getColumnCount()).thenReturn(0);
+ when(resultSet2.next()).thenReturn(true).thenReturn(false);
+ when(statement.getUpdateCount()).thenReturn(-1);
+
+ final ResultSetIterator resultSetIterator = jdbcDataSource.new ResultSetIterator("query");
+ assertSame(resultSet1, resultSetIterator.getResultSet());
+ assertTrue(resultSetIterator.hasnext());
+ assertSame(resultSet2, resultSetIterator.getResultSet());
+ assertFalse(resultSetIterator.hasnext());
+
+ verify(dataSource).getConnection();
+ verify(connection, times(2)).setAutoCommit(false);
+ verify(connection).createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ verify(statement).setFetchSize(500);
+ verify(statement).setMaxRows(0);
+ verify(statement).execute("query");
+ verify(statement, times(2)).getResultSet();
+ verify(resultSet1).getMetaData();
+ verify(metaData1).getColumnCount();
+ verify(resultSet1).next();
+ verify(resultSet1).close();
+ verify(resultSet2).getMetaData();
+ verify(metaData2).getColumnCount();
+ verify(resultSet2, times(2)).next();
+ verify(resultSet2).close();
+ verify(statement, times(2)).getMoreResults();
+ verify(statement).getUpdateCount();
+ verify(statement).close();
+ }
+
+ @Test
+ public void testRetrieveFromDriverManager() throws Exception {
+ // we're not (directly) using a Mockito based mock class here because it won't have a consistent class name
+ // that will work with DriverManager's class bindings
+ MockDriver mockDriver = new MockDriver(connection);
+ DriverManager.registerDriver(mockDriver);
+ try {
+ props.put(JdbcDataSource.DRIVER, MockDriver.class.getName());
+ props.put(JdbcDataSource.URL, MockDriver.MY_JDBC_URL);
+ props.put("holdability", "HOLD_CURSORS_OVER_COMMIT");
+
+ Connection conn = jdbcDataSource.createConnectionFactory(context, props).call();
+
+ verify(connection).setAutoCommit(false);
+ verify(connection).setHoldability(1);
+
+ assertSame("connection", conn, connection);
+ } catch(Exception e) {
+ throw e;
+ } finally {
+ DriverManager.deregisterDriver(mockDriver);
+ }
+ }
+
+
+ @Test
+ public void testEmptyResultSet() throws Exception {
+ MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
+
+ props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
+ when(dataSource.getConnection()).thenReturn(connection);
+
+ jdbcDataSource.init(context, props);
+
+ Statement statement = mock(Statement.class);
+ when(connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY))
+ .thenReturn(statement);
+ when(statement.execute("query")).thenReturn(true);
+ ResultSet resultSet = mock(ResultSet.class);
+ when(statement.getResultSet()).thenReturn(resultSet);
+ ResultSetMetaData metaData = mock(ResultSetMetaData.class);
+ when(resultSet.getMetaData()).thenReturn(metaData);
+ when(metaData.getColumnCount()).thenReturn(0);
+ when(resultSet.next()).thenReturn(false);
+ when(statement.getMoreResults()).thenReturn(false);
+ when(statement.getUpdateCount()).thenReturn(-1);
+
+ Iterator<Map<String,Object>> resultSetIterator = jdbcDataSource.getData("query");
+ resultSetIterator.hasNext();
+ resultSetIterator.hasNext();
+
+ verify(connection).setAutoCommit(false);
+ verify(connection).createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ verify(statement).setFetchSize(500);
+ verify(statement).setMaxRows(0);
+ verify(statement).execute("query");
+ verify(statement).getResultSet();
+ verify(resultSet).getMetaData();
+ verify(metaData).getColumnCount();
+ verify(resultSet).next();
+ verify(resultSet).close();
+ verify(statement).getMoreResults();
+ verify(statement).getUpdateCount();
+ verify(statement).close();
+ }
+
+ @Test
+ @Ignore("Needs a Mock database server to work")
+ public void testBasic() throws Exception {
+ JdbcDataSource dataSource = new JdbcDataSource();
+ Properties p = new Properties();
+ p.put("driver", "com.mysql.jdbc.Driver");
+ p.put("url", "jdbc:mysql://127.0.0.1/autos");
+ p.put("user", "root");
+ p.put("password", "");
+
+ List<Map<String, String>> flds = new ArrayList<>();
+ Map<String, String> f = new HashMap<>();
+ f.put("column", "trim_id");
+ f.put("type", "long");
+ flds.add(f);
+ f = new HashMap<>();
+ f.put("column", "msrp");
+ f.put("type", "float");
+ flds.add(f);
+
+ Context c = getContext(null, null,
+ dataSource, Context.FULL_DUMP, flds, null);
+ dataSource.init(c, p);
+ Iterator<Map<String, Object>> i = dataSource
+ .getData("select make,model,year,msrp,trim_id from atrimlisting where make='Acura'");
+ int count = 0;
+ Object msrp = null;
+ Object trim_id = null;
+ while (i.hasNext()) {
+ Map<String, Object> map = i.next();
+ msrp = map.get("msrp");
+ trim_id = map.get("trim_id");
+ count++;
+ }
+ assertEquals(5, count);
+ assertEquals(Float.class, msrp.getClass());
+ assertEquals(Long.class, trim_id.getClass());
+ }
+
+ private String createEncryptionKeyFile() throws IOException {
+ File tmpdir = createTempDir().toFile();
+ byte[] content = "secret".getBytes(StandardCharsets.UTF_8);
+ createFile(tmpdir, "enckeyfile.txt", content, false);
+ return new File(tmpdir, "enckeyfile.txt").getAbsolutePath();
+ }
+
+ /**
+ * A stub driver that returns our mocked connection for connection URL {@link #MY_JDBC_URL}.
+ * <p>
+ * This class is used instead of a Mockito mock because {@link DriverManager} uses the class
+ * name to lookup the driver and also requires the driver to behave in a sane way, if other
+ * drivers are registered in the runtime. A simple Mockito mock is likely to break
+ * depending on JVM runtime version. So this class implements a full {@link Driver},
+ * so {@code DriverManager} can do whatever it wants to find the correct driver for a URL.
+ */
+ public static final class MockDriver implements Driver {
+ public static final String MY_JDBC_URL = "jdbc:fakedb";
+ private final Connection conn;
+
+ public MockDriver() throws SQLException {
+ throw new AssertionError("The driver should never be directly instantiated by DIH's JdbcDataSource");
+ }
+
+ MockDriver(Connection conn) throws SQLException {
+ this.conn = conn;
+ }
+
+ @Override
+ public boolean acceptsURL(String url) throws java.sql.SQLException {
+ return MY_JDBC_URL.equals(url);
+ }
+
+ @Override
+ public Connection connect(String url, Properties info) throws java.sql.SQLException {
+ return acceptsURL(url) ? conn : null;
+ }
+
+ @Override
+ public int getMajorVersion() {
+ return 1;
+ }
+
+ @Override
+ public int getMinorVersion() {
+ return 0;
+ }
+
+ @SuppressForbidden(reason="Required by JDBC")
+ @Override
+ public java.util.logging.Logger getParentLogger() throws java.sql.SQLFeatureNotSupportedException {
+ throw new java.sql.SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public java.sql.DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException {
+ return new java.sql.DriverPropertyInfo[0];
+ }
+
+ @Override
+ public boolean jdbcCompliant() {
+ // we are not fully compliant:
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestJdbcDataSourceConvertType.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestJdbcDataSourceConvertType.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestJdbcDataSourceConvertType.java
new file mode 100644
index 0000000..94f437c
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestJdbcDataSourceConvertType.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Properties;
+
+@ThreadLeakAction({ThreadLeakAction.Action.WARN})
+@ThreadLeakLingering(linger = 0)
+@ThreadLeakZombies(ThreadLeakZombies.Consequence.CONTINUE)
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
+public class TestJdbcDataSourceConvertType extends AbstractDataImportHandlerTestCase {
+ public void testConvertType() throws Throwable {
+ final Locale loc = Locale.getDefault();
+ assumeFalse("Derby is not happy with locale sr-Latn-*",
+ Objects.equals(new Locale("sr").getLanguage(), loc.getLanguage()) &&
+ Objects.equals("Latn", loc.getScript()));
+
+ // ironically convertType=false causes BigDecimal to String conversion
+ convertTypeTest("false", String.class);
+
+ // convertType=true uses the "long" conversion (see mapping of some_i to "long")
+ convertTypeTest("true", Long.class);
+ }
+
+ private void convertTypeTest(String convertType, Class resultClass) throws Throwable {
+ JdbcDataSource dataSource = new JdbcDataSource();
+ Properties p = new Properties();
+ p.put("driver", "org.apache.derby.jdbc.EmbeddedDriver");
+ p.put("url", "jdbc:derby:memory:tempDB;create=true;territory=en_US");
+ p.put("convertType", convertType);
+
+ List<Map<String, String>> flds = new ArrayList<>();
+ Map<String, String> f = new HashMap<>();
+ f.put("column", "some_i");
+ f.put("type", "long");
+ flds.add(f);
+
+ Context c = getContext(null, null,
+ dataSource, Context.FULL_DUMP, flds, null);
+ dataSource.init(c, p);
+ Iterator<Map<String, Object>> i = dataSource
+ .getData("select 1 as id, CAST(9999 AS DECIMAL) as \"some_i\" from sysibm.sysdummy1");
+ assertTrue(i.hasNext());
+ Map<String, Object> map = i.next();
+ Object val = map.get("some_i");
+ assertEquals(resultClass, val.getClass());
+
+ dataSource.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestLineEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestLineEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestLineEntityProcessor.java
new file mode 100644
index 0000000..eb9989c
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestLineEntityProcessor.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.*;
+
+
+/**
+ * <p> Test for TestLineEntityProcessor </p>
+ *
+ *
+ * @since solr 1.4
+ */
+public class TestLineEntityProcessor extends AbstractDataImportHandlerTestCase {
+
+ @Test
+ /************************************************************************/
+ public void testSimple() throws IOException {
+
+ /* we want to create the equiv of :-
+ * <entity name="list_all_files"
+ * processor="LineEntityProcessor"
+ * fileName="dummy.lis"
+ * />
+ */
+
+ Map attrs = createMap(
+ LineEntityProcessor.URL, "dummy.lis",
+ LineEntityProcessor.ACCEPT_LINE_REGEX, null,
+ LineEntityProcessor.SKIP_LINE_REGEX, null
+ );
+
+ Context c = getContext(
+ null, //parentEntity
+ new VariableResolver(), //resolver
+ getDataSource(filecontents), //parentDataSource
+ Context.FULL_DUMP, //currProcess
+ Collections.EMPTY_LIST, //entityFields
+ attrs //entityAttrs
+ );
+ LineEntityProcessor ep = new LineEntityProcessor();
+ ep.init(c);
+
+ /// call the entity processor to the list of lines
+ if (VERBOSE) System.out.print("\n");
+ List<String> fList = new ArrayList<>();
+ while (true) {
+ Map<String, Object> f = ep.nextRow();
+ if (f == null) break;
+ fList.add((String) f.get("rawLine"));
+ if (VERBOSE) System.out.print(" rawLine='" + f.get("rawLine") + "'\n");
+ }
+ assertEquals(24, fList.size());
+ }
+
+ @Test
+ /************************************************************************/
+ public void testOnly_xml_files() throws IOException {
+
+ /* we want to create the equiv of :-
+ * <entity name="list_all_files"
+ * processor="LineEntityProcessor"
+ * fileName="dummy.lis"
+ * acceptLineRegex="xml"
+ * />
+ */
+ Map attrs = createMap(
+ LineEntityProcessor.URL, "dummy.lis",
+ LineEntityProcessor.ACCEPT_LINE_REGEX, "xml",
+ LineEntityProcessor.SKIP_LINE_REGEX, null
+ );
+
+ Context c = getContext(
+ null, //parentEntity
+ new VariableResolver(), //resolver
+ getDataSource(filecontents), //parentDataSource
+ Context.FULL_DUMP, //currProcess
+ Collections.EMPTY_LIST, //entityFields
+ attrs //entityAttrs
+ );
+ LineEntityProcessor ep = new LineEntityProcessor();
+ ep.init(c);
+
+ /// call the entity processor to the list of lines
+ List<String> fList = new ArrayList<>();
+ while (true) {
+ Map<String, Object> f = ep.nextRow();
+ if (f == null) break;
+ fList.add((String) f.get("rawLine"));
+ }
+ assertEquals(5, fList.size());
+ }
+
+ @Test
+ /************************************************************************/
+ public void testOnly_xml_files_no_xsd() throws IOException {
+ /* we want to create the equiv of :-
+ * <entity name="list_all_files"
+ * processor="LineEntityProcessor"
+ * fileName="dummy.lis"
+ * acceptLineRegex="\\.xml"
+ * omitLineRegex="\\.xsd"
+ * />
+ */
+ Map attrs = createMap(
+ LineEntityProcessor.URL, "dummy.lis",
+ LineEntityProcessor.ACCEPT_LINE_REGEX, "\\.xml",
+ LineEntityProcessor.SKIP_LINE_REGEX, "\\.xsd"
+ );
+
+ Context c = getContext(
+ null, //parentEntity
+ new VariableResolver(), //resolver
+ getDataSource(filecontents), //parentDataSource
+ Context.FULL_DUMP, //currProcess
+ Collections.EMPTY_LIST, //entityFields
+ attrs //entityAttrs
+ );
+ LineEntityProcessor ep = new LineEntityProcessor();
+ ep.init(c);
+
+ /// call the entity processor to walk the directory
+ List<String> fList = new ArrayList<>();
+ while (true) {
+ Map<String, Object> f = ep.nextRow();
+ if (f == null) break;
+ fList.add((String) f.get("rawLine"));
+ }
+ assertEquals(4, fList.size());
+ }
+
+ @Test
+ /************************************************************************/
+ public void testNo_xsd_files() throws IOException {
+ /* we want to create the equiv of :-
+ * <entity name="list_all_files"
+ * processor="LineEntityProcessor"
+ * fileName="dummy.lis"
+ * omitLineRegex="\\.xsd"
+ * />
+ */
+ Map attrs = createMap(
+ LineEntityProcessor.URL, "dummy.lis",
+ LineEntityProcessor.SKIP_LINE_REGEX, "\\.xsd"
+ );
+
+ Context c = getContext(
+ null, //parentEntity
+ new VariableResolver(), //resolver
+ getDataSource(filecontents), //parentDataSource
+ Context.FULL_DUMP, //currProcess
+ Collections.EMPTY_LIST, //entityFields
+ attrs //entityAttrs
+ );
+ LineEntityProcessor ep = new LineEntityProcessor();
+ ep.init(c);
+
+ /// call the entity processor to walk the directory
+ List<String> fList = new ArrayList<>();
+ while (true) {
+ Map<String, Object> f = ep.nextRow();
+ if (f == null) break;
+ fList.add((String) f.get("rawLine"));
+ }
+ assertEquals(18, fList.size());
+ }
+
+ /**
+ * ********************************************************************
+ */
+ public static Map<String, String> createField(
+ String col, // DIH column name
+ String type, // field type from schema.xml
+ String srcCol, // DIH transformer attribute 'sourceColName'
+ String re, // DIH regex attribute 'regex'
+ String rw, // DIH regex attribute 'replaceWith'
+ String gn // DIH regex attribute 'groupNames'
+ ) {
+ HashMap<String, String> vals = new HashMap<>();
+ vals.put("column", col);
+ vals.put("type", type);
+ vals.put("sourceColName", srcCol);
+ vals.put("regex", re);
+ vals.put("replaceWith", rw);
+ vals.put("groupNames", gn);
+ return vals;
+ }
+
+ private DataSource<Reader> getDataSource(final String xml) {
+ return new DataSource<Reader>() {
+ @Override
+ public void init(Context context, Properties initProps) {
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public Reader getData(String query) {
+ return new StringReader(xml);
+ }
+ };
+ }
+
+ private static final String filecontents =
+ "\n" +
+ "# this is what the output from 'find . -ls; looks like, athough the format\n" +
+ "# of the time stamp varies depending on the age of the file and your LANG \n" +
+ "# env setting\n" +
+ "412577 0 drwxr-xr-x 6 user group 204 1 Apr 10:53 /Volumes/spare/ts\n" +
+ "412582 0 drwxr-xr-x 13 user group 442 1 Apr 10:18 /Volumes/spare/ts/config\n" +
+ "412583 24 -rwxr-xr-x 1 user group 8318 1 Apr 11:10 /Volumes/spare/ts/config/dc.xsd\n" +
+ "412584 32 -rwxr-xr-x 1 user group 12847 1 Apr 11:10 /Volumes/spare/ts/config/dcterms.xsd\n" +
+ "412585 8 -rwxr-xr-x 1 user group 3156 1 Apr 11:10 /Volumes/spare/ts/config/s-deliver.css\n" +
+ "412586 192 -rwxr-xr-x 1 user group 97764 1 Apr 11:10 /Volumes/spare/ts/config/s-deliver.xsl\n" +
+ "412587 224 -rwxr-xr-x 1 user group 112700 1 Apr 11:10 /Volumes/spare/ts/config/sml-delivery-2.1.xsd\n" +
+ "412588 208 -rwxr-xr-x 1 user group 103419 1 Apr 11:10 /Volumes/spare/ts/config/sml-delivery-norm-2.0.dtd\n" +
+ "412589 248 -rwxr-xr-x 1 user group 125296 1 Apr 11:10 /Volumes/spare/ts/config/sml-delivery-norm-2.1.dtd\n" +
+ "412590 72 -rwxr-xr-x 1 user group 36256 1 Apr 11:10 /Volumes/spare/ts/config/jm.xsd\n" +
+ "412591 8 -rwxr-xr-x 1 user group 990 1 Apr 11:10 /Volumes/spare/ts/config/video.gif\n" +
+ "412592 8 -rwxr-xr-x 1 user group 1498 1 Apr 11:10 /Volumes/spare/ts/config/xlink.xsd\n" +
+ "412593 8 -rwxr-xr-x 1 user group 1155 1 Apr 11:10 /Volumes/spare/ts/config/xml.xsd\n" +
+ "412594 0 drwxr-xr-x 4 user group 136 1 Apr 10:18 /Volumes/spare/ts/acm19\n" +
+ "412621 0 drwxr-xr-x 57 user group 1938 1 Apr 10:18 /Volumes/spare/ts/acm19/data\n" +
+ "412622 24 -rwxr-xr-x 1 user group 8894 1 Apr 11:09 /Volumes/spare/ts/acm19/data/00000510.xml\n" +
+ "412623 32 -rwxr-xr-x 1 user group 14124 1 Apr 11:09 /Volumes/spare/ts/acm19/data/00000603.xml\n" +
+ "412624 24 -rwxr-xr-x 1 user group 11976 1 Apr 11:09 /Volumes/spare/ts/acm19/data/00001292.xml\n" +
+ "# tacked on an extra line to cause a file to be deleted.\n" +
+ "DELETE /Volumes/spare/ts/acm19/data/00001292old.xml\n" +
+ "";
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNestedChildren.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNestedChildren.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNestedChildren.java
new file mode 100644
index 0000000..ca1bfda
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNestedChildren.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.lang.invoke.MethodHandles;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestNestedChildren extends AbstractDIHJdbcTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @Test
+ public void test() throws Exception {
+ h.query("/dataimport", generateRequest());
+ assertQ(req("*:*"), "//*[@numFound='1']");
+ assertQ(req("third_s:CHICKEN"), "//*[@numFound='1']");
+ }
+
+ @Override
+ protected String generateConfig() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("<dataConfig> \n");
+ sb.append("<dataSource name=\"derby\" driver=\"org.apache.derby.jdbc.EmbeddedDriver\" url=\"jdbc:derby:memory:derbyDB;territory=en_US\" /> \n");
+ sb.append("<document name=\"TestSimplePropertiesWriter\"> \n");
+ sb.append("<entity name=\"FIRST\" processor=\"SqlEntityProcessor\" dataSource=\"derby\" ");
+ sb.append(" query=\"select 1 as id, 'PORK' as FIRST_S from sysibm.sysdummy1 \" >\n");
+ sb.append(" <field column=\"FIRST_S\" name=\"first_s\" /> \n");
+ sb.append(" <entity name=\"SECOND\" processor=\"SqlEntityProcessor\" dataSource=\"derby\" ");
+ sb.append(" query=\"select 1 as id, 2 as SECOND_ID, 'BEEF' as SECOND_S from sysibm.sysdummy1 WHERE 1=${FIRST.ID}\" >\n");
+ sb.append(" <field column=\"SECOND_S\" name=\"second_s\" /> \n");
+ sb.append(" <entity name=\"THIRD\" processor=\"SqlEntityProcessor\" dataSource=\"derby\" ");
+ sb.append(" query=\"select 1 as id, 'CHICKEN' as THIRD_S from sysibm.sysdummy1 WHERE 2=${SECOND.SECOND_ID}\" >\n");
+ sb.append(" <field column=\"THIRD_S\" name=\"third_s\" /> \n");
+ sb.append(" </entity>\n");
+ sb.append(" </entity>\n");
+ sb.append("</entity>\n");
+ sb.append("</document> \n");
+ sb.append("</dataConfig> \n");
+ String config = sb.toString();
+ log.debug(config);
+ return config;
+ }
+
+ @Override
+ protected Database setAllowedDatabases() {
+ return Database.DERBY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java
new file mode 100644
index 0000000..1307927
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestNonWritablePersistFile extends AbstractDataImportHandlerTestCase {
+ private static final String FULLIMPORT_QUERY = "select * from x";
+
+ private static final String DELTA_QUERY = "select id from x where last_modified > NOW";
+
+ private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
+
+ private static final String dataConfig_delta =
+ "<dataConfig>" +
+ " <dataSource type=\"MockDataSource\"/>\n" +
+ " <document>\n" +
+ " <entity name=\"x\" transformer=\"TemplateTransformer\"" +
+ " query=\"" + FULLIMPORT_QUERY + "\"" +
+ " deletedPkQuery=\"" + DELETED_PK_QUERY + "\"" +
+ " deltaImportQuery=\"select * from x where id='${dih.delta.id}'\"" +
+ " deltaQuery=\"" + DELTA_QUERY + "\">\n" +
+ " <field column=\"id\" name=\"id\"/>\n" +
+ " <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n" +
+ " <field column=\"desc\" />\n" +
+ " </entity>\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>\n";
+ private static String tmpSolrHome;
+
+ private static File f;
+
+ @BeforeClass
+ public static void createTempSolrHomeAndCore() throws Exception {
+ tmpSolrHome = createTempDir().toFile().getAbsolutePath();
+ FileUtils.copyDirectory(getFile("dih/solr"), new File(tmpSolrHome).getAbsoluteFile());
+ initCore("dataimport-solrconfig.xml", "dataimport-schema.xml",
+ new File(tmpSolrHome).getAbsolutePath());
+
+ // See SOLR-2551
+ String configDir = h.getCore().getResourceLoader().getConfigDir();
+ String filePath = configDir;
+ if (configDir != null && !configDir.endsWith(File.separator))
+ filePath += File.separator;
+ filePath += "dataimport.properties";
+ f = new File(filePath);
+ // execute the test only if we are able to set file to read only mode
+ assumeTrue("No dataimport.properties file", f.exists() || f.createNewFile());
+ assumeTrue("dataimport.properties can't be set read only", f.setReadOnly());
+ assumeFalse("dataimport.properties is still writable even though " +
+ "marked readonly - test running as superuser?", f.canWrite());
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ if (f != null) {
+ f.setWritable(true);
+ }
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testNonWritablePersistFile() throws Exception {
+ ignoreException("Properties is not writable");
+
+ @SuppressWarnings("rawtypes")
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("id", "1"));
+ MockDataSource.setIterator(FULLIMPORT_QUERY, parentRow.iterator());
+
+ @SuppressWarnings("rawtypes")
+ List childRow = new ArrayList();
+ childRow.add(createMap("desc", "hello"));
+ MockDataSource.setIterator("select * from y where y.A='1'",
+ childRow.iterator());
+
+ runFullImport(dataConfig_delta);
+ assertQ(req("id:1"), "//*[@numFound='0']");
+ }
+}
[13/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/UrlEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/UrlEvaluator.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/UrlEvaluator.java
new file mode 100644
index 0000000..8a6654c
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/UrlEvaluator.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+
+import java.net.URLEncoder;
+import java.util.List;
+
+/**
+ * <p>Escapes reserved characters in Solr queries</p>
+ *
+ * @see org.apache.solr.client.solrj.util.ClientUtils#escapeQueryChars(String)
+ */
+public class UrlEvaluator extends Evaluator {
+ @Override
+ public String evaluate(String expression, Context context) {
+ List<Object> l = parseParams(expression, context.getVariableResolver());
+ if (l.size() != 1) {
+ throw new DataImportHandlerException(SEVERE, "'encodeUrl' must have at least one parameter ");
+ }
+ String s = l.get(0).toString();
+
+ try {
+ return URLEncoder.encode(s.toString(), "UTF-8");
+ } catch (Exception e) {
+ wrapAndThrow(SEVERE, e, "Unable to encode expression: " + expression + " with value: " + s);
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/VariableResolver.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/VariableResolver.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/VariableResolver.java
new file mode 100644
index 0000000..090e21b
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/VariableResolver.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.WeakHashMap;
+import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.solr.common.util.Cache;
+import org.apache.solr.common.util.MapBackedCache;
+import org.apache.solr.update.processor.TemplateUpdateProcessorFactory;
+
+import static org.apache.solr.update.processor.TemplateUpdateProcessorFactory.Resolved;
+
+/**
+ * <p>
+ * A set of nested maps that can resolve variables by namespaces. Variables are
+ * enclosed with a dollar sign then an opening curly brace, ending with a
+ * closing curly brace. Namespaces are delimited with '.' (period).
+ * </p>
+ * <p>
+ * This class also has special logic to resolve evaluator calls by recognizing
+ * the reserved function namespace: dataimporter.functions.xxx
+ * </p>
+ * <p>
+ * This class caches strings that have already been resolved from the current
+ * dih import.
+ * </p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ *
+ * @since solr 1.3
+ */
+public class VariableResolver {
+
+ private static final Pattern DOT_PATTERN = Pattern.compile("[.]");
+ private static final Pattern EVALUATOR_FORMAT_PATTERN = Pattern
+ .compile("^(\\w*?)\\((.*?)\\)$");
+ private Map<String,Object> rootNamespace;
+ private Map<String,Evaluator> evaluators;
+ private Cache<String,Resolved> cache = new MapBackedCache<>(new WeakHashMap<>());
+ private Function<String,Object> fun = this::resolve;
+
+ public static final String FUNCTIONS_NAMESPACE = "dataimporter.functions.";
+ public static final String FUNCTIONS_NAMESPACE_SHORT = "dih.functions.";
+
+ public VariableResolver() {
+ rootNamespace = new HashMap<>();
+ }
+
+ public VariableResolver(Properties defaults) {
+ rootNamespace = new HashMap<>();
+ for (Map.Entry<Object,Object> entry : defaults.entrySet()) {
+ rootNamespace.put(entry.getKey().toString(), entry.getValue());
+ }
+ }
+
+ public VariableResolver(Map<String,Object> defaults) {
+ rootNamespace = new HashMap<>(defaults);
+ }
+
+ /**
+ * Resolves a given value with a name
+ *
+ * @param name
+ * the String to be resolved
+ * @return an Object which is the result of evaluation of given name
+ */
+ public Object resolve(String name) {
+ Object r = null;
+ if (name != null) {
+ String[] nameParts = DOT_PATTERN.split(name);
+ CurrentLevel cr = currentLevelMap(nameParts,
+ rootNamespace, false);
+ Map<String,Object> currentLevel = cr.map;
+ r = currentLevel.get(nameParts[nameParts.length - 1]);
+ if (r == null && name.startsWith(FUNCTIONS_NAMESPACE)
+ && name.length() > FUNCTIONS_NAMESPACE.length()) {
+ return resolveEvaluator(FUNCTIONS_NAMESPACE, name);
+ }
+ if (r == null && name.startsWith(FUNCTIONS_NAMESPACE_SHORT)
+ && name.length() > FUNCTIONS_NAMESPACE_SHORT.length()) {
+ return resolveEvaluator(FUNCTIONS_NAMESPACE_SHORT, name);
+ }
+ if (r == null) {
+ StringBuilder sb = new StringBuilder();
+ for(int i=cr.level ; i<nameParts.length ; i++) {
+ if(sb.length()>0) {
+ sb.append(".");
+ }
+ sb.append(nameParts[i]);
+ }
+ r = cr.map.get(sb.toString());
+ }
+ if (r == null) {
+ r = System.getProperty(name);
+ }
+ }
+ return r == null ? "" : r;
+ }
+
+ private Object resolveEvaluator(String namespace, String name) {
+ if (evaluators == null) {
+ return "";
+ }
+ Matcher m = EVALUATOR_FORMAT_PATTERN.matcher(name
+ .substring(namespace.length()));
+ if (m.find()) {
+ String fname = m.group(1);
+ Evaluator evaluator = evaluators.get(fname);
+ if (evaluator == null) return "";
+ ContextImpl ctx = new ContextImpl(null, this, null, null, null, null,
+ null);
+ String g2 = m.group(2);
+ return evaluator.evaluate(g2, ctx);
+ } else {
+ return "";
+ }
+ }
+
+ /**
+ * Given a String with place holders, replace them with the value tokens.
+ *
+ * @return the string with the placeholders replaced with their values
+ */
+ public String replaceTokens(String template) {
+ return TemplateUpdateProcessorFactory.replaceTokens(template, cache, fun, TemplateUpdateProcessorFactory.DOLLAR_BRACES_PLACEHOLDER_PATTERN);
+ }
+ public void addNamespace(String name, Map<String,Object> newMap) {
+ if (newMap != null) {
+ if (name != null) {
+ String[] nameParts = DOT_PATTERN.split(name);
+ Map<String,Object> nameResolveLevel = currentLevelMap(nameParts,
+ rootNamespace, false).map;
+ nameResolveLevel.put(nameParts[nameParts.length - 1], newMap);
+ } else {
+ for (Map.Entry<String,Object> entry : newMap.entrySet()) {
+ String[] keyParts = DOT_PATTERN.split(entry.getKey());
+ Map<String,Object> currentLevel = rootNamespace;
+ currentLevel = currentLevelMap(keyParts, currentLevel, false).map;
+ currentLevel.put(keyParts[keyParts.length - 1], entry.getValue());
+ }
+ }
+ }
+ }
+
+ public List<String> getVariables(String expr) {
+ return TemplateUpdateProcessorFactory.getVariables(expr, cache, TemplateUpdateProcessorFactory.DOLLAR_BRACES_PLACEHOLDER_PATTERN);
+ }
+
+ static class CurrentLevel {
+ final Map<String,Object> map;
+ final int level;
+ CurrentLevel(int level, Map<String,Object> map) {
+ this.level = level;
+ this.map = map;
+ }
+ }
+
+ private CurrentLevel currentLevelMap(String[] keyParts,
+ Map<String,Object> currentLevel, boolean includeLastLevel) {
+ int j = includeLastLevel ? keyParts.length : keyParts.length - 1;
+ for (int i = 0; i < j; i++) {
+ Object o = currentLevel.get(keyParts[i]);
+ if (o == null) {
+ if(i == j-1) {
+ Map<String,Object> nextLevel = new HashMap<>();
+ currentLevel.put(keyParts[i], nextLevel);
+ currentLevel = nextLevel;
+ } else {
+ return new CurrentLevel(i, currentLevel);
+ }
+ } else if (o instanceof Map<?,?>) {
+ @SuppressWarnings("unchecked")
+ Map<String,Object> nextLevel = (Map<String,Object>) o;
+ currentLevel = nextLevel;
+ } else {
+ throw new AssertionError(
+ "Non-leaf nodes should be of type java.util.Map");
+ }
+ }
+ return new CurrentLevel(j-1, currentLevel);
+ }
+
+ public void removeNamespace(String name) {
+ rootNamespace.remove(name);
+ }
+
+ public void setEvaluators(Map<String,Evaluator> evaluators) {
+ this.evaluators = evaluators;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java
new file mode 100644
index 0000000..c93b581
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java
@@ -0,0 +1,548 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+import org.apache.solr.core.SolrCore;
+import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.solr.util.SystemIdResolver;
+import org.apache.solr.common.util.XMLErrorLogger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.commons.io.IOUtils;
+
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+import java.io.CharArrayReader;
+import java.io.CharArrayWriter;
+import java.io.Reader;
+import java.lang.invoke.MethodHandles;
+import java.util.*;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * <p> An implementation of {@link EntityProcessor} which uses a streaming xpath parser to extract values out of XML documents.
+ * It is typically used in conjunction with {@link URLDataSource} or {@link FileDataSource}. </p> <p> Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a> for more
+ * details. </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ *
+ * @see XPathRecordReader
+ * @since solr 1.3
+ */
+public class XPathEntityProcessor extends EntityProcessorBase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static final XMLErrorLogger xmllog = new XMLErrorLogger(log);
+
+ private static final Map<String, Object> END_MARKER = new HashMap<>();
+
+ protected List<String> placeHolderVariables;
+
+ protected List<String> commonFields;
+
+ private String pk;
+
+ private XPathRecordReader xpathReader;
+
+ protected DataSource<Reader> dataSource;
+
+ protected javax.xml.transform.Transformer xslTransformer;
+
+ protected boolean useSolrAddXml = false;
+
+ protected boolean streamRows = false;
+
+ // Amount of time to block reading/writing to queue when streaming
+ protected int blockingQueueTimeOut = 10;
+
+ // Units for pumpTimeOut
+ protected TimeUnit blockingQueueTimeOutUnits = TimeUnit.SECONDS;
+
+ // Number of rows to queue for asynchronous processing
+ protected int blockingQueueSize = 1000;
+
+ protected Thread publisherThread;
+
+ protected boolean reinitXPathReader = true;
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void init(Context context) {
+ super.init(context);
+ if (reinitXPathReader)
+ initXpathReader(context.getVariableResolver());
+ pk = context.getEntityAttribute("pk");
+ dataSource = context.getDataSource();
+ rowIterator = null;
+
+ }
+
+ private void initXpathReader(VariableResolver resolver) {
+ reinitXPathReader = false;
+ useSolrAddXml = Boolean.parseBoolean(context
+ .getEntityAttribute(USE_SOLR_ADD_SCHEMA));
+ streamRows = Boolean.parseBoolean(context
+ .getEntityAttribute(STREAM));
+ if (context.getResolvedEntityAttribute("batchSize") != null) {
+ blockingQueueSize = Integer.parseInt(context.getEntityAttribute("batchSize"));
+ }
+ if (context.getResolvedEntityAttribute("readTimeOut") != null) {
+ blockingQueueTimeOut = Integer.parseInt(context.getEntityAttribute("readTimeOut"));
+ }
+ String xslt = context.getEntityAttribute(XSL);
+ if (xslt != null) {
+ xslt = context.replaceTokens(xslt);
+ try {
+ // create an instance of TransformerFactory
+ TransformerFactory transFact = TransformerFactory.newInstance();
+ final SolrCore core = context.getSolrCore();
+ final StreamSource xsltSource;
+ if (core != null) {
+ final ResourceLoader loader = core.getResourceLoader();
+ transFact.setURIResolver(new SystemIdResolver(loader).asURIResolver());
+ xsltSource = new StreamSource(loader.openResource(xslt),
+ SystemIdResolver.createSystemIdFromResourceName(xslt));
+ } else {
+ // fallback for tests
+ xsltSource = new StreamSource(xslt);
+ }
+ transFact.setErrorListener(xmllog);
+ try {
+ xslTransformer = transFact.newTransformer(xsltSource);
+ } finally {
+ // some XML parsers are broken and don't close the byte stream (but they should according to spec)
+ IOUtils.closeQuietly(xsltSource.getInputStream());
+ }
+ log.info("Using xslTransformer: "
+ + xslTransformer.getClass().getName());
+ } catch (Exception e) {
+ throw new DataImportHandlerException(SEVERE,
+ "Error initializing XSL ", e);
+ }
+ }
+
+ if (useSolrAddXml) {
+ // Support solr add documents
+ xpathReader = new XPathRecordReader("/add/doc");
+ xpathReader.addField("name", "/add/doc/field/@name", true);
+ xpathReader.addField("value", "/add/doc/field", true);
+ } else {
+ String forEachXpath = context.getResolvedEntityAttribute(FOR_EACH);
+ if (forEachXpath == null)
+ throw new DataImportHandlerException(SEVERE,
+ "Entity : " + context.getEntityAttribute("name")
+ + " must have a 'forEach' attribute");
+ if (forEachXpath.equals(context.getEntityAttribute(FOR_EACH))) reinitXPathReader = true;
+
+ try {
+ xpathReader = new XPathRecordReader(forEachXpath);
+ for (Map<String, String> field : context.getAllEntityFields()) {
+ if (field.get(XPATH) == null)
+ continue;
+ int flags = 0;
+ if ("true".equals(field.get("flatten"))) {
+ flags = XPathRecordReader.FLATTEN;
+ }
+ String xpath = field.get(XPATH);
+ xpath = context.replaceTokens(xpath);
+ //!xpath.equals(field.get(XPATH) means the field xpath has a template
+ //in that case ensure that the XPathRecordReader is reinitialized
+ //for each xml
+ if (!xpath.equals(field.get(XPATH)) && !context.isRootEntity()) reinitXPathReader = true;
+ xpathReader.addField(field.get(DataImporter.COLUMN),
+ xpath,
+ Boolean.parseBoolean(field.get(DataImporter.MULTI_VALUED)),
+ flags);
+ }
+ } catch (RuntimeException e) {
+ throw new DataImportHandlerException(SEVERE,
+ "Exception while reading xpaths for fields", e);
+ }
+ }
+ String url = context.getEntityAttribute(URL);
+ List<String> l = url == null ? Collections.EMPTY_LIST : resolver.getVariables(url);
+ for (String s : l) {
+ if (s.startsWith(entityName + ".")) {
+ if (placeHolderVariables == null)
+ placeHolderVariables = new ArrayList<>();
+ placeHolderVariables.add(s.substring(entityName.length() + 1));
+ }
+ }
+ for (Map<String, String> fld : context.getAllEntityFields()) {
+ if (fld.get(COMMON_FIELD) != null && "true".equals(fld.get(COMMON_FIELD))) {
+ if (commonFields == null)
+ commonFields = new ArrayList<>();
+ commonFields.add(fld.get(DataImporter.COLUMN));
+ }
+ }
+
+ }
+
+ @Override
+ public Map<String, Object> nextRow() {
+ Map<String, Object> result;
+
+ if (!context.isRootEntity())
+ return fetchNextRow();
+
+ while (true) {
+ result = fetchNextRow();
+
+ if (result == null)
+ return null;
+
+ if (pk == null || result.get(pk) != null)
+ return result;
+ }
+ }
+
+ @Override
+ public void postTransform(Map<String, Object> r) {
+ readUsefulVars(r);
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map<String, Object> fetchNextRow() {
+ Map<String, Object> r = null;
+ while (true) {
+ if (rowIterator == null)
+ initQuery(context.replaceTokens(context.getEntityAttribute(URL)));
+ r = getNext();
+ if (r == null) {
+ Object hasMore = context.getSessionAttribute(HAS_MORE, Context.SCOPE_ENTITY);
+ try {
+ if ("true".equals(hasMore) || Boolean.TRUE.equals(hasMore)) {
+ String url = (String) context.getSessionAttribute(NEXT_URL, Context.SCOPE_ENTITY);
+ if (url == null)
+ url = context.getEntityAttribute(URL);
+ addNamespace();
+ initQuery(context.replaceTokens(url));
+ r = getNext();
+ if (r == null)
+ return null;
+ } else {
+ return null;
+ }
+ } finally {
+ context.setSessionAttribute(HAS_MORE,null,Context.SCOPE_ENTITY);
+ context.setSessionAttribute(NEXT_URL,null,Context.SCOPE_ENTITY);
+ }
+ }
+ addCommonFields(r);
+ return r;
+ }
+ }
+
+ private void addNamespace() {
+ Map<String, Object> namespace = new HashMap<>();
+ Set<String> allNames = new HashSet<>();
+ if (commonFields != null) allNames.addAll(commonFields);
+ if (placeHolderVariables != null) allNames.addAll(placeHolderVariables);
+ if(allNames.isEmpty()) return;
+
+ for (String name : allNames) {
+ Object val = context.getSessionAttribute(name, Context.SCOPE_ENTITY);
+ if (val != null) namespace.put(name, val);
+ }
+ ((VariableResolver)context.getVariableResolver()).addNamespace(entityName, namespace);
+ }
+
+ private void addCommonFields(Map<String, Object> r) {
+ if(commonFields != null){
+ for (String commonField : commonFields) {
+ if(r.get(commonField) == null) {
+ Object val = context.getSessionAttribute(commonField, Context.SCOPE_ENTITY);
+ if(val != null) r.put(commonField, val);
+ }
+
+ }
+ }
+
+ }
+
+ private void initQuery(String s) {
+ Reader data = null;
+ try {
+ final List<Map<String, Object>> rows = new ArrayList<>();
+ try {
+ data = dataSource.getData(s);
+ } catch (Exception e) {
+ if (ABORT.equals(onError)) {
+ wrapAndThrow(SEVERE, e);
+ } else if (SKIP.equals(onError)) {
+ if (log.isDebugEnabled()) log.debug("Skipping url : " + s, e);
+ wrapAndThrow(DataImportHandlerException.SKIP, e);
+ } else {
+ log.warn("Failed for url : " + s, e);
+ rowIterator = Collections.EMPTY_LIST.iterator();
+ return;
+ }
+ }
+ if (xslTransformer != null) {
+ try {
+ SimpleCharArrayReader caw = new SimpleCharArrayReader();
+ xslTransformer.transform(new StreamSource(data),
+ new StreamResult(caw));
+ data = caw.getReader();
+ } catch (TransformerException e) {
+ if (ABORT.equals(onError)) {
+ wrapAndThrow(SEVERE, e, "Exception in applying XSL Transformation");
+ } else if (SKIP.equals(onError)) {
+ wrapAndThrow(DataImportHandlerException.SKIP, e);
+ } else {
+ log.warn("Failed for url : " + s, e);
+ rowIterator = Collections.EMPTY_LIST.iterator();
+ return;
+ }
+ }
+ }
+ if (streamRows) {
+ rowIterator = getRowIterator(data, s);
+ } else {
+ try {
+ xpathReader.streamRecords(data, (record, xpath) -> rows.add(readRow(record, xpath)));
+ } catch (Exception e) {
+ String msg = "Parsing failed for xml, url:" + s + " rows processed:" + rows.size();
+ if (rows.size() > 0) msg += " last row: " + rows.get(rows.size() - 1);
+ if (ABORT.equals(onError)) {
+ wrapAndThrow(SEVERE, e, msg);
+ } else if (SKIP.equals(onError)) {
+ log.warn(msg, e);
+ Map<String, Object> map = new HashMap<>();
+ map.put(DocBuilder.SKIP_DOC, Boolean.TRUE);
+ rows.add(map);
+ } else if (CONTINUE.equals(onError)) {
+ log.warn(msg, e);
+ }
+ }
+ rowIterator = rows.iterator();
+ }
+ } finally {
+ if (!streamRows) {
+ closeIt(data);
+ }
+
+ }
+ }
+
+ private void closeIt(Reader data) {
+ try {
+ data.close();
+ } catch (Exception e) { /* Ignore */
+ }
+ }
+
+ protected Map<String, Object> readRow(Map<String, Object> record, String xpath) {
+ if (useSolrAddXml) {
+ List<String> names = (List<String>) record.get("name");
+ List<String> values = (List<String>) record.get("value");
+ Map<String, Object> row = new HashMap<>();
+ for (int i = 0; i < names.size() && i < values.size(); i++) {
+ if (row.containsKey(names.get(i))) {
+ Object existing = row.get(names.get(i));
+ if (existing instanceof List) {
+ List list = (List) existing;
+ list.add(values.get(i));
+ } else {
+ List list = new ArrayList();
+ list.add(existing);
+ list.add(values.get(i));
+ row.put(names.get(i), list);
+ }
+ } else {
+ row.put(names.get(i), values.get(i));
+ }
+ }
+ return row;
+ } else {
+ record.put(XPATH_FIELD_NAME, xpath);
+ return record;
+ }
+ }
+
+
+ private static class SimpleCharArrayReader extends CharArrayWriter {
+ public Reader getReader() {
+ return new CharArrayReader(super.buf, 0, super.count);
+ }
+
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map<String, Object> readUsefulVars(Map<String, Object> r) {
+ Object val = r.get(HAS_MORE);
+ if (val != null)
+ context.setSessionAttribute(HAS_MORE, val,Context.SCOPE_ENTITY);
+ val = r.get(NEXT_URL);
+ if (val != null)
+ context.setSessionAttribute(NEXT_URL, val,Context.SCOPE_ENTITY);
+ if (placeHolderVariables != null) {
+ for (String s : placeHolderVariables) {
+ val = r.get(s);
+ context.setSessionAttribute(s, val,Context.SCOPE_ENTITY);
+ }
+ }
+ if (commonFields != null) {
+ for (String s : commonFields) {
+ Object commonVal = r.get(s);
+ if (commonVal != null) {
+ context.setSessionAttribute(s, commonVal,Context.SCOPE_ENTITY);
+ }
+ }
+ }
+ return r;
+
+ }
+
+ private Iterator<Map<String, Object>> getRowIterator(final Reader data, final String s) {
+ //nothing atomic about it. I just needed a StongReference
+ final AtomicReference<Exception> exp = new AtomicReference<>();
+ final BlockingQueue<Map<String, Object>> blockingQueue = new ArrayBlockingQueue<>(blockingQueueSize);
+ final AtomicBoolean isEnd = new AtomicBoolean(false);
+ final AtomicBoolean throwExp = new AtomicBoolean(true);
+ publisherThread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ xpathReader.streamRecords(data, (record, xpath) -> {
+ if (isEnd.get()) {
+ throwExp.set(false);
+ //To end the streaming . otherwise the parsing will go on forever
+ //though consumer has gone away
+ throw new RuntimeException("BREAK");
+ }
+ Map<String, Object> row;
+ try {
+ row = readRow(record, xpath);
+ } catch (Exception e) {
+ isEnd.set(true);
+ return;
+ }
+ offer(row);
+ });
+ } catch (Exception e) {
+ if(throwExp.get()) exp.set(e);
+ } finally {
+ closeIt(data);
+ if (!isEnd.get()) {
+ offer(END_MARKER);
+ }
+ }
+ }
+
+ private void offer(Map<String, Object> row) {
+ try {
+ while (!blockingQueue.offer(row, blockingQueueTimeOut, blockingQueueTimeOutUnits)) {
+ if (isEnd.get()) return;
+ log.debug("Timeout elapsed writing records. Perhaps buffer size should be increased.");
+ }
+ } catch (InterruptedException e) {
+ return;
+ } finally {
+ synchronized (this) {
+ notifyAll();
+ }
+ }
+ }
+ };
+
+ publisherThread.start();
+
+ return new Iterator<Map<String, Object>>() {
+ private Map<String, Object> lastRow;
+ int count = 0;
+
+ @Override
+ public boolean hasNext() {
+ return !isEnd.get();
+ }
+
+ @Override
+ public Map<String, Object> next() {
+ Map<String, Object> row;
+
+ do {
+ try {
+ row = blockingQueue.poll(blockingQueueTimeOut, blockingQueueTimeOutUnits);
+ if (row == null) {
+ log.debug("Timeout elapsed reading records.");
+ }
+ } catch (InterruptedException e) {
+ log.debug("Caught InterruptedException while waiting for row. Aborting.");
+ isEnd.set(true);
+ return null;
+ }
+ } while (row == null);
+
+ if (row == END_MARKER) {
+ isEnd.set(true);
+ if (exp.get() != null) {
+ String msg = "Parsing failed for xml, url:" + s + " rows processed in this xml:" + count;
+ if (lastRow != null) msg += " last row in this xml:" + lastRow;
+ if (ABORT.equals(onError)) {
+ wrapAndThrow(SEVERE, exp.get(), msg);
+ } else if (SKIP.equals(onError)) {
+ wrapAndThrow(DataImportHandlerException.SKIP, exp.get());
+ } else {
+ log.warn(msg, exp.get());
+ }
+ }
+ return null;
+ }
+ count++;
+ return lastRow = row;
+ }
+
+ @Override
+ public void remove() {
+ /*no op*/
+ }
+ };
+
+ }
+
+
+ public static final String URL = "url";
+
+ public static final String HAS_MORE = "$hasMore";
+
+ public static final String NEXT_URL = "$nextUrl";
+
+ public static final String XPATH_FIELD_NAME = "$forEach";
+
+ public static final String FOR_EACH = "forEach";
+
+ public static final String XPATH = "xpath";
+
+ public static final String COMMON_FIELD = "commonField";
+
+ public static final String USE_SOLR_ADD_SCHEMA = "useSolrAddSchema";
+
+ public static final String XSL = "xsl";
+
+ public static final String STREAM = "stream";
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java
new file mode 100644
index 0000000..13daf49
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java
@@ -0,0 +1,669 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.common.util.XMLErrorLogger;
+import org.apache.solr.common.EmptyEntityResolver;
+import javax.xml.stream.XMLInputFactory;
+import static javax.xml.stream.XMLStreamConstants.*;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.lang.invoke.MethodHandles;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>
+ * A streaming xpath parser which uses StAX for XML parsing. It supports only
+ * a subset of xpath syntax.
+ * </p><pre>
+ * /a/b/subject[@qualifier='fullTitle']
+ * /a/b/subject[@qualifier=]/subtag
+ * /a/b/subject/@qualifier
+ * //a
+ * //a/b...
+ * /a//b
+ * /a//b...
+ * /a/b/c
+ * </pre>
+ * A record is a Map<String,Object> . The key is the provided name
+ * and the value is a String or a List<String>
+ *
+ * This class is thread-safe for parsing xml. But adding fields is not
+ * thread-safe. The recommended usage is to addField() in one thread and
+ * then share the instance across threads.
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 1.3
+ */
+public class XPathRecordReader {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static final XMLErrorLogger XMLLOG = new XMLErrorLogger(log);
+
+ private Node rootNode = new Node("/", null);
+
+ /**
+ * The FLATTEN flag indicates that all text and cdata under a specific
+ * tag should be recursivly fetched and appended to the current Node's
+ * value.
+ */
+ public static final int FLATTEN = 1;
+
+ /**
+ * A constructor called with a '|' separated list of Xpath expressions
+ * which define sub sections of the XML stream that are to be emitted as
+ * separate records.
+ *
+ * @param forEachXpath The XPATH for which a record is emitted. Once the
+ * xpath tag is encountered, the Node.parse method starts collecting wanted
+ * fields and at the close of the tag, a record is emitted containing all
+ * fields collected since the tag start. Once
+ * emitted the collected fields are cleared. Any fields collected in the
+ * parent tag or above will also be included in the record, but these are
+ * not cleared after emitting the record.
+ *
+ * It uses the ' | ' syntax of XPATH to pass in multiple xpaths.
+ */
+ public XPathRecordReader(String forEachXpath) {
+ String[] splits = forEachXpath.split("\\|");
+ for (String split : splits) {
+ split = split.trim();
+ if (split.startsWith("//"))
+ throw new RuntimeException("forEach cannot start with '//': " + split);
+ if (split.length() == 0)
+ continue;
+ // The created Node has a name set to the full forEach attribute xpath
+ addField0(split, split, false, true, 0);
+ }
+ }
+
+ /**
+ * A wrapper around <code>addField0</code> to create a series of
+ * Nodes based on the supplied Xpath and a given fieldName. The created
+ * nodes are inserted into a Node tree.
+ *
+ * @param name The name for this field in the emitted record
+ * @param xpath The xpath expression for this field
+ * @param multiValued If 'true' then the emitted record will have values in
+ * a List<String>
+ */
+ public synchronized XPathRecordReader addField(String name, String xpath, boolean multiValued) {
+ addField0(xpath, name, multiValued, false, 0);
+ return this;
+ }
+
+ /**
+ * A wrapper around <code>addField0</code> to create a series of
+ * Nodes based on the supplied Xpath and a given fieldName. The created
+ * nodes are inserted into a Node tree.
+ *
+ * @param name The name for this field in the emitted record
+ * @param xpath The xpath expression for this field
+ * @param multiValued If 'true' then the emitted record will have values in
+ * a List<String>
+ * @param flags FLATTEN: Recursively combine text from all child XML elements
+ */
+ public synchronized XPathRecordReader addField(String name, String xpath, boolean multiValued, int flags) {
+ addField0(xpath, name, multiValued, false, flags);
+ return this;
+ }
+
+ /**
+ * Splits the XPATH into a List of xpath segments and calls build() to
+ * construct a tree of Nodes representing xpath segments. The resulting
+ * tree structure ends up describing all the Xpaths we are interested in.
+ *
+ * @param xpath The xpath expression for this field
+ * @param name The name for this field in the emitted record
+ * @param multiValued If 'true' then the emitted record will have values in
+ * a List<String>
+ * @param isRecord Flags that this XPATH is from a forEach statement
+ * @param flags The only supported flag is 'FLATTEN'
+ */
+ private void addField0(String xpath, String name, boolean multiValued,
+ boolean isRecord, int flags) {
+ if (!xpath.startsWith("/"))
+ throw new RuntimeException("xpath must start with '/' : " + xpath);
+ List<String> paths = splitEscapeQuote(xpath);
+ // deal with how split behaves when separator starts a string!
+ if ("".equals(paths.get(0).trim()))
+ paths.remove(0);
+ rootNode.build(paths, name, multiValued, isRecord, flags);
+ rootNode.buildOptimise(null);
+ }
+
+ /**
+ * Uses {@link #streamRecords streamRecords} to parse the XML source but with
+ * a handler that collects all the emitted records into a single List which
+ * is returned upon completion.
+ *
+ * @param r the stream reader
+ * @return results a List of emitted records
+ */
+ public List<Map<String, Object>> getAllRecords(Reader r) {
+ final List<Map<String, Object>> results = new ArrayList<>();
+ streamRecords(r, (record, s) -> results.add(record));
+ return results;
+ }
+
+ /**
+ * Creates an XML stream reader on top of whatever reader has been
+ * configured. Then calls parse() with a handler which is
+ * invoked forEach record emitted.
+ *
+ * @param r the stream reader
+ * @param handler The callback instance
+ */
+ public void streamRecords(Reader r, Handler handler) {
+ try {
+ XMLStreamReader parser = factory.createXMLStreamReader(r);
+ rootNode.parse(parser, handler, new HashMap<>(),
+ new Stack<>(), false);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+
+ /**
+ * For each node/leaf in the Node tree there is one object of this class.
+ * This tree of objects represents all the XPaths we are interested in.
+ * For each Xpath segment of interest we create a node. In most cases the
+ * node (branch) is rather basic , but for the final portion (leaf) of any
+ * Xpath we add more information to the Node. When parsing the XML document
+ * we step though this tree as we stream records from the reader. If the XML
+ * document departs from this tree we skip start tags till we are back on
+ * the tree.
+ */
+ private static class Node {
+ String name; // genrally: segment of the Xpath represented by this Node
+ String fieldName; // the fieldname in the emitted record (key of the map)
+ String xpathName; // the segment of the Xpath represented by this Node
+ String forEachPath; // the full Xpath from the forEach entity attribute
+ List<Node> attributes; // List of attribute Nodes associated with this Node
+ List<Node> childNodes; // List of immediate child Nodes of this node
+ List<Node> wildCardNodes; // List of '//' style decendants of this Node
+ List<Map.Entry<String, String>> attribAndValues;
+ Node wildAncestor; // ancestor Node containing '//' style decendants
+ Node parent; // parent Node in the tree
+ boolean hasText=false; // flag: store/emit streamed text for this node
+ boolean multiValued=false; //flag: this fields values are returned as a List
+ boolean isRecord=false; //flag: this Node starts a new record
+ private boolean flatten; //flag: child text is also to be emitted
+
+
+ public Node(String name, Node p) {
+ // Create a basic Node, suitable for the mid portions of any Xpath.
+ // Node.xpathName and Node.name are set to same value.
+ xpathName = this.name = name;
+ parent = p;
+ }
+
+ public Node(String name, String fieldName, boolean multiValued) {
+ // This is only called from build() when describing an attribute.
+ this.name = name; // a segment from the Xpath
+ this.fieldName = fieldName; // name to store collected values against
+ this.multiValued = multiValued; // return collected values in a List
+ }
+
+ /**
+ * This is the method where all the XML parsing happens. For each
+ * tag/subtag read from the source, this method is called recursively.
+ *
+ */
+ private void parse(XMLStreamReader parser,
+ Handler handler,
+ Map<String, Object> values,
+ Stack<Set<String>> stack, // lists of values to purge
+ boolean recordStarted
+ ) throws IOException, XMLStreamException {
+ Set<String> valuesAddedinThisFrame = null;
+ if (isRecord) {
+ // This Node is a match for an XPATH from a forEach attribute,
+ // prepare for the clean up that will occurr when the record
+ // is emitted after its END_ELEMENT is matched
+ recordStarted = true;
+ valuesAddedinThisFrame = new HashSet<>();
+ stack.push(valuesAddedinThisFrame);
+ } else if (recordStarted) {
+ // This node is a child of some parent which matched against forEach
+ // attribute. Continue to add values to an existing record.
+ valuesAddedinThisFrame = stack.peek();
+ }
+
+ try {
+ /* The input stream has deposited us at this Node in our tree of
+ * intresting nodes. Depending on how this node is of interest,
+ * process further tokens from the input stream and decide what
+ * we do next
+ */
+ if (attributes != null) {
+ // we interested in storing attributes from the input stream
+ for (Node node : attributes) {
+ String value = parser.getAttributeValue(null, node.name);
+ if (value != null || (recordStarted && !isRecord)) {
+ putText(values, value, node.fieldName, node.multiValued);
+ valuesAddedinThisFrame.add(node.fieldName);
+ }
+ }
+ }
+
+ Set<Node> childrenFound = new HashSet<>();
+ int event = -1;
+ int flattenedStarts=0; // our tag depth when flattening elements
+ StringBuilder text = new StringBuilder();
+
+ while (true) {
+ event = parser.next();
+
+ if (event == END_ELEMENT) {
+ if (flattenedStarts > 0) flattenedStarts--;
+ else {
+ if (hasText && valuesAddedinThisFrame != null) {
+ valuesAddedinThisFrame.add(fieldName);
+ putText(values, text.toString(), fieldName, multiValued);
+ }
+ if (isRecord) handler.handle(getDeepCopy(values), forEachPath);
+ if (childNodes != null && recordStarted && !isRecord && !childrenFound.containsAll(childNodes)) {
+ // nonReccord nodes where we have not collected text for ALL
+ // the child nodes.
+ for (Node n : childNodes) {
+ // For the multivalue child nodes where we could have, but
+ // didnt, collect text. Push a null string into values.
+ if (!childrenFound.contains(n)) n.putNulls(values, valuesAddedinThisFrame);
+ }
+ }
+ return;
+ }
+ }
+ else if (hasText && (event==CDATA || event==CHARACTERS || event==SPACE)) {
+ text.append(parser.getText());
+ }
+ else if (event == START_ELEMENT) {
+ if ( flatten )
+ flattenedStarts++;
+ else
+ handleStartElement(parser, childrenFound, handler, values, stack, recordStarted);
+ }
+ // END_DOCUMENT is least likely to appear and should be
+ // last in if-then-else skip chain
+ else if (event == END_DOCUMENT) return;
+ }
+ }finally {
+ if ((isRecord || !recordStarted) && !stack.empty()) {
+ Set<String> cleanThis = stack.pop();
+ if (cleanThis != null) {
+ for (String fld : cleanThis) values.remove(fld);
+ }
+ }
+ }
+ }
+
+ /**
+ * If a new tag is encountered, check if it is of interest or not by seeing
+ * if it matches against our node tree. If we have deperted from the node
+ * tree then walk back though the tree's ancestor nodes checking to see if
+ * any // expressions exist for the node and compare them against the new
+ * tag. If matched then "jump" to that node, otherwise ignore the tag.
+ *
+ * Note, the list of // expressions found while walking back up the tree
+ * is chached in the HashMap decends. Then if the new tag is to be skipped,
+ * any inner chil tags are compared against the cache and jumped to if
+ * matched.
+ */
+ private void handleStartElement(XMLStreamReader parser, Set<Node> childrenFound,
+ Handler handler, Map<String, Object> values,
+ Stack<Set<String>> stack, boolean recordStarted)
+ throws IOException, XMLStreamException {
+ Node n = getMatchingNode(parser,childNodes);
+ Map<String, Object> decends=new HashMap<>();
+ if (n != null) {
+ childrenFound.add(n);
+ n.parse(parser, handler, values, stack, recordStarted);
+ return;
+ }
+ // The stream has diverged from the tree of interesting elements, but
+ // are there any wildCardNodes ... anywhere in our path from the root?
+ Node dn = this; // checking our Node first!
+
+ do {
+ if (dn.wildCardNodes != null) {
+ // Check to see if the streams tag matches one of the "//" all
+ // decendents type expressions for this node.
+ n = getMatchingNode(parser, dn.wildCardNodes);
+ if (n != null) {
+ childrenFound.add(n);
+ n.parse(parser, handler, values, stack, recordStarted);
+ break;
+ }
+ // add the list of this nodes wild decendents to the cache
+ for (Node nn : dn.wildCardNodes) decends.put(nn.name, nn);
+ }
+ dn = dn.wildAncestor; // leap back along the tree toward root
+ } while (dn != null) ;
+
+ if (n == null) {
+ // we have a START_ELEMENT which is not within the tree of
+ // interesting nodes. Skip over the contents of this element
+ // but recursivly repeat the above for any START_ELEMENTs
+ // found within this element.
+ int count = 1; // we have had our first START_ELEMENT
+ while (count != 0) {
+ int token = parser.next();
+ if (token == START_ELEMENT) {
+ Node nn = (Node) decends.get(parser.getLocalName());
+ if (nn != null) {
+ // We have a //Node which matches the stream's parser.localName
+ childrenFound.add(nn);
+ // Parse the contents of this stream element
+ nn.parse(parser, handler, values, stack, recordStarted);
+ }
+ else count++;
+ }
+ else if (token == END_ELEMENT) count--;
+ }
+ }
+ }
+
+
+ /**
+ * Check if the current tag is to be parsed or not. We step through the
+ * supplied List "searchList" looking for a match. If matched, return the
+ * Node object.
+ */
+ private Node getMatchingNode(XMLStreamReader parser,List<Node> searchL){
+ if (searchL == null)
+ return null;
+ String localName = parser.getLocalName();
+ for (Node n : searchL) {
+ if (n.name.equals(localName)) {
+ if (n.attribAndValues == null)
+ return n;
+ if (checkForAttributes(parser, n.attribAndValues))
+ return n;
+ }
+ }
+ return null;
+ }
+
+ private boolean checkForAttributes(XMLStreamReader parser,
+ List<Map.Entry<String, String>> attrs) {
+ for (Map.Entry<String, String> e : attrs) {
+ String val = parser.getAttributeValue(null, e.getKey());
+ if (val == null)
+ return false;
+ if (e.getValue() != null && !e.getValue().equals(val))
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * A recursive routine that walks the Node tree from a supplied start
+ * pushing a null string onto every multiValued fieldName's List of values
+ * where a value has not been provided from the stream.
+ */
+ private void putNulls(Map<String, Object> values, Set<String> valuesAddedinThisFrame) {
+ if (attributes != null) {
+ for (Node n : attributes) {
+ if (n.multiValued) {
+ putANull(n.fieldName, values, valuesAddedinThisFrame);
+ }
+ }
+ }
+ if (hasText && multiValued) {
+ putANull(fieldName, values, valuesAddedinThisFrame);
+ }
+ if (childNodes != null) {
+ for (Node childNode : childNodes) {
+ childNode.putNulls(values, valuesAddedinThisFrame);
+ }
+ }
+ }
+
+ private void putANull(String thisFieldName, Map<String, Object> values, Set<String> valuesAddedinThisFrame) {
+ putText(values, null, thisFieldName, true);
+ if( valuesAddedinThisFrame != null) {
+ valuesAddedinThisFrame.add(thisFieldName);
+ }
+ }
+
+ /**
+ * Add the field name and text into the values Map. If it is a non
+ * multivalued field, then the text is simply placed in the object
+ * portion of the Map. If it is a multivalued field then the text is
+ * pushed onto a List which is the object portion of the Map.
+ */
+ @SuppressWarnings("unchecked")
+ private void putText(Map<String, Object> values, String value,
+ String fieldName, boolean multiValued) {
+ if (multiValued) {
+ List<String> v = (List<String>) values.get(fieldName);
+ if (v == null) {
+ v = new ArrayList<>();
+ values.put(fieldName, v);
+ }
+ v.add(value);
+ } else {
+ values.put(fieldName, value);
+ }
+ }
+
+
+ /**
+ * Walk the Node tree propagating any wildDescentant information to
+ * child nodes. This allows us to optimise the performance of the
+ * main parse method.
+ */
+ private void buildOptimise(Node wa) {
+ wildAncestor=wa;
+ if ( wildCardNodes != null ) wa = this;
+ if ( childNodes != null )
+ for ( Node n : childNodes ) n.buildOptimise(wa);
+ }
+
+ /**
+ * Build a Node tree structure representing all Xpaths of intrest to us.
+ * This must be done before parsing of the XML stream starts. Each node
+ * holds one portion of an Xpath. Taking each Xpath segment in turn this
+ * method walks the Node tree and finds where the new segment should be
+ * inserted. It creates a Node representing a field's name, XPATH and
+ * some flags and inserts the Node into the Node tree.
+ */
+ private void build(
+ List<String> paths, // a List of segments from the split xpaths
+ String fieldName, // the fieldName assoc with this Xpath
+ boolean multiValued, // flag if this fieldName is multiValued or not
+ boolean record, // is this xpath a record or a field
+ int flags // are we to flatten matching xpaths
+ ) {
+ // recursivly walk the paths Lists adding new Nodes as required
+ String xpseg = paths.remove(0); // shift out next Xpath segment
+
+ if (paths.isEmpty() && xpseg.startsWith("@")) {
+ // we have reached end of element portion of Xpath and can now only
+ // have an element attribute. Add it to this nodes list of attributes
+ if (attributes == null) {
+ attributes = new ArrayList<>();
+ }
+ xpseg = xpseg.substring(1); // strip the '@'
+ attributes.add(new Node(xpseg, fieldName, multiValued));
+ }
+ else if ( xpseg.length() == 0) {
+ // we have a '//' selector for all decendents of the current nodes
+ xpseg = paths.remove(0); // shift out next Xpath segment
+ if (wildCardNodes == null) wildCardNodes = new ArrayList<>();
+ Node n = getOrAddNode(xpseg, wildCardNodes);
+ if (paths.isEmpty()) {
+ // We are current a leaf node.
+ // xpath with content we want to store and return
+ n.hasText = true; // we have to store text found here
+ n.fieldName = fieldName; // name to store collected text against
+ n.multiValued = multiValued; // true: text be stored in a List
+ n.flatten = flags == FLATTEN; // true: store text from child tags
+ }
+ else {
+ // recurse to handle next paths segment
+ n.build(paths, fieldName, multiValued, record, flags);
+ }
+ }
+ else {
+ if (childNodes == null)
+ childNodes = new ArrayList<>();
+ // does this "name" already exist as a child node.
+ Node n = getOrAddNode(xpseg,childNodes);
+ if (paths.isEmpty()) {
+ // We have emptied paths, we are for the moment a leaf of the tree.
+ // When parsing the actual input we have traversed to a position
+ // where we actutally have to do something. getOrAddNode() will
+ // have created and returned a new minimal Node with name and
+ // xpathName already populated. We need to add more information.
+ if (record) {
+ // forEach attribute
+ n.isRecord = true; // flag: forEach attribute, prepare to emit rec
+ n.forEachPath = fieldName; // the full forEach attribute xpath
+ } else {
+ // xpath with content we want to store and return
+ n.hasText = true; // we have to store text found here
+ n.fieldName = fieldName; // name to store collected text against
+ n.multiValued = multiValued; // true: text be stored in a List
+ n.flatten = flags == FLATTEN; // true: store text from child tags
+ }
+ } else {
+ // recurse to handle next paths segment
+ n.build(paths, fieldName, multiValued, record, flags);
+ }
+ }
+ }
+
+ private Node getOrAddNode(String xpathName, List<Node> searchList ) {
+ for (Node n : searchList)
+ if (n.xpathName.equals(xpathName)) return n;
+ // new territory! add a new node for this Xpath bitty
+ Node n = new Node(xpathName, this); // a minimal Node initialization
+ Matcher m = ATTRIB_PRESENT_WITHVAL.matcher(xpathName);
+ if (m.find()) {
+ n.name = m.group(1);
+ int start = m.start(2);
+ while (true) {
+ HashMap<String, String> attribs = new HashMap<>();
+ if (!m.find(start))
+ break;
+ attribs.put(m.group(3), m.group(5));
+ start = m.end(6);
+ if (n.attribAndValues == null)
+ n.attribAndValues = new ArrayList<>();
+ n.attribAndValues.addAll(attribs.entrySet());
+ }
+ }
+ searchList.add(n);
+ return n;
+ }
+
+ /**
+ * Copies a supplied Map to a new Map which is returned. Used to copy a
+ * records values. If a fields value is a List then they have to be
+ * deep-copied for thread safety
+ */
+ private static Map<String, Object> getDeepCopy(Map<String, Object> values) {
+ Map<String, Object> result = new HashMap<>();
+ for (Map.Entry<String, Object> entry : values.entrySet()) {
+ if (entry.getValue() instanceof List) {
+ result.put(entry.getKey(), new ArrayList((List) entry.getValue()));
+ } else {
+ result.put(entry.getKey(), entry.getValue());
+ }
+ }
+ return result;
+ }
+ } // end of class Node
+
+
+ /**
+ * The Xpath is split into segments using the '/' as a separator. However
+ * this method deals with special cases where there is a slash '/' character
+ * inside the attribute value e.g. x/@html='text/html'. We split by '/' but
+ * then reassemble things were the '/' appears within a quoted sub-string.
+ *
+ * We have already enforced that the string must begin with a separator. This
+ * method depends heavily on how split behaves if the string starts with the
+ * separator or if a sequence of multiple separator's appear.
+ */
+ private static List<String> splitEscapeQuote(String str) {
+ List<String> result = new LinkedList<>();
+ String[] ss = str.split("/");
+ for (int i=0; i<ss.length; i++) { // i=1: skip separator at start of string
+ StringBuilder sb = new StringBuilder();
+ int quoteCount = 0;
+ while (true) {
+ sb.append(ss[i]);
+ for (int j=0; j<ss[i].length(); j++)
+ if (ss[i].charAt(j) == '\'') quoteCount++;
+ // have we got a split inside quoted sub-string?
+ if ((quoteCount % 2) == 0) break;
+ // yes!; replace the '/' and loop to concat next token
+ i++;
+ sb.append("/");
+ }
+ result.add(sb.toString());
+ }
+ return result;
+ }
+
+ static XMLInputFactory factory = XMLInputFactory.newInstance();
+ static {
+ EmptyEntityResolver.configureXMLInputFactory(factory);
+ factory.setXMLReporter(XMLLOG);
+ try {
+ // The java 1.6 bundled stax parser (sjsxp) does not currently have a thread-safe
+ // XMLInputFactory, as that implementation tries to cache and reuse the
+ // XMLStreamReader. Setting the parser-specific "reuse-instance" property to false
+ // prevents this.
+ // All other known open-source stax parsers (and the bea ref impl)
+ // have thread-safe factories.
+ factory.setProperty("reuse-instance", Boolean.FALSE);
+ } catch (IllegalArgumentException ex) {
+ // Other implementations will likely throw this exception since "reuse-instance"
+ // isimplementation specific.
+ log.debug("Unable to set the 'reuse-instance' property for the input chain: " + factory);
+ }
+ }
+
+ /**Implement this interface to stream records as and when one is found.
+ *
+ */
+ public interface Handler {
+ /**
+ * @param record The record map. The key is the field name as provided in
+ * the addField() methods. The value can be a single String (for single
+ * valued fields) or a List<String> (for multiValued).
+ * @param xpath The forEach XPATH for which this record is being emitted
+ * If there is any change all parsing will be aborted and the Exception
+ * is propagated up
+ */
+ void handle(Map<String, Object> record, String xpath);
+ }
+
+ private static final Pattern ATTRIB_PRESENT_WITHVAL = Pattern
+ .compile("(\\S*?)?(\\[@)(\\S*?)(='(.*?)')?(\\])");
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java
new file mode 100644
index 0000000..64a776c
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.zookeeper.KeeperException.NodeExistsException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>
+ * A SolrCloud-friendly extension of {@link SimplePropertiesWriter}.
+ * This implementation ignores the "directory" parameter, saving
+ * the properties file under /configs/[solrcloud collection name]/
+ */
+public class ZKPropertiesWriter extends SimplePropertiesWriter {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private String path;
+ private SolrZkClient zkClient;
+
+ @Override
+ public void init(DataImporter dataImporter, Map<String, String> params) {
+ super.init(dataImporter, params);
+ zkClient = dataImporter.getCore().getCoreContainer().getZkController().getZkClient();
+ }
+
+ @Override
+ protected void findDirectory(DataImporter dataImporter, Map<String, String> params) {
+ String collection = dataImporter.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName();
+ path = "/configs/" + collection + "/" + filename;
+ }
+
+ @Override
+ public boolean isWritable() {
+ return true;
+ }
+
+ @Override
+ public void persist(Map<String, Object> propObjs) {
+ Properties existing = mapToProperties(readIndexerProperties());
+ existing.putAll(mapToProperties(propObjs));
+ StringWriter output = new StringWriter();
+ try {
+ existing.store(output, null);
+ byte[] bytes = output.toString().getBytes(StandardCharsets.UTF_8);
+ if (!zkClient.exists(path, false)) {
+ try {
+ zkClient.makePath(path, false);
+ } catch (NodeExistsException e) {}
+ }
+ zkClient.setData(path, bytes, false);
+ } catch (Exception e) {
+ SolrZkClient.checkInterrupted(e);
+ log.warn("Could not persist properties to " + path + " :" + e.getClass(), e);
+ }
+ }
+
+ @Override
+ public Map<String, Object> readIndexerProperties() {
+ Properties props = new Properties();
+ try {
+ byte[] data = zkClient.getData(path, null, null, true);
+ if (data != null) {
+ props.load(new StringReader(new String(data, StandardCharsets.UTF_8)));
+ }
+ } catch (Exception e) {
+ SolrZkClient.checkInterrupted(e);
+ log.warn("Could not read DIH properties from " + path + " :" + e.getClass(), e);
+ }
+ return propertiesToMap(props);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Zipper.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Zipper.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Zipper.java
new file mode 100644
index 0000000..096b3a8
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Zipper.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.solr.handler.dataimport.DIHCacheSupport.Relation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+class Zipper {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private final DIHCacheSupport.Relation relation;
+
+ @SuppressWarnings("rawtypes")
+ private Comparable parentId;
+ @SuppressWarnings("rawtypes")
+ private Comparable lastChildId;
+
+ private Iterator<Map<String,Object>> rowIterator;
+ private PeekingIterator<Map<String,Object>> peeker;
+
+ /** @return initialized zipper or null */
+ public static Zipper createOrNull(Context context){
+ if("zipper".equals(context.getEntityAttribute("join"))){
+ DIHCacheSupport.Relation r = new DIHCacheSupport.Relation(context);
+ if(r.doKeyLookup){
+ return new Zipper(r);
+ }
+ }
+ return null;
+ }
+
+
+ private Zipper(Relation relation) {
+ this.relation = relation;
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public Map<String,Object> supplyNextChild(
+ Iterator<Map<String,Object>> rowIterator) {
+ preparePeeker(rowIterator);
+
+ while(peeker.hasNext()){
+ Map<String,Object> current = peeker.peek();
+ Comparable childId = (Comparable) current.get(relation.primaryKey);
+
+ if(lastChildId!=null && lastChildId.compareTo(childId)>0){
+ throw new IllegalArgumentException("expect increasing foreign keys for "+relation+
+ " got: "+lastChildId+","+childId);
+ }
+ lastChildId = childId;
+ int cmp = childId.compareTo(parentId);
+ if(cmp==0){
+ Map<String,Object> child = peeker.next();
+ assert child==current: "peeker should be right but "+current+" != " + child;
+ log.trace("yeild child {} entry {}",relation, current);
+ return child;// TODO it's for one->many for many->one it should be just peek()
+ }else{
+ if(cmp<0){ // child belongs to 10th and parent is 20th, skip for the next one
+ Map<String,Object> child = peeker.next();
+ assert child==current: "peeker should be right but "+current+" != " + child;
+ log.trace("skip child {}, {} > {}",relation, parentId, childId);
+ }else{ // child belongs to 20th and parent is 10th, no more children, go to next parent
+ log.trace("childen is over {}, {} < {}", relation, parentId, current);
+ return null;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ private void preparePeeker(Iterator<Map<String,Object>> rowIterator) {
+ if(this.rowIterator==null){
+ this.rowIterator = rowIterator;
+ peeker = Iterators.peekingIterator(rowIterator);
+ }else{
+ assert this.rowIterator==rowIterator: "rowIterator should never change but "+this.rowIterator+
+ " supplied before has been changed to "+rowIterator;
+ }
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void onNewParent(Context context) {
+ Comparable newParent = (Comparable) context.resolve(relation.foreignKey);
+ if(parentId!=null && parentId.compareTo(newParent)>=0){
+ throw new IllegalArgumentException("expect strictly increasing primary keys for "+relation+
+ " got: "+parentId+","+newParent);
+ }
+ log.trace("{}: {}->{}",relation, newParent, parentId);
+ parentId = newParent;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/ConfigNameConstants.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/ConfigNameConstants.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/ConfigNameConstants.java
new file mode 100644
index 0000000..a34e31d
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/ConfigNameConstants.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport.config;
+
+import org.apache.solr.handler.dataimport.SolrWriter;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+public class ConfigNameConstants {
+ public static final String SCRIPT = "script";
+
+ public static final String NAME = "name";
+
+ public static final String PROCESSOR = "processor";
+
+ public static final String PROPERTY_WRITER = "propertyWriter";
+
+ public static final String IMPORTER_NS = "dataimporter";
+
+ public static final String IMPORTER_NS_SHORT = "dih";
+
+ public static final String ROOT_ENTITY = "rootEntity";
+
+ public static final String CHILD = "child";
+
+ public static final String FUNCTION = "function";
+
+ public static final String CLASS = "class";
+
+ public static final String DATA_SRC = "dataSource";
+
+ public static final Set<String> RESERVED_WORDS;
+ static{
+ Set<String> rw = new HashSet<>();
+ rw.add(IMPORTER_NS);
+ rw.add(IMPORTER_NS_SHORT);
+ rw.add("request");
+ rw.add("delta");
+ rw.add("functions");
+ rw.add("session");
+ rw.add(SolrWriter.LAST_INDEX_KEY);
+ RESERVED_WORDS = Collections.unmodifiableSet(rw);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java
new file mode 100644
index 0000000..c369296
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport.config;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class ConfigParseUtil {
+
+ public static String getStringAttribute(Element e, String name, String def) {
+ String r = e.getAttribute(name);
+ if (r == null || "".equals(r.trim())) r = def;
+ return r;
+ }
+
+ public static HashMap<String,String> getAllAttributes(Element e) {
+ HashMap<String,String> m = new HashMap<>();
+ NamedNodeMap nnm = e.getAttributes();
+ for (int i = 0; i < nnm.getLength(); i++) {
+ m.put(nnm.item(i).getNodeName(), nnm.item(i).getNodeValue());
+ }
+ return m;
+ }
+
+ public static String getText(Node elem, StringBuilder buffer) {
+ if (elem.getNodeType() != Node.CDATA_SECTION_NODE) {
+ NodeList childs = elem.getChildNodes();
+ for (int i = 0; i < childs.getLength(); i++) {
+ Node child = childs.item(i);
+ short childType = child.getNodeType();
+ if (childType != Node.COMMENT_NODE
+ && childType != Node.PROCESSING_INSTRUCTION_NODE) {
+ getText(child, buffer);
+ }
+ }
+ } else {
+ buffer.append(elem.getNodeValue());
+ }
+
+ return buffer.toString();
+ }
+
+ public static List<Element> getChildNodes(Element e, String byName) {
+ List<Element> result = new ArrayList<>();
+ NodeList l = e.getChildNodes();
+ for (int i = 0; i < l.getLength(); i++) {
+ if (e.equals(l.item(i).getParentNode())
+ && byName.equals(l.item(i).getNodeName())) result.add((Element) l
+ .item(i));
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java
new file mode 100644
index 0000000..0ba13ea
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport.config;
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.solr.handler.dataimport.DataImporter;
+import org.apache.solr.handler.dataimport.DocBuilder;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Element;
+
+/**
+ * <p>
+ * Mapping for data-config.xml
+ * </p>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @since solr 1.3
+ */
+public class DIHConfiguration {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ // TODO - remove from here and add it to entity
+ private final String deleteQuery;
+
+ private final List<Entity> entities;
+ private final String onImportStart;
+ private final String onImportEnd;
+ private final String onError;
+ private final List<Map<String, String>> functions;
+ private final Script script;
+ private final Map<String, Map<String,String>> dataSources;
+ private final PropertyWriter propertyWriter;
+ private final IndexSchema schema;
+ private final Map<String,SchemaField> lowerNameVsSchemaField;
+
+ public DIHConfiguration(Element element, DataImporter di,
+ List<Map<String,String>> functions, Script script,
+ Map<String,Map<String,String>> dataSources, PropertyWriter pw) {
+ schema = di.getSchema();
+ lowerNameVsSchemaField = null == schema ? Collections.<String,SchemaField>emptyMap() : loadSchemaFieldMap();
+ this.deleteQuery = ConfigParseUtil.getStringAttribute(element, "deleteQuery", null);
+ this.onImportStart = ConfigParseUtil.getStringAttribute(element, "onImportStart", null);
+ this.onImportEnd = ConfigParseUtil.getStringAttribute(element, "onImportEnd", null);
+ this.onError = ConfigParseUtil.getStringAttribute(element, "onError", null);
+ List<Entity> modEntities = new ArrayList<>();
+ List<Element> l = ConfigParseUtil.getChildNodes(element, "entity");
+ boolean docRootFound = false;
+ for (Element e : l) {
+ Entity entity = new Entity(docRootFound, e, di, this, null);
+ Map<String, EntityField> fields = gatherAllFields(di, entity);
+ verifyWithSchema(fields);
+ modEntities.add(entity);
+ }
+ this.entities = Collections.unmodifiableList(modEntities);
+ if(functions==null) {
+ functions = Collections.emptyList();
+ }
+ List<Map<String, String>> modFunc = new ArrayList<>(functions.size());
+ for(Map<String, String> f : functions) {
+ modFunc.add(Collections.unmodifiableMap(f));
+ }
+ this.functions = Collections.unmodifiableList(modFunc);
+ this.script = script;
+ this.dataSources = Collections.unmodifiableMap(dataSources);
+ this.propertyWriter = pw;
+ }
+
+ private void verifyWithSchema(Map<String,EntityField> fields) {
+ Map<String,SchemaField> schemaFields = null;
+ if (schema == null) {
+ schemaFields = Collections.emptyMap();
+ } else {
+ schemaFields = schema.getFields();
+ }
+ for (Map.Entry<String,SchemaField> entry : schemaFields.entrySet()) {
+ SchemaField sf = entry.getValue();
+ if (!fields.containsKey(sf.getName())) {
+ if (sf.isRequired()) {
+ log.info(sf.getName() + " is a required field in SolrSchema . But not found in DataConfig");
+ }
+ }
+ }
+ for (Map.Entry<String,EntityField> entry : fields.entrySet()) {
+ EntityField fld = entry.getValue();
+ SchemaField field = getSchemaField(fld.getName());
+ if (field == null && !isSpecialCommand(fld.getName())) {
+ log.info("The field :" + fld.getName() + " present in DataConfig does not have a counterpart in Solr Schema");
+ }
+ }
+ }
+
+ private Map<String,EntityField> gatherAllFields(DataImporter di, Entity e) {
+ Map<String,EntityField> fields = new HashMap<>();
+ if (e.getFields() != null) {
+ for (EntityField f : e.getFields()) {
+ fields.put(f.getName(), f);
+ }
+ }
+ for (Entity e1 : e.getChildren()) {
+ fields.putAll(gatherAllFields(di, e1));
+ }
+ return fields;
+ }
+
+ private Map<String,SchemaField> loadSchemaFieldMap() {
+ Map<String, SchemaField> modLnvsf = new HashMap<>();
+ for (Map.Entry<String, SchemaField> entry : schema.getFields().entrySet()) {
+ modLnvsf.put(entry.getKey().toLowerCase(Locale.ROOT), entry.getValue());
+ }
+ return Collections.unmodifiableMap(modLnvsf);
+ }
+
+ public SchemaField getSchemaField(String caseInsensitiveName) {
+ SchemaField schemaField = null;
+ if(schema!=null) {
+ schemaField = schema.getFieldOrNull(caseInsensitiveName);
+ }
+ if (schemaField == null) {
+ schemaField = lowerNameVsSchemaField.get(caseInsensitiveName.toLowerCase(Locale.ROOT));
+ }
+ return schemaField;
+ }
+
+
+ public String getDeleteQuery() {
+ return deleteQuery;
+ }
+ public List<Entity> getEntities() {
+ return entities;
+ }
+ public String getOnImportStart() {
+ return onImportStart;
+ }
+ public String getOnImportEnd() {
+ return onImportEnd;
+ }
+ public String getOnError() {
+ return onError;
+ }
+ public List<Map<String,String>> getFunctions() {
+ return functions;
+ }
+ public Map<String,Map<String,String>> getDataSources() {
+ return dataSources;
+ }
+ public Script getScript() {
+ return script;
+ }
+ public PropertyWriter getPropertyWriter() {
+ return propertyWriter;
+ }
+
+ public IndexSchema getSchema() {
+ return schema;
+ }
+
+ public static boolean isSpecialCommand(String fld) {
+ return DocBuilder.DELETE_DOC_BY_ID.equals(fld) ||
+ DocBuilder.DELETE_DOC_BY_QUERY.equals(fld) ||
+ DocBuilder.DOC_BOOST.equals(fld) ||
+ DocBuilder.SKIP_DOC.equals(fld) ||
+ DocBuilder.SKIP_ROW.equals(fld);
+
+ }
+}
\ No newline at end of file
[19/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java
deleted file mode 100644
index 1ee18ef..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.lang.invoke.MethodHandles;
-import java.nio.charset.StandardCharsets;
-import java.security.AccessControlException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.IllformedLocaleException;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.lucene.util.IOUtils;
-import org.apache.solr.common.util.SuppressForbidden;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.core.SolrResourceLoader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-/**
- * <p>
- * Writes properties using {@link Properties#store} .
- * The special property "last_index_time" is converted to a formatted date.
- * Users can configure the location, filename, locale and date format to use.
- * </p>
- */
-public class SimplePropertiesWriter extends DIHProperties {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- static final String LAST_INDEX_KEY = "last_index_time";
-
- protected String filename = null;
-
- protected String configDir = null;
-
- protected Locale locale = null;
-
- protected SimpleDateFormat dateFormat = null;
-
- /**
- * The locale to use when writing the properties file. Default is {@link Locale#ROOT}
- */
- public static final String LOCALE = "locale";
- /**
- * The date format to use when writing values for "last_index_time" to the properties file.
- * See {@link SimpleDateFormat} for patterns. Default is yyyy-MM-dd HH:mm:ss .
- */
- public static final String DATE_FORMAT = "dateFormat";
- /**
- * The directory to save the properties file in. Default is the current core's "config" directory.
- */
- public static final String DIRECTORY = "directory";
- /**
- * The filename to save the properties file to. Default is this Handler's name from solrconfig.xml.
- */
- public static final String FILENAME = "filename";
-
- @Override
- public void init(DataImporter dataImporter, Map<String, String> params) {
- if(params.get(FILENAME) != null) {
- filename = params.get(FILENAME);
- } else if(dataImporter.getHandlerName()!=null) {
- filename = dataImporter.getHandlerName() + ".properties";
- } else {
- filename = "dataimport.properties";
- }
- findDirectory(dataImporter, params);
- if(params.get(LOCALE) != null) {
- locale = getLocale(params.get(LOCALE));
- } else {
- locale = Locale.ROOT;
- }
- if(params.get(DATE_FORMAT) != null) {
- dateFormat = new SimpleDateFormat(params.get(DATE_FORMAT), locale);
- } else {
- dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", locale);
- }
- }
-
- @SuppressForbidden(reason = "Usage of outdated locale parsing with Locale#toString() because of backwards compatibility")
- private Locale getLocale(String name) {
- if (name == null) {
- return Locale.ROOT;
- }
- for (final Locale l : Locale.getAvailableLocales()) {
- if(name.equals(l.toString()) || name.equals(l.getDisplayName(Locale.ROOT))) {
- return locale;
- }
- }
- try {
- return new Locale.Builder().setLanguageTag(name).build();
- } catch (IllformedLocaleException ex) {
- throw new DataImportHandlerException(SEVERE, "Unsupported locale for PropertyWriter: " + name);
- }
- }
-
- protected void findDirectory(DataImporter dataImporter, Map<String, String> params) {
- if(params.get(DIRECTORY) != null) {
- configDir = params.get(DIRECTORY);
- } else {
- SolrCore core = dataImporter.getCore();
- if (core == null) {
- configDir = SolrResourceLoader.locateSolrHome().toString();
- } else {
- configDir = core.getResourceLoader().getConfigDir();
- }
- }
- }
-
- private File getPersistFile() {
- final File filePath;
- if (new File(filename).isAbsolute() || configDir == null) {
- filePath = new File(filename);
- } else {
- filePath = new File(new File(configDir), filename);
- }
- return filePath;
- }
-
- @Override
- public boolean isWritable() {
- File persistFile = getPersistFile();
- try {
- return persistFile.exists()
- ? persistFile.canWrite()
- : persistFile.getParentFile().canWrite();
- } catch (AccessControlException e) {
- return false;
- }
- }
-
- @Override
- public String convertDateToString(Date d) {
- return dateFormat.format(d);
- }
- protected Date convertStringToDate(String s) {
- try {
- return dateFormat.parse(s);
- } catch (ParseException e) {
- throw new DataImportHandlerException(SEVERE, "Value for "
- + LAST_INDEX_KEY + " is invalid for date format "
- + dateFormat.toLocalizedPattern() + " : " + s);
- }
- }
- /**
- * {@link DocBuilder} sends the date as an Object because
- * this class knows how to convert it to a String
- */
- protected Properties mapToProperties(Map<String,Object> propObjs) {
- Properties p = new Properties();
- for(Map.Entry<String,Object> entry : propObjs.entrySet()) {
- String key = entry.getKey();
- String val = null;
- String lastKeyPart = key;
- int lastDotPos = key.lastIndexOf('.');
- if(lastDotPos!=-1 && key.length() > lastDotPos+1) {
- lastKeyPart = key.substring(lastDotPos + 1);
- }
- if(LAST_INDEX_KEY.equals(lastKeyPart) && entry.getValue() instanceof Date) {
- val = convertDateToString((Date) entry.getValue());
- } else {
- val = entry.getValue().toString();
- }
- p.put(key, val);
- }
- return p;
- }
- /**
- * We'll send everything back as Strings as this class has
- * already converted them.
- */
- protected Map<String,Object> propertiesToMap(Properties p) {
- Map<String,Object> theMap = new HashMap<>();
- for(Map.Entry<Object,Object> entry : p.entrySet()) {
- String key = entry.getKey().toString();
- Object val = entry.getValue().toString();
- theMap.put(key, val);
- }
- return theMap;
- }
-
- @Override
- public void persist(Map<String, Object> propObjs) {
- Writer propOutput = null;
- Properties existingProps = mapToProperties(readIndexerProperties());
- Properties newProps = mapToProperties(propObjs);
- try {
- existingProps.putAll(newProps);
- propOutput = new OutputStreamWriter(new FileOutputStream(getPersistFile()), StandardCharsets.UTF_8);
- existingProps.store(propOutput, null);
- log.info("Wrote last indexed time to " + filename);
- } catch (Exception e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Unable to persist Index Start Time", e);
- } finally {
- IOUtils.closeWhileHandlingException(propOutput);
- }
- }
-
- @Override
- public Map<String, Object> readIndexerProperties() {
- Properties props = new Properties();
- InputStream propInput = null;
- try {
- String filePath = configDir;
- if (configDir != null && !configDir.endsWith(File.separator)) {
- filePath += File.separator;
- }
- filePath += filename;
- propInput = new FileInputStream(filePath);
- props.load(new InputStreamReader(propInput, StandardCharsets.UTF_8));
- log.info("Read " + filename);
- } catch (Exception e) {
- log.warn("Unable to read: " + filename);
- } finally {
- IOUtils.closeWhileHandlingException(propInput);
- }
- return propertiesToMap(props);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java
deleted file mode 100644
index 7732673..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java
+++ /dev/null
@@ -1,321 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-import org.apache.http.client.HttpClient;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.HttpClientUtil;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder;
-import org.apache.solr.client.solrj.impl.XMLResponseParser;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.CursorMarkParams;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * <p>
- * An implementation of {@link EntityProcessor} which fetches values from a
- * separate Solr implementation using the SolrJ client library. Yield a row per
- * Solr document.
- * </p>
- * <p>
- * Limitations:
- * All configuration is evaluated at the beginning;
- * Only one query is walked;
- * </p>
- */
-public class SolrEntityProcessor extends EntityProcessorBase {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public static final String SOLR_SERVER = "url";
- public static final String QUERY = "query";
- public static final String TIMEOUT = "timeout";
-
- public static final int TIMEOUT_SECS = 5 * 60; // 5 minutes
- public static final int ROWS_DEFAULT = 50;
-
- private SolrClient solrClient = null;
- private String queryString;
- private int rows = ROWS_DEFAULT;
- private String[] filterQueries;
- private String[] fields;
- private String requestHandler;// 'qt' param
- private int timeout = TIMEOUT_SECS;
-
- @Override
- public void destroy() {
- try {
- solrClient.close();
- } catch (IOException e) {
-
- } finally {
- HttpClientUtil.close(((HttpSolrClient) solrClient).getHttpClient());
- }
- }
-
- /**
- * Factory method that returns a {@link HttpClient} instance used for interfacing with a source Solr service.
- * One can override this method to return a differently configured {@link HttpClient} instance.
- * For example configure https and http authentication.
- *
- * @return a {@link HttpClient} instance used for interfacing with a source Solr service
- */
- protected HttpClient getHttpClient() {
- return HttpClientUtil.createClient(null);
- }
-
- @Override
- protected void firstInit(Context context) {
- super.firstInit(context);
-
- try {
- String serverPath = context.getResolvedEntityAttribute(SOLR_SERVER);
- if (serverPath == null) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "SolrEntityProcessor: parameter 'url' is required");
- }
-
- HttpClient client = getHttpClient();
- URL url = new URL(serverPath);
- // (wt="javabin|xml") default is javabin
- if ("xml".equals(context.getResolvedEntityAttribute(CommonParams.WT))) {
- // TODO: it doesn't matter for this impl when passing a client currently, but we should close this!
- solrClient = new Builder(url.toExternalForm())
- .withHttpClient(client)
- .withResponseParser(new XMLResponseParser())
- .build();
- log.info("using XMLResponseParser");
- } else {
- // TODO: it doesn't matter for this impl when passing a client currently, but we should close this!
- solrClient = new Builder(url.toExternalForm())
- .withHttpClient(client)
- .build();
- log.info("using BinaryResponseParser");
- }
- } catch (MalformedURLException e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE, e);
- }
- }
-
- @Override
- public Map<String,Object> nextRow() {
- buildIterator();
- return getNext();
- }
-
- /**
- * The following method changes the rowIterator mutable field. It requires
- * external synchronization.
- */
- protected void buildIterator() {
- if (rowIterator != null) {
- SolrDocumentListIterator documentListIterator = (SolrDocumentListIterator) rowIterator;
- if (!documentListIterator.hasNext() && documentListIterator.hasMoreRows()) {
- nextPage();
- }
- } else {
- boolean cursor = Boolean.parseBoolean(context
- .getResolvedEntityAttribute(CursorMarkParams.CURSOR_MARK_PARAM));
- rowIterator = !cursor ? new SolrDocumentListIterator(new SolrDocumentList())
- : new SolrDocumentListCursor(new SolrDocumentList(), CursorMarkParams.CURSOR_MARK_START);
- nextPage();
- }
- }
-
- protected void nextPage() {
- ((SolrDocumentListIterator)rowIterator).doQuery();
- }
-
- class SolrDocumentListCursor extends SolrDocumentListIterator {
-
- private final String cursorMark;
-
- public SolrDocumentListCursor(SolrDocumentList solrDocumentList, String cursorMark) {
- super(solrDocumentList);
- this.cursorMark = cursorMark;
- }
-
- @Override
- protected void passNextPage(SolrQuery solrQuery) {
- String timeoutAsString = context.getResolvedEntityAttribute(TIMEOUT);
- if (timeoutAsString != null) {
- throw new DataImportHandlerException(SEVERE,"cursorMark can't be used with timeout");
- }
-
- solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark);
- }
-
- @Override
- protected Iterator<Map<String,Object>> createNextPageIterator(QueryResponse response) {
- return
- new SolrDocumentListCursor(response.getResults(),
- response.getNextCursorMark()) ;
- }
- }
-
- class SolrDocumentListIterator implements Iterator<Map<String,Object>> {
-
- private final int start;
- private final int size;
- private final long numFound;
- private final Iterator<SolrDocument> solrDocumentIterator;
-
- public SolrDocumentListIterator(SolrDocumentList solrDocumentList) {
- this.solrDocumentIterator = solrDocumentList.iterator();
- this.numFound = solrDocumentList.getNumFound();
- // SolrQuery has the start field of type int while SolrDocumentList of
- // type long. We are always querying with an int so we can't receive a
- // long as output. That's the reason why the following cast seems safe
- this.start = (int) solrDocumentList.getStart();
- this.size = solrDocumentList.size();
- }
-
- protected QueryResponse doQuery() {
- SolrEntityProcessor.this.queryString = context.getResolvedEntityAttribute(QUERY);
- if (SolrEntityProcessor.this.queryString == null) {
- throw new DataImportHandlerException(
- DataImportHandlerException.SEVERE,
- "SolrEntityProcessor: parameter 'query' is required"
- );
- }
-
- String rowsP = context.getResolvedEntityAttribute(CommonParams.ROWS);
- if (rowsP != null) {
- rows = Integer.parseInt(rowsP);
- }
-
- String sortParam = context.getResolvedEntityAttribute(CommonParams.SORT);
-
- String fqAsString = context.getResolvedEntityAttribute(CommonParams.FQ);
- if (fqAsString != null) {
- SolrEntityProcessor.this.filterQueries = fqAsString.split(",");
- }
-
- String fieldsAsString = context.getResolvedEntityAttribute(CommonParams.FL);
- if (fieldsAsString != null) {
- SolrEntityProcessor.this.fields = fieldsAsString.split(",");
- }
- SolrEntityProcessor.this.requestHandler = context.getResolvedEntityAttribute(CommonParams.QT);
-
-
- SolrQuery solrQuery = new SolrQuery(queryString);
- solrQuery.setRows(rows);
-
- if (sortParam!=null) {
- solrQuery.setParam(CommonParams.SORT, sortParam);
- }
-
- passNextPage(solrQuery);
-
- if (fields != null) {
- for (String field : fields) {
- solrQuery.addField(field);
- }
- }
- solrQuery.setRequestHandler(requestHandler);
- solrQuery.setFilterQueries(filterQueries);
-
-
- QueryResponse response = null;
- try {
- response = solrClient.query(solrQuery);
- } catch (SolrServerException | IOException | SolrException e) {
- if (ABORT.equals(onError)) {
- wrapAndThrow(SEVERE, e);
- } else if (SKIP.equals(onError)) {
- wrapAndThrow(DataImportHandlerException.SKIP_ROW, e);
- }
- }
-
- if (response != null) {
- SolrEntityProcessor.this.rowIterator = createNextPageIterator(response);
- }
- return response;
- }
-
- protected Iterator<Map<String,Object>> createNextPageIterator(QueryResponse response) {
- return new SolrDocumentListIterator(response.getResults());
- }
-
- protected void passNextPage(SolrQuery solrQuery) {
- String timeoutAsString = context.getResolvedEntityAttribute(TIMEOUT);
- if (timeoutAsString != null) {
- SolrEntityProcessor.this.timeout = Integer.parseInt(timeoutAsString);
- }
-
- solrQuery.setTimeAllowed(timeout * 1000);
-
- solrQuery.setStart(getStart() + getSize());
- }
-
- @Override
- public boolean hasNext() {
- return solrDocumentIterator.hasNext();
- }
-
- @Override
- public Map<String,Object> next() {
- SolrDocument solrDocument = solrDocumentIterator.next();
-
- HashMap<String,Object> map = new HashMap<>();
- Collection<String> fields = solrDocument.getFieldNames();
- for (String field : fields) {
- Object fieldValue = solrDocument.getFieldValue(field);
- map.put(field, fieldValue);
- }
- return map;
- }
-
- public int getStart() {
- return start;
- }
-
- public int getSize() {
- return size;
- }
-
- public boolean hasMoreRows() {
- return numFound > start + size;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrQueryEscapingEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrQueryEscapingEvaluator.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrQueryEscapingEvaluator.java
deleted file mode 100644
index aece031..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrQueryEscapingEvaluator.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-
-import java.util.List;
-
-import org.apache.solr.client.solrj.util.ClientUtils;
-
-public class SolrQueryEscapingEvaluator extends Evaluator {
- @Override
- public String evaluate(String expression, Context context) {
- List<Object> l = parseParams(expression, context.getVariableResolver());
- if (l.size() != 1) {
- throw new DataImportHandlerException(SEVERE, "'escapeQueryChars' must have at least one parameter ");
- }
- String s = l.get(0).toString();
- return ClientUtils.escapeQueryChars(s);
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java
deleted file mode 100644
index 3964f3f..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.params.UpdateParams;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.update.AddUpdateCommand;
-import org.apache.solr.update.CommitUpdateCommand;
-import org.apache.solr.update.DeleteUpdateCommand;
-import org.apache.solr.update.RollbackUpdateCommand;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.*;
-import java.lang.invoke.MethodHandles;
-import java.nio.charset.StandardCharsets;
-
-/**
- * <p> Writes documents to SOLR. </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 1.3
- */
-public class SolrWriter extends DIHWriterBase implements DIHWriter {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public static final String LAST_INDEX_KEY = "last_index_time";
-
- private final UpdateRequestProcessor processor;
- private final int commitWithin;
-
- SolrQueryRequest req;
-
- public SolrWriter(UpdateRequestProcessor processor, SolrQueryRequest req) {
- this.processor = processor;
- this.req = req;
- commitWithin = (req != null) ? req.getParams().getInt(UpdateParams.COMMIT_WITHIN, -1): -1;
- }
-
- @Override
- public void close() {
- try {
- processor.finish();
- } catch (IOException e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Unable to call finish() on UpdateRequestProcessor", e);
- } finally {
- deltaKeys = null;
- try {
- processor.close();
- } catch (IOException e) {
- SolrException.log(log, e);
- }
- }
- }
- @Override
- public boolean upload(SolrInputDocument d) {
- try {
- AddUpdateCommand command = new AddUpdateCommand(req);
- command.solrDoc = d;
- command.commitWithin = commitWithin;
- processor.processAdd(command);
- } catch (Exception e) {
- log.warn("Error creating document : " + d, e);
- return false;
- }
-
- return true;
- }
-
- @Override
- public void deleteDoc(Object id) {
- try {
- log.info("Deleting document: " + id);
- DeleteUpdateCommand delCmd = new DeleteUpdateCommand(req);
- delCmd.setId(id.toString());
- processor.processDelete(delCmd);
- } catch (IOException e) {
- log.error("Exception while deleteing: " + id, e);
- }
- }
-
- @Override
- public void deleteByQuery(String query) {
- try {
- log.info("Deleting documents from Solr with query: " + query);
- DeleteUpdateCommand delCmd = new DeleteUpdateCommand(req);
- delCmd.query = query;
- processor.processDelete(delCmd);
- } catch (IOException e) {
- log.error("Exception while deleting by query: " + query, e);
- }
- }
-
- @Override
- public void commit(boolean optimize) {
- try {
- CommitUpdateCommand commit = new CommitUpdateCommand(req,optimize);
- processor.processCommit(commit);
- } catch (Exception e) {
- log.error("Exception while solr commit.", e);
- }
- }
-
- @Override
- public void rollback() {
- try {
- RollbackUpdateCommand rollback = new RollbackUpdateCommand(req);
- processor.processRollback(rollback);
- } catch (Exception e) {
- log.error("Exception during rollback command.", e);
- }
- }
-
- @Override
- public void doDeleteAll() {
- try {
- DeleteUpdateCommand deleteCommand = new DeleteUpdateCommand(req);
- deleteCommand.query = "*:*";
- processor.processDelete(deleteCommand);
- } catch (IOException e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Exception in full dump while deleting all documents.", e);
- }
- }
-
- static String getResourceAsString(InputStream in) throws IOException {
- ByteArrayOutputStream baos = new ByteArrayOutputStream(1024);
- byte[] buf = new byte[1024];
- int sz = 0;
- try {
- while ((sz = in.read(buf)) != -1) {
- baos.write(buf, 0, sz);
- }
- } finally {
- try {
- in.close();
- } catch (Exception e) {
-
- }
- }
- return new String(baos.toByteArray(), StandardCharsets.UTF_8);
- }
-
- static String getDocCount() {
- if (DocBuilder.INSTANCE.get() != null) {
- return ""
- + (DocBuilder.INSTANCE.get().importStatistics.docCount.get() + 1);
- } else {
- return null;
- }
- }
- @Override
- public void init(Context context) {
- /* NO-OP */
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SortedMapBackedCache.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SortedMapBackedCache.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SortedMapBackedCache.java
deleted file mode 100644
index bb84ba9..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SortedMapBackedCache.java
+++ /dev/null
@@ -1,238 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-public class SortedMapBackedCache implements DIHCache {
- private SortedMap<Object,List<Map<String,Object>>> theMap = null;
- private boolean isOpen = false;
- private boolean isReadOnly = false;
- String primaryKeyName = null;
-
- @SuppressWarnings("unchecked")
- @Override
- public void add(Map<String,Object> rec) {
- checkOpen(true);
- checkReadOnly();
-
- if (rec == null || rec.size() == 0) {
- return;
- }
-
- if (primaryKeyName == null) {
- primaryKeyName = rec.keySet().iterator().next();
- }
-
- Object pk = rec.get(primaryKeyName);
- if (pk instanceof Collection<?>) {
- Collection<Object> c = (Collection<Object>) pk;
- if (c.size() != 1) {
- throw new RuntimeException(
- "The primary key must have exactly 1 element.");
- }
- pk = c.iterator().next();
- }
- //Rows with null keys are not added.
- if(pk==null) {
- return;
- }
- List<Map<String,Object>> thisKeysRecs = theMap.get(pk);
- if (thisKeysRecs == null) {
- thisKeysRecs = new ArrayList<>();
- theMap.put(pk, thisKeysRecs);
- }
- thisKeysRecs.add(rec);
- }
-
- private void checkOpen(boolean shouldItBe) {
- if (!isOpen && shouldItBe) {
- throw new IllegalStateException(
- "Must call open() before using this cache.");
- }
- if (isOpen && !shouldItBe) {
- throw new IllegalStateException("The cache is already open.");
- }
- }
-
- private void checkReadOnly() {
- if (isReadOnly) {
- throw new IllegalStateException("Cache is read-only.");
- }
- }
-
- @Override
- public void close() {
- isOpen = false;
- }
-
- @Override
- public void delete(Object key) {
- checkOpen(true);
- checkReadOnly();
- if(key==null) {
- return;
- }
- theMap.remove(key);
- }
-
- @Override
- public void deleteAll() {
- deleteAll(false);
- }
-
- private void deleteAll(boolean readOnlyOk) {
- if (!readOnlyOk) {
- checkReadOnly();
- }
- if (theMap != null) {
- theMap.clear();
- }
- }
-
- @Override
- public void destroy() {
- deleteAll(true);
- theMap = null;
- isOpen = false;
- }
-
- @Override
- public void flush() {
- checkOpen(true);
- checkReadOnly();
- }
-
- @Override
- public Iterator<Map<String,Object>> iterator(Object key) {
- checkOpen(true);
- if(key==null) {
- return null;
- }
- if(key instanceof Iterable<?>) {
- List<Map<String,Object>> vals = new ArrayList<>();
- Iterator<?> iter = ((Iterable<?>) key).iterator();
- while(iter.hasNext()) {
- List<Map<String,Object>> val = theMap.get(iter.next());
- if(val!=null) {
- vals.addAll(val);
- }
- }
- if(vals.size()==0) {
- return null;
- }
- return vals.iterator();
- }
- List<Map<String,Object>> val = theMap.get(key);
- if (val == null) {
- return null;
- }
- return val.iterator();
- }
-
- @Override
- public Iterator<Map<String,Object>> iterator() {
- return new Iterator<Map<String, Object>>() {
- private Iterator<Map.Entry<Object,List<Map<String,Object>>>> theMapIter;
- private List<Map<String,Object>> currentKeyResult = null;
- private Iterator<Map<String,Object>> currentKeyResultIter = null;
-
- {
- theMapIter = theMap.entrySet().iterator();
- }
-
- @Override
- public boolean hasNext() {
- if (currentKeyResultIter != null) {
- if (currentKeyResultIter.hasNext()) {
- return true;
- } else {
- currentKeyResult = null;
- currentKeyResultIter = null;
- }
- }
-
- Map.Entry<Object,List<Map<String,Object>>> next = null;
- if (theMapIter.hasNext()) {
- next = theMapIter.next();
- currentKeyResult = next.getValue();
- currentKeyResultIter = currentKeyResult.iterator();
- if (currentKeyResultIter.hasNext()) {
- return true;
- }
- }
- return false;
- }
-
- @Override
- public Map<String,Object> next() {
- if (currentKeyResultIter != null) {
- if (currentKeyResultIter.hasNext()) {
- return currentKeyResultIter.next();
- } else {
- currentKeyResult = null;
- currentKeyResultIter = null;
- }
- }
-
- Map.Entry<Object,List<Map<String,Object>>> next = null;
- if (theMapIter.hasNext()) {
- next = theMapIter.next();
- currentKeyResult = next.getValue();
- currentKeyResultIter = currentKeyResult.iterator();
- if (currentKeyResultIter.hasNext()) {
- return currentKeyResultIter.next();
- }
- }
- return null;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
-
- @Override
- public void open(Context context) {
- checkOpen(false);
- isOpen = true;
- if (theMap == null) {
- theMap = new TreeMap<>();
- }
-
- String pkName = CachePropertyUtil.getAttributeValueAsString(context,
- DIHCacheSupport.CACHE_PRIMARY_KEY);
- if (pkName != null) {
- primaryKeyName = pkName;
- }
- isReadOnly = false;
- String readOnlyStr = CachePropertyUtil.getAttributeValueAsString(context,
- DIHCacheSupport.CACHE_READ_ONLY);
- if ("true".equalsIgnoreCase(readOnlyStr)) {
- isReadOnly = true;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SqlEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SqlEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SqlEntityProcessor.java
deleted file mode 100644
index 19c6d0f..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SqlEntityProcessor.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.lang.invoke.MethodHandles;
-import java.util.Iterator;
-import java.util.Map;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * <p>
- * An {@link EntityProcessor} instance which provides support for reading from
- * databases. It is used in conjunction with {@link JdbcDataSource}. This is the default
- * {@link EntityProcessor} if none is specified explicitly in data-config.xml
- * </p>
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- *
- * @since solr 1.3
- */
-public class SqlEntityProcessor extends EntityProcessorBase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- protected DataSource<Iterator<Map<String, Object>>> dataSource;
-
- @Override
- @SuppressWarnings("unchecked")
- public void init(Context context) {
- super.init(context);
- dataSource = context.getDataSource();
- }
-
- protected void initQuery(String q) {
- try {
- DataImporter.QUERY_COUNT.get().incrementAndGet();
- rowIterator = dataSource.getData(q);
- this.query = q;
- } catch (DataImportHandlerException e) {
- throw e;
- } catch (Exception e) {
- log.error( "The query failed '" + q + "'", e);
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE, e);
- }
- }
-
- @Override
- public Map<String, Object> nextRow() {
- if (rowIterator == null) {
- String q = getQuery();
- initQuery(context.replaceTokens(q));
- }
- return getNext();
- }
-
- @Override
- public Map<String, Object> nextModifiedRowKey() {
- if (rowIterator == null) {
- String deltaQuery = context.getEntityAttribute(DELTA_QUERY);
- if (deltaQuery == null)
- return null;
- initQuery(context.replaceTokens(deltaQuery));
- }
- return getNext();
- }
-
- @Override
- public Map<String, Object> nextDeletedRowKey() {
- if (rowIterator == null) {
- String deletedPkQuery = context.getEntityAttribute(DEL_PK_QUERY);
- if (deletedPkQuery == null)
- return null;
- initQuery(context.replaceTokens(deletedPkQuery));
- }
- return getNext();
- }
-
- @Override
- public Map<String, Object> nextModifiedParentRowKey() {
- if (rowIterator == null) {
- String parentDeltaQuery = context.getEntityAttribute(PARENT_DELTA_QUERY);
- if (parentDeltaQuery == null)
- return null;
- log.info("Running parentDeltaQuery for Entity: "
- + context.getEntityAttribute("name"));
- initQuery(context.replaceTokens(parentDeltaQuery));
- }
- return getNext();
- }
-
- public String getQuery() {
- String queryString = context.getEntityAttribute(QUERY);
- if (Context.FULL_DUMP.equals(context.currentProcess())) {
- return queryString;
- }
- if (Context.DELTA_DUMP.equals(context.currentProcess())) {
- String deltaImportQuery = context.getEntityAttribute(DELTA_IMPORT_QUERY);
- if(deltaImportQuery != null) return deltaImportQuery;
- }
- log.warn("'deltaImportQuery' attribute is not specified for entity : "+ entityName);
- return getDeltaImportQuery(queryString);
- }
-
- public String getDeltaImportQuery(String queryString) {
- StringBuilder sb = new StringBuilder(queryString);
- if (SELECT_WHERE_PATTERN.matcher(queryString).find()) {
- sb.append(" and ");
- } else {
- sb.append(" where ");
- }
- boolean first = true;
- String[] primaryKeys = context.getEntityAttribute("pk").split(",");
- for (String primaryKey : primaryKeys) {
- if (!first) {
- sb.append(" and ");
- }
- first = false;
- Object val = context.resolve("dataimporter.delta." + primaryKey);
- if (val == null) {
- Matcher m = DOT_PATTERN.matcher(primaryKey);
- if (m.find()) {
- val = context.resolve("dataimporter.delta." + m.group(1));
- }
- }
- sb.append(primaryKey).append(" = ");
- if (val instanceof Number) {
- sb.append(val.toString());
- } else {
- sb.append("'").append(val.toString()).append("'");
- }
- }
- return sb.toString();
- }
-
- private static Pattern SELECT_WHERE_PATTERN = Pattern.compile(
- "^\\s*(select\\b.*?\\b)(where).*", Pattern.CASE_INSENSITIVE);
-
- public static final String QUERY = "query";
-
- public static final String DELTA_QUERY = "deltaQuery";
-
- public static final String DELTA_IMPORT_QUERY = "deltaImportQuery";
-
- public static final String PARENT_DELTA_QUERY = "parentDeltaQuery";
-
- public static final String DEL_PK_QUERY = "deletedPkQuery";
-
- public static final Pattern DOT_PATTERN = Pattern.compile(".*?\\.(.*)$");
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SqlEscapingEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SqlEscapingEvaluator.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SqlEscapingEvaluator.java
deleted file mode 100644
index 7f9c26e..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SqlEscapingEvaluator.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-
-import java.util.List;
-
-/**
- * <p> Escapes values in SQL queries. It escapes the value of the given expression
- * by replacing all occurrences of single-quotes by two single-quotes and similarily
- * for double-quotes </p>
- */
-public class SqlEscapingEvaluator extends Evaluator {
- @Override
- public String evaluate(String expression, Context context) {
- List<Object> l = parseParams(expression, context.getVariableResolver());
- if (l.size() != 1) {
- throw new DataImportHandlerException(SEVERE, "'escapeSql' must have at least one parameter ");
- }
- String s = l.get(0).toString();
- // escape single quote with two single quotes, double quote
- // with two doule quotes, and backslash with double backslash.
- // See: http://dev.mysql.com/doc/refman/4.1/en/mysql-real-escape-string.html
- return s.replaceAll("'", "''").replaceAll("\"", "\"\"").replaceAll("\\\\", "\\\\\\\\");
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/TemplateTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/TemplateTransformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/TemplateTransformer.java
deleted file mode 100644
index f655edd..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/TemplateTransformer.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.lang.invoke.MethodHandles;
-import java.util.HashMap;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * <p>
- * A {@link Transformer} which can put values into a column by resolving an expression
- * containing other columns
- * </p>
- * <p>
- * For example:<br>
- * <field column="name" template="${e.lastName}, ${e.firstName}
- * ${e.middleName}" /> will produce the name by combining values from
- * lastName, firstName and middleName fields as given in the template attribute.
- * </p>
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- *
- * @since solr 1.3
- */
-public class TemplateTransformer extends Transformer {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private Map<String ,List<String>> templateVsVars = new HashMap<>();
-
- @Override
- @SuppressWarnings("unchecked")
- public Object transformRow(Map<String, Object> row, Context context) {
-
-
- VariableResolver resolver = context.getVariableResolver();
- // Add current row to the copy of resolver map
-
- for (Map<String, String> map : context.getAllEntityFields()) {
- map.entrySet();
- String expr = map.get(TEMPLATE);
- if (expr == null)
- continue;
-
- String column = map.get(DataImporter.COLUMN);
-
- // Verify if all variables can be resolved or not
- boolean resolvable = true;
- List<String> variables = this.templateVsVars.get(expr);
- if(variables == null){
- variables = resolver.getVariables(expr);
- this.templateVsVars.put(expr, variables);
- }
- for (String v : variables) {
- if (resolver.resolve(v) == null) {
- log.warn("Unable to resolve variable: " + v
- + " while parsing expression: " + expr);
- resolvable = false;
- }
- }
-
- if (!resolvable)
- continue;
- if(variables.size() == 1 && expr.startsWith("${") && expr.endsWith("}")){
- addToRow(column, row, resolver.resolve(variables.get(0)));
- } else {
- addToRow(column, row, resolver.replaceTokens(expr));
- }
- }
-
- return row;
- }
-
- private void addToRow(String key, Map<String, Object> row, Object value) {
- Object prevVal = row.get(key);
- if (prevVal != null) {
- if (prevVal instanceof List) {
- ((List) prevVal).add(value);
- } else {
- ArrayList<Object> valList = new ArrayList<Object>();
- valList.add(prevVal);
- valList.add(value);
- row.put(key, valList);
- }
- } else {
- row.put(key, value);
- }
- }
-
- public static final String TEMPLATE = "template";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Transformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Transformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Transformer.java
deleted file mode 100644
index c7923e1..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Transformer.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.Map;
-
-/**
- * <p>
- * Use this API to implement a custom transformer for any given entity
- * </p>
- * <p>
- * Implementations of this abstract class must provide a public no-args constructor.
- * </p>
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- *
- * @since solr 1.3
- */
-public abstract class Transformer {
- /**
- * The input is a row of data and the output has to be a new row.
- *
- * @param context The current context
- * @param row A row of data
- * @return The changed data. It must be a {@link Map}<{@link String}, {@link Object}> if it returns
- * only one row or if there are multiple rows to be returned it must
- * be a {@link java.util.List}<{@link Map}<{@link String}, {@link Object}>>
- */
- public abstract Object transformRow(Map<String, Object> row, Context context);
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/URLDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/URLDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/URLDataSource.java
deleted file mode 100644
index 145ffc4..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/URLDataSource.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.lang.invoke.MethodHandles;
-import java.net.URL;
-import java.net.URLConnection;
-import java.nio.charset.StandardCharsets;
-import java.util.Properties;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * <p> A data source implementation which can be used to read character files using HTTP. </p> <p> Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a> for more
- * details. </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- *
- * @since solr 1.4
- */
-public class URLDataSource extends DataSource<Reader> {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private String baseUrl;
-
- private String encoding;
-
- private int connectionTimeout = CONNECTION_TIMEOUT;
-
- private int readTimeout = READ_TIMEOUT;
-
- private Context context;
-
- private Properties initProps;
-
- public URLDataSource() {
- }
-
- @Override
- public void init(Context context, Properties initProps) {
- this.context = context;
- this.initProps = initProps;
-
- baseUrl = getInitPropWithReplacements(BASE_URL);
- if (getInitPropWithReplacements(ENCODING) != null)
- encoding = getInitPropWithReplacements(ENCODING);
- String cTimeout = getInitPropWithReplacements(CONNECTION_TIMEOUT_FIELD_NAME);
- String rTimeout = getInitPropWithReplacements(READ_TIMEOUT_FIELD_NAME);
- if (cTimeout != null) {
- try {
- connectionTimeout = Integer.parseInt(cTimeout);
- } catch (NumberFormatException e) {
- log.warn("Invalid connection timeout: " + cTimeout);
- }
- }
- if (rTimeout != null) {
- try {
- readTimeout = Integer.parseInt(rTimeout);
- } catch (NumberFormatException e) {
- log.warn("Invalid read timeout: " + rTimeout);
- }
- }
- }
-
- @Override
- public Reader getData(String query) {
- URL url = null;
- try {
- if (URIMETHOD.matcher(query).find()) url = new URL(query);
- else url = new URL(baseUrl + query);
-
- log.debug("Accessing URL: " + url.toString());
-
- URLConnection conn = url.openConnection();
- conn.setConnectTimeout(connectionTimeout);
- conn.setReadTimeout(readTimeout);
- InputStream in = conn.getInputStream();
- String enc = encoding;
- if (enc == null) {
- String cType = conn.getContentType();
- if (cType != null) {
- Matcher m = CHARSET_PATTERN.matcher(cType);
- if (m.find()) {
- enc = m.group(1);
- }
- }
- }
- if (enc == null)
- enc = UTF_8;
- DataImporter.QUERY_COUNT.get().incrementAndGet();
- return new InputStreamReader(in, enc);
- } catch (Exception e) {
- log.error("Exception thrown while getting data", e);
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Exception in invoking url " + url, e);
- }
- }
-
- @Override
- public void close() {
- }
-
- public String getBaseUrl() {
- return baseUrl;
- }
-
- private String getInitPropWithReplacements(String propertyName) {
- final String expr = initProps.getProperty(propertyName);
- if (expr == null) {
- return null;
- }
- return context.replaceTokens(expr);
- }
-
- static final Pattern URIMETHOD = Pattern.compile("\\w{3,}:/");
-
- private static final Pattern CHARSET_PATTERN = Pattern.compile(".*?charset=(.*)$", Pattern.CASE_INSENSITIVE);
-
- public static final String ENCODING = "encoding";
-
- public static final String BASE_URL = "baseUrl";
-
- public static final String UTF_8 = StandardCharsets.UTF_8.name();
-
- public static final String CONNECTION_TIMEOUT_FIELD_NAME = "connectionTimeout";
-
- public static final String READ_TIMEOUT_FIELD_NAME = "readTimeout";
-
- public static final int CONNECTION_TIMEOUT = 5000;
-
- public static final int READ_TIMEOUT = 10000;
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/UrlEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/UrlEvaluator.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/UrlEvaluator.java
deleted file mode 100644
index 8a6654c..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/UrlEvaluator.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-
-import java.net.URLEncoder;
-import java.util.List;
-
-/**
- * <p>Escapes reserved characters in Solr queries</p>
- *
- * @see org.apache.solr.client.solrj.util.ClientUtils#escapeQueryChars(String)
- */
-public class UrlEvaluator extends Evaluator {
- @Override
- public String evaluate(String expression, Context context) {
- List<Object> l = parseParams(expression, context.getVariableResolver());
- if (l.size() != 1) {
- throw new DataImportHandlerException(SEVERE, "'encodeUrl' must have at least one parameter ");
- }
- String s = l.get(0).toString();
-
- try {
- return URLEncoder.encode(s.toString(), "UTF-8");
- } catch (Exception e) {
- wrapAndThrow(SEVERE, e, "Unable to encode expression: " + expression + " with value: " + s);
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
deleted file mode 100644
index 090e21b..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.WeakHashMap;
-import java.util.function.Function;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.solr.common.util.Cache;
-import org.apache.solr.common.util.MapBackedCache;
-import org.apache.solr.update.processor.TemplateUpdateProcessorFactory;
-
-import static org.apache.solr.update.processor.TemplateUpdateProcessorFactory.Resolved;
-
-/**
- * <p>
- * A set of nested maps that can resolve variables by namespaces. Variables are
- * enclosed with a dollar sign then an opening curly brace, ending with a
- * closing curly brace. Namespaces are delimited with '.' (period).
- * </p>
- * <p>
- * This class also has special logic to resolve evaluator calls by recognizing
- * the reserved function namespace: dataimporter.functions.xxx
- * </p>
- * <p>
- * This class caches strings that have already been resolved from the current
- * dih import.
- * </p>
- * <b>This API is experimental and may change in the future.</b>
- *
- *
- * @since solr 1.3
- */
-public class VariableResolver {
-
- private static final Pattern DOT_PATTERN = Pattern.compile("[.]");
- private static final Pattern EVALUATOR_FORMAT_PATTERN = Pattern
- .compile("^(\\w*?)\\((.*?)\\)$");
- private Map<String,Object> rootNamespace;
- private Map<String,Evaluator> evaluators;
- private Cache<String,Resolved> cache = new MapBackedCache<>(new WeakHashMap<>());
- private Function<String,Object> fun = this::resolve;
-
- public static final String FUNCTIONS_NAMESPACE = "dataimporter.functions.";
- public static final String FUNCTIONS_NAMESPACE_SHORT = "dih.functions.";
-
- public VariableResolver() {
- rootNamespace = new HashMap<>();
- }
-
- public VariableResolver(Properties defaults) {
- rootNamespace = new HashMap<>();
- for (Map.Entry<Object,Object> entry : defaults.entrySet()) {
- rootNamespace.put(entry.getKey().toString(), entry.getValue());
- }
- }
-
- public VariableResolver(Map<String,Object> defaults) {
- rootNamespace = new HashMap<>(defaults);
- }
-
- /**
- * Resolves a given value with a name
- *
- * @param name
- * the String to be resolved
- * @return an Object which is the result of evaluation of given name
- */
- public Object resolve(String name) {
- Object r = null;
- if (name != null) {
- String[] nameParts = DOT_PATTERN.split(name);
- CurrentLevel cr = currentLevelMap(nameParts,
- rootNamespace, false);
- Map<String,Object> currentLevel = cr.map;
- r = currentLevel.get(nameParts[nameParts.length - 1]);
- if (r == null && name.startsWith(FUNCTIONS_NAMESPACE)
- && name.length() > FUNCTIONS_NAMESPACE.length()) {
- return resolveEvaluator(FUNCTIONS_NAMESPACE, name);
- }
- if (r == null && name.startsWith(FUNCTIONS_NAMESPACE_SHORT)
- && name.length() > FUNCTIONS_NAMESPACE_SHORT.length()) {
- return resolveEvaluator(FUNCTIONS_NAMESPACE_SHORT, name);
- }
- if (r == null) {
- StringBuilder sb = new StringBuilder();
- for(int i=cr.level ; i<nameParts.length ; i++) {
- if(sb.length()>0) {
- sb.append(".");
- }
- sb.append(nameParts[i]);
- }
- r = cr.map.get(sb.toString());
- }
- if (r == null) {
- r = System.getProperty(name);
- }
- }
- return r == null ? "" : r;
- }
-
- private Object resolveEvaluator(String namespace, String name) {
- if (evaluators == null) {
- return "";
- }
- Matcher m = EVALUATOR_FORMAT_PATTERN.matcher(name
- .substring(namespace.length()));
- if (m.find()) {
- String fname = m.group(1);
- Evaluator evaluator = evaluators.get(fname);
- if (evaluator == null) return "";
- ContextImpl ctx = new ContextImpl(null, this, null, null, null, null,
- null);
- String g2 = m.group(2);
- return evaluator.evaluate(g2, ctx);
- } else {
- return "";
- }
- }
-
- /**
- * Given a String with place holders, replace them with the value tokens.
- *
- * @return the string with the placeholders replaced with their values
- */
- public String replaceTokens(String template) {
- return TemplateUpdateProcessorFactory.replaceTokens(template, cache, fun, TemplateUpdateProcessorFactory.DOLLAR_BRACES_PLACEHOLDER_PATTERN);
- }
- public void addNamespace(String name, Map<String,Object> newMap) {
- if (newMap != null) {
- if (name != null) {
- String[] nameParts = DOT_PATTERN.split(name);
- Map<String,Object> nameResolveLevel = currentLevelMap(nameParts,
- rootNamespace, false).map;
- nameResolveLevel.put(nameParts[nameParts.length - 1], newMap);
- } else {
- for (Map.Entry<String,Object> entry : newMap.entrySet()) {
- String[] keyParts = DOT_PATTERN.split(entry.getKey());
- Map<String,Object> currentLevel = rootNamespace;
- currentLevel = currentLevelMap(keyParts, currentLevel, false).map;
- currentLevel.put(keyParts[keyParts.length - 1], entry.getValue());
- }
- }
- }
- }
-
- public List<String> getVariables(String expr) {
- return TemplateUpdateProcessorFactory.getVariables(expr, cache, TemplateUpdateProcessorFactory.DOLLAR_BRACES_PLACEHOLDER_PATTERN);
- }
-
- static class CurrentLevel {
- final Map<String,Object> map;
- final int level;
- CurrentLevel(int level, Map<String,Object> map) {
- this.level = level;
- this.map = map;
- }
- }
-
- private CurrentLevel currentLevelMap(String[] keyParts,
- Map<String,Object> currentLevel, boolean includeLastLevel) {
- int j = includeLastLevel ? keyParts.length : keyParts.length - 1;
- for (int i = 0; i < j; i++) {
- Object o = currentLevel.get(keyParts[i]);
- if (o == null) {
- if(i == j-1) {
- Map<String,Object> nextLevel = new HashMap<>();
- currentLevel.put(keyParts[i], nextLevel);
- currentLevel = nextLevel;
- } else {
- return new CurrentLevel(i, currentLevel);
- }
- } else if (o instanceof Map<?,?>) {
- @SuppressWarnings("unchecked")
- Map<String,Object> nextLevel = (Map<String,Object>) o;
- currentLevel = nextLevel;
- } else {
- throw new AssertionError(
- "Non-leaf nodes should be of type java.util.Map");
- }
- }
- return new CurrentLevel(j-1, currentLevel);
- }
-
- public void removeNamespace(String name) {
- rootNamespace.remove(name);
- }
-
- public void setEvaluators(Map<String,Evaluator> evaluators) {
- this.evaluators = evaluators;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java
deleted file mode 100644
index c93b581..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-import org.apache.solr.core.SolrCore;
-import org.apache.lucene.analysis.util.ResourceLoader;
-import org.apache.solr.util.SystemIdResolver;
-import org.apache.solr.common.util.XMLErrorLogger;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.commons.io.IOUtils;
-
-import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.stream.StreamResult;
-import javax.xml.transform.stream.StreamSource;
-import java.io.CharArrayReader;
-import java.io.CharArrayWriter;
-import java.io.Reader;
-import java.lang.invoke.MethodHandles;
-import java.util.*;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-
-/**
- * <p> An implementation of {@link EntityProcessor} which uses a streaming xpath parser to extract values out of XML documents.
- * It is typically used in conjunction with {@link URLDataSource} or {@link FileDataSource}. </p> <p> Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a> for more
- * details. </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- *
- * @see XPathRecordReader
- * @since solr 1.3
- */
-public class XPathEntityProcessor extends EntityProcessorBase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private static final XMLErrorLogger xmllog = new XMLErrorLogger(log);
-
- private static final Map<String, Object> END_MARKER = new HashMap<>();
-
- protected List<String> placeHolderVariables;
-
- protected List<String> commonFields;
-
- private String pk;
-
- private XPathRecordReader xpathReader;
-
- protected DataSource<Reader> dataSource;
-
- protected javax.xml.transform.Transformer xslTransformer;
-
- protected boolean useSolrAddXml = false;
-
- protected boolean streamRows = false;
-
- // Amount of time to block reading/writing to queue when streaming
- protected int blockingQueueTimeOut = 10;
-
- // Units for pumpTimeOut
- protected TimeUnit blockingQueueTimeOutUnits = TimeUnit.SECONDS;
-
- // Number of rows to queue for asynchronous processing
- protected int blockingQueueSize = 1000;
-
- protected Thread publisherThread;
-
- protected boolean reinitXPathReader = true;
-
- @Override
- @SuppressWarnings("unchecked")
- public void init(Context context) {
- super.init(context);
- if (reinitXPathReader)
- initXpathReader(context.getVariableResolver());
- pk = context.getEntityAttribute("pk");
- dataSource = context.getDataSource();
- rowIterator = null;
-
- }
-
- private void initXpathReader(VariableResolver resolver) {
- reinitXPathReader = false;
- useSolrAddXml = Boolean.parseBoolean(context
- .getEntityAttribute(USE_SOLR_ADD_SCHEMA));
- streamRows = Boolean.parseBoolean(context
- .getEntityAttribute(STREAM));
- if (context.getResolvedEntityAttribute("batchSize") != null) {
- blockingQueueSize = Integer.parseInt(context.getEntityAttribute("batchSize"));
- }
- if (context.getResolvedEntityAttribute("readTimeOut") != null) {
- blockingQueueTimeOut = Integer.parseInt(context.getEntityAttribute("readTimeOut"));
- }
- String xslt = context.getEntityAttribute(XSL);
- if (xslt != null) {
- xslt = context.replaceTokens(xslt);
- try {
- // create an instance of TransformerFactory
- TransformerFactory transFact = TransformerFactory.newInstance();
- final SolrCore core = context.getSolrCore();
- final StreamSource xsltSource;
- if (core != null) {
- final ResourceLoader loader = core.getResourceLoader();
- transFact.setURIResolver(new SystemIdResolver(loader).asURIResolver());
- xsltSource = new StreamSource(loader.openResource(xslt),
- SystemIdResolver.createSystemIdFromResourceName(xslt));
- } else {
- // fallback for tests
- xsltSource = new StreamSource(xslt);
- }
- transFact.setErrorListener(xmllog);
- try {
- xslTransformer = transFact.newTransformer(xsltSource);
- } finally {
- // some XML parsers are broken and don't close the byte stream (but they should according to spec)
- IOUtils.closeQuietly(xsltSource.getInputStream());
- }
- log.info("Using xslTransformer: "
- + xslTransformer.getClass().getName());
- } catch (Exception e) {
- throw new DataImportHandlerException(SEVERE,
- "Error initializing XSL ", e);
- }
- }
-
- if (useSolrAddXml) {
- // Support solr add documents
- xpathReader = new XPathRecordReader("/add/doc");
- xpathReader.addField("name", "/add/doc/field/@name", true);
- xpathReader.addField("value", "/add/doc/field", true);
- } else {
- String forEachXpath = context.getResolvedEntityAttribute(FOR_EACH);
- if (forEachXpath == null)
- throw new DataImportHandlerException(SEVERE,
- "Entity : " + context.getEntityAttribute("name")
- + " must have a 'forEach' attribute");
- if (forEachXpath.equals(context.getEntityAttribute(FOR_EACH))) reinitXPathReader = true;
-
- try {
- xpathReader = new XPathRecordReader(forEachXpath);
- for (Map<String, String> field : context.getAllEntityFields()) {
- if (field.get(XPATH) == null)
- continue;
- int flags = 0;
- if ("true".equals(field.get("flatten"))) {
- flags = XPathRecordReader.FLATTEN;
- }
- String xpath = field.get(XPATH);
- xpath = context.replaceTokens(xpath);
- //!xpath.equals(field.get(XPATH) means the field xpath has a template
- //in that case ensure that the XPathRecordReader is reinitialized
- //for each xml
- if (!xpath.equals(field.get(XPATH)) && !context.isRootEntity()) reinitXPathReader = true;
- xpathReader.addField(field.get(DataImporter.COLUMN),
- xpath,
- Boolean.parseBoolean(field.get(DataImporter.MULTI_VALUED)),
- flags);
- }
- } catch (RuntimeException e) {
- throw new DataImportHandlerException(SEVERE,
- "Exception while reading xpaths for fields", e);
- }
- }
- String url = context.getEntityAttribute(URL);
- List<String> l = url == null ? Collections.EMPTY_LIST : resolver.getVariables(url);
- for (String s : l) {
- if (s.startsWith(entityName + ".")) {
- if (placeHolderVariables == null)
- placeHolderVariables = new ArrayList<>();
- placeHolderVariables.add(s.substring(entityName.length() + 1));
- }
- }
- for (Map<String, String> fld : context.getAllEntityFields()) {
- if (fld.get(COMMON_FIELD) != null && "true".equals(fld.get(COMMON_FIELD))) {
- if (commonFields == null)
- commonFields = new ArrayList<>();
- commonFields.add(fld.get(DataImporter.COLUMN));
- }
- }
-
- }
-
- @Override
- public Map<String, Object> nextRow() {
- Map<String, Object> result;
-
- if (!context.isRootEntity())
- return fetchNextRow();
-
- while (true) {
- result = fetchNextRow();
-
- if (result == null)
- return null;
-
- if (pk == null || result.get(pk) != null)
- return result;
- }
- }
-
- @Override
- public void postTransform(Map<String, Object> r) {
- readUsefulVars(r);
- }
-
- @SuppressWarnings("unchecked")
- private Map<String, Object> fetchNextRow() {
- Map<String, Object> r = null;
- while (true) {
- if (rowIterator == null)
- initQuery(context.replaceTokens(context.getEntityAttribute(URL)));
- r = getNext();
- if (r == null) {
- Object hasMore = context.getSessionAttribute(HAS_MORE, Context.SCOPE_ENTITY);
- try {
- if ("true".equals(hasMore) || Boolean.TRUE.equals(hasMore)) {
- String url = (String) context.getSessionAttribute(NEXT_URL, Context.SCOPE_ENTITY);
- if (url == null)
- url = context.getEntityAttribute(URL);
- addNamespace();
- initQuery(context.replaceTokens(url));
- r = getNext();
- if (r == null)
- return null;
- } else {
- return null;
- }
- } finally {
- context.setSessionAttribute(HAS_MORE,null,Context.SCOPE_ENTITY);
- context.setSessionAttribute(NEXT_URL,null,Context.SCOPE_ENTITY);
- }
- }
- addCommonFields(r);
- return r;
- }
- }
-
- private void addNamespace() {
- Map<String, Object> namespace = new HashMap<>();
- Set<String> allNames = new HashSet<>();
- if (commonFields != null) allNames.addAll(commonFields);
- if (placeHolderVariables != null) allNames.addAll(placeHolderVariables);
- if(allNames.isEmpty()) return;
-
- for (String name : allNames) {
- Object val = context.getSessionAttribute(name, Context.SCOPE_ENTITY);
- if (val != null) namespace.put(name, val);
- }
- ((VariableResolver)context.getVariableResolver()).addNamespace(entityName, namespace);
- }
-
- private void addCommonFields(Map<String, Object> r) {
- if(commonFields != null){
- for (String commonField : commonFields) {
- if(r.get(commonField) == null) {
- Object val = context.getSessionAttribute(commonField, Context.SCOPE_ENTITY);
- if(val != null) r.put(commonField, val);
- }
-
- }
- }
-
- }
-
- private void initQuery(String s) {
- Reader data = null;
- try {
- final List<Map<String, Object>> rows = new ArrayList<>();
- try {
- data = dataSource.getData(s);
- } catch (Exception e) {
- if (ABORT.equals(onError)) {
- wrapAndThrow(SEVERE, e);
- } else if (SKIP.equals(onError)) {
- if (log.isDebugEnabled()) log.debug("Skipping url : " + s, e);
- wrapAndThrow(DataImportHandlerException.SKIP, e);
- } else {
- log.warn("Failed for url : " + s, e);
- rowIterator = Collections.EMPTY_LIST.iterator();
- return;
- }
- }
- if (xslTransformer != null) {
- try {
- SimpleCharArrayReader caw = new SimpleCharArrayReader();
- xslTransformer.transform(new StreamSource(data),
- new StreamResult(caw));
- data = caw.getReader();
- } catch (TransformerException e) {
- if (ABORT.equals(onError)) {
- wrapAndThrow(SEVERE, e, "Exception in applying XSL Transformation");
- } else if (SKIP.equals(onError)) {
- wrapAndThrow(DataImportHandlerException.SKIP, e);
- } else {
- log.warn("Failed for url : " + s, e);
- rowIterator = Collections.EMPTY_LIST.iterator();
- return;
- }
- }
- }
- if (streamRows) {
- rowIterator = getRowIterator(data, s);
- } else {
- try {
- xpathReader.streamRecords(data, (record, xpath) -> rows.add(readRow(record, xpath)));
- } catch (Exception e) {
- String msg = "Parsing failed for xml, url:" + s + " rows processed:" + rows.size();
- if (rows.size() > 0) msg += " last row: " + rows.get(rows.size() - 1);
- if (ABORT.equals(onError)) {
- wrapAndThrow(SEVERE, e, msg);
- } else if (SKIP.equals(onError)) {
- log.warn(msg, e);
- Map<String, Object> map = new HashMap<>();
- map.put(DocBuilder.SKIP_DOC, Boolean.TRUE);
- rows.add(map);
- } else if (CONTINUE.equals(onError)) {
- log.warn(msg, e);
- }
- }
- rowIterator = rows.iterator();
- }
- } finally {
- if (!streamRows) {
- closeIt(data);
- }
-
- }
- }
-
- private void closeIt(Reader data) {
- try {
- data.close();
- } catch (Exception e) { /* Ignore */
- }
- }
-
- protected Map<String, Object> readRow(Map<String, Object> record, String xpath) {
- if (useSolrAddXml) {
- List<String> names = (List<String>) record.get("name");
- List<String> values = (List<String>) record.get("value");
- Map<String, Object> row = new HashMap<>();
- for (int i = 0; i < names.size() && i < values.size(); i++) {
- if (row.containsKey(names.get(i))) {
- Object existing = row.get(names.get(i));
- if (existing instanceof List) {
- List list = (List) existing;
- list.add(values.get(i));
- } else {
- List list = new ArrayList();
- list.add(existing);
- list.add(values.get(i));
- row.put(names.get(i), list);
- }
- } else {
- row.put(names.get(i), values.get(i));
- }
- }
- return row;
- } else {
- record.put(XPATH_FIELD_NAME, xpath);
- return record;
- }
- }
-
-
- private static class SimpleCharArrayReader extends CharArrayWriter {
- public Reader getReader() {
- return new CharArrayReader(super.buf, 0, super.count);
- }
-
- }
-
- @SuppressWarnings("unchecked")
- private Map<String, Object> readUsefulVars(Map<String, Object> r) {
- Object val = r.get(HAS_MORE);
- if (val != null)
- context.setSessionAttribute(HAS_MORE, val,Context.SCOPE_ENTITY);
- val = r.get(NEXT_URL);
- if (val != null)
- context.setSessionAttribute(NEXT_URL, val,Context.SCOPE_ENTITY);
- if (placeHolderVariables != null) {
- for (String s : placeHolderVariables) {
- val = r.get(s);
- context.setSessionAttribute(s, val,Context.SCOPE_ENTITY);
- }
- }
- if (commonFields != null) {
- for (String s : commonFields) {
- Object commonVal = r.get(s);
- if (commonVal != null) {
- context.setSessionAttribute(s, commonVal,Context.SCOPE_ENTITY);
- }
- }
- }
- return r;
-
- }
-
- private Iterator<Map<String, Object>> getRowIterator(final Reader data, final String s) {
- //nothing atomic about it. I just needed a StongReference
- final AtomicReference<Exception> exp = new AtomicReference<>();
- final BlockingQueue<Map<String, Object>> blockingQueue = new ArrayBlockingQueue<>(blockingQueueSize);
- final AtomicBoolean isEnd = new AtomicBoolean(false);
- final AtomicBoolean throwExp = new AtomicBoolean(true);
- publisherThread = new Thread() {
- @Override
- public void run() {
- try {
- xpathReader.streamRecords(data, (record, xpath) -> {
- if (isEnd.get()) {
- throwExp.set(false);
- //To end the streaming . otherwise the parsing will go on forever
- //though consumer has gone away
- throw new RuntimeException("BREAK");
- }
- Map<String, Object> row;
- try {
- row = readRow(record, xpath);
- } catch (Exception e) {
- isEnd.set(true);
- return;
- }
- offer(row);
- });
- } catch (Exception e) {
- if(throwExp.get()) exp.set(e);
- } finally {
- closeIt(data);
- if (!isEnd.get()) {
- offer(END_MARKER);
- }
- }
- }
-
- private void offer(Map<String, Object> row) {
- try {
- while (!blockingQueue.offer(row, blockingQueueTimeOut, blockingQueueTimeOutUnits)) {
- if (isEnd.get()) return;
- log.debug("Timeout elapsed writing records. Perhaps buffer size should be increased.");
- }
- } catch (InterruptedException e) {
- return;
- } finally {
- synchronized (this) {
- notifyAll();
- }
- }
- }
- };
-
- publisherThread.start();
-
- return new Iterator<Map<String, Object>>() {
- private Map<String, Object> lastRow;
- int count = 0;
-
- @Override
- public boolean hasNext() {
- return !isEnd.get();
- }
-
- @Override
- public Map<String, Object> next() {
- Map<String, Object> row;
-
- do {
- try {
- row = blockingQueue.poll(blockingQueueTimeOut, blockingQueueTimeOutUnits);
- if (row == null) {
- log.debug("Timeout elapsed reading records.");
- }
- } catch (InterruptedException e) {
- log.debug("Caught InterruptedException while waiting for row. Aborting.");
- isEnd.set(true);
- return null;
- }
- } while (row == null);
-
- if (row == END_MARKER) {
- isEnd.set(true);
- if (exp.get() != null) {
- String msg = "Parsing failed for xml, url:" + s + " rows processed in this xml:" + count;
- if (lastRow != null) msg += " last row in this xml:" + lastRow;
- if (ABORT.equals(onError)) {
- wrapAndThrow(SEVERE, exp.get(), msg);
- } else if (SKIP.equals(onError)) {
- wrapAndThrow(DataImportHandlerException.SKIP, exp.get());
- } else {
- log.warn(msg, exp.get());
- }
- }
- return null;
- }
- count++;
- return lastRow = row;
- }
-
- @Override
- public void remove() {
- /*no op*/
- }
- };
-
- }
-
-
- public static final String URL = "url";
-
- public static final String HAS_MORE = "$hasMore";
-
- public static final String NEXT_URL = "$nextUrl";
-
- public static final String XPATH_FIELD_NAME = "$forEach";
-
- public static final String FOR_EACH = "forEach";
-
- public static final String XPATH = "xpath";
-
- public static final String COMMON_FIELD = "commonField";
-
- public static final String USE_SOLR_ADD_SCHEMA = "useSolrAddSchema";
-
- public static final String XSL = "xsl";
-
- public static final String STREAM = "stream";
-
-}
[02/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
deleted file mode 100644
index 72da77a..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
+++ /dev/null
@@ -1,491 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.File;
-import java.io.Reader;
-import java.io.StringReader;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
-import org.junit.Test;
-
-/**
- * <p>
- * Test for XPathEntityProcessor
- * </p>
- *
- *
- * @since solr 1.3
- */
-public class TestXPathEntityProcessor extends AbstractDataImportHandlerTestCase {
- boolean simulateSlowReader;
- boolean simulateSlowResultProcessor;
- int rowsToRead = -1;
-
- @Test
- public void withFieldsAndXpath() throws Exception {
- File tmpdir = createTempDir().toFile();
-
- createFile(tmpdir, "x.xsl", xsl.getBytes(StandardCharsets.UTF_8), false);
- Map entityAttrs = createMap("name", "e", "url", "cd.xml",
- XPathEntityProcessor.FOR_EACH, "/catalog/cd");
- List fields = new ArrayList();
- fields.add(createMap("column", "title", "xpath", "/catalog/cd/title"));
- fields.add(createMap("column", "artist", "xpath", "/catalog/cd/artist"));
- fields.add(createMap("column", "year", "xpath", "/catalog/cd/year"));
- Context c = getContext(null,
- new VariableResolver(), getDataSource(cdData), Context.FULL_DUMP, fields, entityAttrs);
- XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
- xPathEntityProcessor.init(c);
- List<Map<String, Object>> result = new ArrayList<>();
- while (true) {
- Map<String, Object> row = xPathEntityProcessor.nextRow();
- if (row == null)
- break;
- result.add(row);
- }
- assertEquals(3, result.size());
- assertEquals("Empire Burlesque", result.get(0).get("title"));
- assertEquals("Bonnie Tyler", result.get(1).get("artist"));
- assertEquals("1982", result.get(2).get("year"));
- }
-
- @Test
- public void testMultiValued() throws Exception {
- Map entityAttrs = createMap("name", "e", "url", "testdata.xml",
- XPathEntityProcessor.FOR_EACH, "/root");
- List fields = new ArrayList();
- fields.add(createMap("column", "a", "xpath", "/root/a", DataImporter.MULTI_VALUED, "true"));
- Context c = getContext(null,
- new VariableResolver(), getDataSource(testXml), Context.FULL_DUMP, fields, entityAttrs);
- XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
- xPathEntityProcessor.init(c);
- List<Map<String, Object>> result = new ArrayList<>();
- while (true) {
- Map<String, Object> row = xPathEntityProcessor.nextRow();
- if (row == null)
- break;
- result.add(row);
- }
- List l = (List)result.get(0).get("a");
- assertEquals(3, l.size());
- assertEquals("1", l.get(0));
- assertEquals("2", l.get(1));
- assertEquals("ü", l.get(2));
- }
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- @Test
- public void testMultiValuedWithMultipleDocuments() throws Exception {
- Map entityAttrs = createMap("name", "e", "url", "testdata.xml", XPathEntityProcessor.FOR_EACH, "/documents/doc");
- List fields = new ArrayList();
- fields.add(createMap("column", "id", "xpath", "/documents/doc/id", DataImporter.MULTI_VALUED, "false"));
- fields.add(createMap("column", "a", "xpath", "/documents/doc/a", DataImporter.MULTI_VALUED, "true"));
- fields.add(createMap("column", "s1dataA", "xpath", "/documents/doc/sec1/s1dataA", DataImporter.MULTI_VALUED, "true"));
- fields.add(createMap("column", "s1dataB", "xpath", "/documents/doc/sec1/s1dataB", DataImporter.MULTI_VALUED, "true"));
- fields.add(createMap("column", "s1dataC", "xpath", "/documents/doc/sec1/s1dataC", DataImporter.MULTI_VALUED, "true"));
-
- Context c = getContext(null,
- new VariableResolver(), getDataSource(textMultipleDocuments), Context.FULL_DUMP, fields, entityAttrs);
- XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
- xPathEntityProcessor.init(c);
- List<Map<String, Object>> result = new ArrayList<>();
- while (true) {
- Map<String, Object> row = xPathEntityProcessor.nextRow();
- if (row == null)
- break;
- result.add(row);
- }
- {
- assertEquals("1", result.get(0).get("id"));
- List a = (List)result.get(0).get("a");
- List s1dataA = (List)result.get(0).get("s1dataA");
- List s1dataB = (List)result.get(0).get("s1dataB");
- List s1dataC = (List)result.get(0).get("s1dataC");
- assertEquals(2, a.size());
- assertEquals("id1-a1", a.get(0));
- assertEquals("id1-a2", a.get(1));
- assertEquals(3, s1dataA.size());
- assertEquals("id1-s1dataA-1", s1dataA.get(0));
- assertNull(s1dataA.get(1));
- assertEquals("id1-s1dataA-3", s1dataA.get(2));
- assertEquals(3, s1dataB.size());
- assertEquals("id1-s1dataB-1", s1dataB.get(0));
- assertEquals("id1-s1dataB-2", s1dataB.get(1));
- assertEquals("id1-s1dataB-3", s1dataB.get(2));
- assertEquals(3, s1dataC.size());
- assertNull(s1dataC.get(0));
- assertNull(s1dataC.get(1));
- assertNull(s1dataC.get(2));
- }
- {
- assertEquals("2", result.get(1).get("id"));
- List a = (List)result.get(1).get("a");
- List s1dataA = (List)result.get(1).get("s1dataA");
- List s1dataB = (List)result.get(1).get("s1dataB");
- List s1dataC = (List)result.get(1).get("s1dataC");
- assertTrue(a==null || a.size()==0);
- assertEquals(1, s1dataA.size());
- assertNull(s1dataA.get(0));
- assertEquals(1, s1dataB.size());
- assertEquals("id2-s1dataB-1", s1dataB.get(0));
- assertEquals(1, s1dataC.size());
- assertNull(s1dataC.get(0));
- }
- {
- assertEquals("3", result.get(2).get("id"));
- List a = (List)result.get(2).get("a");
- List s1dataA = (List)result.get(2).get("s1dataA");
- List s1dataB = (List)result.get(2).get("s1dataB");
- List s1dataC = (List)result.get(2).get("s1dataC");
- assertTrue(a==null || a.size()==0);
- assertEquals(1, s1dataA.size());
- assertEquals("id3-s1dataA-1", s1dataA.get(0));
- assertEquals(1, s1dataB.size());
- assertNull(s1dataB.get(0));
- assertEquals(1, s1dataC.size());
- assertNull(s1dataC.get(0));
- }
- {
- assertEquals("4", result.get(3).get("id"));
- List a = (List)result.get(3).get("a");
- List s1dataA = (List)result.get(3).get("s1dataA");
- List s1dataB = (List)result.get(3).get("s1dataB");
- List s1dataC = (List)result.get(3).get("s1dataC");
- assertTrue(a==null || a.size()==0);
- assertEquals(1, s1dataA.size());
- assertEquals("id4-s1dataA-1", s1dataA.get(0));
- assertEquals(1, s1dataB.size());
- assertEquals("id4-s1dataB-1", s1dataB.get(0));
- assertEquals(1, s1dataC.size());
- assertEquals("id4-s1dataC-1", s1dataC.get(0));
- }
- {
- assertEquals("5", result.get(4).get("id"));
- List a = (List)result.get(4).get("a");
- List s1dataA = (List)result.get(4).get("s1dataA");
- List s1dataB = (List)result.get(4).get("s1dataB");
- List s1dataC = (List)result.get(4).get("s1dataC");
- assertTrue(a==null || a.size()==0);
- assertEquals(1, s1dataA.size());
- assertNull(s1dataA.get(0));
- assertEquals(1, s1dataB.size());
- assertNull(s1dataB.get(0));
- assertEquals(1, s1dataC.size());
- assertEquals("id5-s1dataC-1", s1dataC.get(0));
- }
- {
- assertEquals("6", result.get(5).get("id"));
- List a = (List)result.get(5).get("a");
- List s1dataA = (List)result.get(5).get("s1dataA");
- List s1dataB = (List)result.get(5).get("s1dataB");
- List s1dataC = (List)result.get(5).get("s1dataC");
- assertTrue(a==null || a.size()==0);
- assertEquals(3, s1dataA.size());
- assertEquals("id6-s1dataA-1", s1dataA.get(0));
- assertEquals("id6-s1dataA-2", s1dataA.get(1));
- assertNull(s1dataA.get(2));
- assertEquals(3, s1dataB.size());
- assertEquals("id6-s1dataB-1", s1dataB.get(0));
- assertEquals("id6-s1dataB-2", s1dataB.get(1));
- assertEquals("id6-s1dataB-3", s1dataB.get(2));
- assertEquals(3, s1dataC.size());
- assertEquals("id6-s1dataC-1", s1dataC.get(0));
- assertNull(s1dataC.get(1));
- assertEquals("id6-s1dataC-3", s1dataC.get(2));
- }
- }
-
- @Test
- public void testMultiValuedFlatten() throws Exception {
- Map entityAttrs = createMap("name", "e", "url", "testdata.xml",
- XPathEntityProcessor.FOR_EACH, "/root");
- List fields = new ArrayList();
- fields.add(createMap("column", "a", "xpath", "/root/a" ,"flatten","true"));
- Context c = getContext(null,
- new VariableResolver(), getDataSource(testXmlFlatten), Context.FULL_DUMP, fields, entityAttrs);
- XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
- xPathEntityProcessor.init(c);
- Map<String, Object> result = null;
- while (true) {
- Map<String, Object> row = xPathEntityProcessor.nextRow();
- if (row == null)
- break;
- result = row;
- }
- assertEquals("1B2", result.get("a"));
- }
-
- @Test
- public void withFieldsAndXpathStream() throws Exception {
- final Object monitor = new Object();
- final boolean[] done = new boolean[1];
-
- Map entityAttrs = createMap("name", "e", "url", "cd.xml",
- XPathEntityProcessor.FOR_EACH, "/catalog/cd", "stream", "true", "batchSize","1");
- List fields = new ArrayList();
- fields.add(createMap("column", "title", "xpath", "/catalog/cd/title"));
- fields.add(createMap("column", "artist", "xpath", "/catalog/cd/artist"));
- fields.add(createMap("column", "year", "xpath", "/catalog/cd/year"));
- Context c = getContext(null,
- new VariableResolver(), getDataSource(cdData), Context.FULL_DUMP, fields, entityAttrs);
- XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor() {
- private int count;
-
- @Override
- protected Map<String, Object> readRow(Map<String, Object> record,
- String xpath) {
- synchronized (monitor) {
- if (simulateSlowReader && !done[0]) {
- try {
- monitor.wait(100);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- return super.readRow(record, xpath);
- }
- };
-
- if (simulateSlowResultProcessor) {
- xPathEntityProcessor.blockingQueueSize = 1;
- }
- xPathEntityProcessor.blockingQueueTimeOut = 1;
- xPathEntityProcessor.blockingQueueTimeOutUnits = TimeUnit.MICROSECONDS;
-
- xPathEntityProcessor.init(c);
- List<Map<String, Object>> result = new ArrayList<>();
- while (true) {
- if (rowsToRead >= 0 && result.size() >= rowsToRead) {
- Thread.currentThread().interrupt();
- }
- Map<String, Object> row = xPathEntityProcessor.nextRow();
- if (row == null)
- break;
- result.add(row);
- if (simulateSlowResultProcessor) {
- synchronized (xPathEntityProcessor.publisherThread) {
- if (xPathEntityProcessor.publisherThread.isAlive()) {
- xPathEntityProcessor.publisherThread.wait(1000);
- }
- }
- }
- }
-
- synchronized (monitor) {
- done[0] = true;
- monitor.notify();
- }
-
- // confirm that publisher thread stops.
- xPathEntityProcessor.publisherThread.join(1000);
- assertEquals("Expected thread to stop", false, xPathEntityProcessor.publisherThread.isAlive());
-
- assertEquals(rowsToRead < 0 ? 3 : rowsToRead, result.size());
-
- if (rowsToRead < 0) {
- assertEquals("Empire Burlesque", result.get(0).get("title"));
- assertEquals("Bonnie Tyler", result.get(1).get("artist"));
- assertEquals("1982", result.get(2).get("year"));
- }
- }
-
- @Test
- public void withFieldsAndXpathStreamContinuesOnTimeout() throws Exception {
- simulateSlowReader = true;
- withFieldsAndXpathStream();
- }
-
- @Test
- public void streamWritesMessageAfterBlockedAttempt() throws Exception {
- simulateSlowResultProcessor = true;
- withFieldsAndXpathStream();
- }
-
- @Test
- public void streamStopsAfterInterrupt() throws Exception {
- simulateSlowResultProcessor = true;
- rowsToRead = 1;
- withFieldsAndXpathStream();
- }
-
- @Test
- public void withDefaultSolrAndXsl() throws Exception {
- File tmpdir = createTempDir().toFile();
- AbstractDataImportHandlerTestCase.createFile(tmpdir, "x.xsl", xsl.getBytes(StandardCharsets.UTF_8),
- false);
-
- Map entityAttrs = createMap("name", "e",
- XPathEntityProcessor.USE_SOLR_ADD_SCHEMA, "true", "xsl", ""
- + new File(tmpdir, "x.xsl").toURI(), "url", "cd.xml");
- Context c = getContext(null,
- new VariableResolver(), getDataSource(cdData), Context.FULL_DUMP, null, entityAttrs);
- XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
- xPathEntityProcessor.init(c);
- List<Map<String, Object>> result = new ArrayList<>();
- while (true) {
- Map<String, Object> row = xPathEntityProcessor.nextRow();
- if (row == null)
- break;
- result.add(row);
- }
- assertEquals(3, result.size());
- assertEquals("Empire Burlesque", result.get(0).get("title"));
- assertEquals("Bonnie Tyler", result.get(1).get("artist"));
- assertEquals("1982", result.get(2).get("year"));
- }
-
- private DataSource<Reader> getDataSource(final String xml) {
- return new DataSource<Reader>() {
-
- @Override
- public void init(Context context, Properties initProps) {
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public Reader getData(String query) {
- return new StringReader(xml);
- }
- };
- }
-
- private static final String xsl = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
- + "<xsl:stylesheet version=\"1.0\"\n"
- + "xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\">\n"
- + "<xsl:output version='1.0' method='xml' encoding='UTF-8' indent='yes'/>\n"
- + "\n"
- + "<xsl:template match=\"/\">\n"
- + " <add> \n"
- + " <xsl:for-each select=\"catalog/cd\">\n"
- + " <doc>\n"
- + " <field name=\"title\"><xsl:value-of select=\"title\"/></field>\n"
- + " <field name=\"artist\"><xsl:value-of select=\"artist\"/></field>\n"
- + " <field name=\"country\"><xsl:value-of select=\"country\"/></field>\n"
- + " <field name=\"company\"><xsl:value-of select=\"company\"/></field> \n"
- + " <field name=\"price\"><xsl:value-of select=\"price\"/></field>\n"
- + " <field name=\"year\"><xsl:value-of select=\"year\"/></field> \n"
- + " </doc>\n"
- + " </xsl:for-each>\n"
- + " </add> \n"
- + "</xsl:template>\n" + "</xsl:stylesheet>";
-
- private static final String cdData = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
- + "<?xml-stylesheet type=\"text/xsl\" href=\"solr.xsl\"?>\n"
- + "<catalog>\n"
- + "\t<cd>\n"
- + "\t\t<title>Empire Burlesque</title>\n"
- + "\t\t<artist>Bob Dylan</artist>\n"
- + "\t\t<country>USA</country>\n"
- + "\t\t<company>Columbia</company>\n"
- + "\t\t<price>10.90</price>\n"
- + "\t\t<year>1985</year>\n"
- + "\t</cd>\n"
- + "\t<cd>\n"
- + "\t\t<title>Hide your heart</title>\n"
- + "\t\t<artist>Bonnie Tyler</artist>\n"
- + "\t\t<country>UK</country>\n"
- + "\t\t<company>CBS Records</company>\n"
- + "\t\t<price>9.90</price>\n"
- + "\t\t<year>1988</year>\n"
- + "\t</cd>\n"
- + "\t<cd>\n"
- + "\t\t<title>Greatest Hits</title>\n"
- + "\t\t<artist>Dolly Parton</artist>\n"
- + "\t\t<country>USA</country>\n"
- + "\t\t<company>RCA</company>\n"
- + "\t\t<price>9.90</price>\n"
- + "\t\t<year>1982</year>\n" + "\t</cd>\n" + "</catalog>\t";
-
- private static final String testXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE root [\n<!ENTITY uuml \"ü\" >\n]>\n<root><a>1</a><a>2</a><a>ü</a></root>";
-
- private static final String testXmlFlatten = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><root><a>1<b>B</b>2</a></root>";
-
- private static final String textMultipleDocuments =
- "<?xml version=\"1.0\" ?>" +
- "<documents>" +
- " <doc>" +
- " <id>1</id>" +
- " <a>id1-a1</a>" +
- " <a>id1-a2</a>" +
- " <sec1>" +
- " <s1dataA>id1-s1dataA-1</s1dataA>" +
- " <s1dataB>id1-s1dataB-1</s1dataB>" +
- " </sec1>" +
- " <sec1>" +
- " <s1dataB>id1-s1dataB-2</s1dataB>" +
- " </sec1>" +
- " <sec1>" +
- " <s1dataA>id1-s1dataA-3</s1dataA>" +
- " <s1dataB>id1-s1dataB-3</s1dataB>" +
- " </sec1>" +
- " </doc>" +
- " <doc>" +
- " <id>2</id>" +
- " <sec1>" +
- " <s1dataB>id2-s1dataB-1</s1dataB>" +
- " </sec1>" +
- " </doc>" +
- " <doc>" +
- " <id>3</id>" +
- " <sec1>" +
- " <s1dataA>id3-s1dataA-1</s1dataA>" +
- " </sec1>" +
- " </doc>" +
- " <doc>" +
- " <id>4</id>" +
- " <sec1>" +
- " <s1dataA>id4-s1dataA-1</s1dataA>" +
- " <s1dataB>id4-s1dataB-1</s1dataB>" +
- " <s1dataC>id4-s1dataC-1</s1dataC>" +
- " </sec1>" +
- " </doc>" +
- " <doc>" +
- " <id>5</id>" +
- " <sec1>" +
- " <s1dataC>id5-s1dataC-1</s1dataC>" +
- " </sec1>" +
- " </doc>" +
- " <doc>" +
- " <id>6</id>" +
- " <sec1>" +
- " <s1dataA>id6-s1dataA-1</s1dataA>" +
- " <s1dataB>id6-s1dataB-1</s1dataB>" +
- " <s1dataC>id6-s1dataC-1</s1dataC>" +
- " </sec1>" +
- " <sec1>" +
- " <s1dataA>id6-s1dataA-2</s1dataA>" +
- " <s1dataB>id6-s1dataB-2</s1dataB>" +
- " </sec1>" +
- " <sec1>" +
- " <s1dataB>id6-s1dataB-3</s1dataB>" +
- " <s1dataC>id6-s1dataC-3</s1dataC>" +
- " </sec1>" +
- " </doc>" +
- "</documents>"
- ;
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathRecordReader.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathRecordReader.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathRecordReader.java
deleted file mode 100644
index d8e3cbe..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathRecordReader.java
+++ /dev/null
@@ -1,598 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.junit.Test;
-
-/**
- * <p> Test for XPathRecordReader </p>
- *
- *
- * @since solr 1.3
- */
-public class TestXPathRecordReader extends AbstractDataImportHandlerTestCase {
- @Test
- public void testBasic() {
- String xml="<root>\n"
- + " <b><c>Hello C1</c>\n"
- + " <c>Hello C1</c>\n"
- + " </b>\n"
- + " <b><c>Hello C2</c>\n"
- + " </b>\n"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/b");
- rr.addField("c", "/root/b/c", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(2, l.size());
- assertEquals(2, ((List) l.get(0).get("c")).size());
- assertEquals(1, ((List) l.get(1).get("c")).size());
- }
-
- @Test
- public void testAttributes() {
- String xml="<root>\n"
- + " <b a=\"x0\" b=\"y0\" />\n"
- + " <b a=\"x1\" b=\"y1\" />\n"
- + " <b a=\"x2\" b=\"y2\" />\n"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/b");
- rr.addField("a", "/root/b/@a", false);
- rr.addField("b", "/root/b/@b", false);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(3, l.size());
- assertEquals("x0", l.get(0).get("a"));
- assertEquals("x1", l.get(1).get("a"));
- assertEquals("x2", l.get(2).get("a"));
- assertEquals("y0", l.get(0).get("b"));
- assertEquals("y1", l.get(1).get("b"));
- assertEquals("y2", l.get(2).get("b"));
- }
-
- @Test
- public void testAttrInRoot(){
- String xml="<r>\n" +
- "<merchantProduct id=\"814636051\" mid=\"189973\">\n" +
- " <in_stock type=\"stock-4\" />\n" +
- " <condition type=\"cond-0\" />\n" +
- " <price>301.46</price>\n" +
- " </merchantProduct>\n" +
- "<merchantProduct id=\"814636052\" mid=\"189974\">\n" +
- " <in_stock type=\"stock-5\" />\n" +
- " <condition type=\"cond-1\" />\n" +
- " <price>302.46</price>\n" +
- " </merchantProduct>\n" +
- "\n" +
- "</r>";
- XPathRecordReader rr = new XPathRecordReader("/r/merchantProduct");
- rr.addField("id", "/r/merchantProduct/@id", false);
- rr.addField("mid", "/r/merchantProduct/@mid", false);
- rr.addField("price", "/r/merchantProduct/price", false);
- rr.addField("conditionType", "/r/merchantProduct/condition/@type", false);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- Map<String, Object> m = l.get(0);
- assertEquals("814636051", m.get("id"));
- assertEquals("189973", m.get("mid"));
- assertEquals("301.46", m.get("price"));
- assertEquals("cond-0", m.get("conditionType"));
-
- m = l.get(1);
- assertEquals("814636052", m.get("id"));
- assertEquals("189974", m.get("mid"));
- assertEquals("302.46", m.get("price"));
- assertEquals("cond-1", m.get("conditionType"));
- }
-
- @Test
- public void testAttributes2Level() {
- String xml="<root>\n"
- + "<a>\n <b a=\"x0\" b=\"y0\" />\n"
- + " <b a=\"x1\" b=\"y1\" />\n"
- + " <b a=\"x2\" b=\"y2\" />\n"
- + " </a>"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/a/b");
- rr.addField("a", "/root/a/b/@a", false);
- rr.addField("b", "/root/a/b/@b", false);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(3, l.size());
- assertEquals("x0", l.get(0).get("a"));
- assertEquals("y1", l.get(1).get("b"));
- }
-
- @Test
- public void testAttributes2LevelHetero() {
- String xml="<root>\n"
- + "<a>\n <b a=\"x0\" b=\"y0\" />\n"
- + " <b a=\"x1\" b=\"y1\" />\n"
- + " <b a=\"x2\" b=\"y2\" />\n"
- + " </a>"
- + "<x>\n <b a=\"x4\" b=\"y4\" />\n"
- + " <b a=\"x5\" b=\"y5\" />\n"
- + " <b a=\"x6\" b=\"y6\" />\n"
- + " </x>"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/a | /root/x");
- rr.addField("a", "/root/a/b/@a", false);
- rr.addField("b", "/root/a/b/@b", false);
- rr.addField("a", "/root/x/b/@a", false);
- rr.addField("b", "/root/x/b/@b", false);
-
- final List<Map<String, Object>> a = new ArrayList<>();
- final List<Map<String, Object>> x = new ArrayList<>();
- rr.streamRecords(new StringReader(xml), (record, xpath) -> {
- if (record == null) return;
- if (xpath.equals("/root/a")) a.add(record);
- if (xpath.equals("/root/x")) x.add(record);
- });
-
- assertEquals(1, a.size());
- assertEquals(1, x.size());
- }
-
- @Test
- public void testAttributes2LevelMissingAttrVal() {
- String xml="<root>\n"
- + "<a>\n <b a=\"x0\" b=\"y0\" />\n"
- + " <b a=\"x1\" b=\"y1\" />\n"
- + " </a>"
- + "<a>\n <b a=\"x3\" />\n"
- + " <b b=\"y4\" />\n"
- + " </a>"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/a");
- rr.addField("a", "/root/a/b/@a", true);
- rr.addField("b", "/root/a/b/@b", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(2, l.size());
- assertNull(((List) l.get(1).get("a")).get(1));
- assertNull(((List) l.get(1).get("b")).get(0));
- }
-
- @Test
- public void testElems2LevelMissing() {
- String xml="<root>\n"
- + "\t<a>\n"
- + "\t <b>\n\t <x>x0</x>\n"
- + "\t <y>y0</y>\n"
- + "\t </b>\n"
- + "\t <b>\n\t <x>x1</x>\n"
- + "\t <y>y1</y>\n"
- + "\t </b>\n"
- + "\t </a>\n"
- + "\t<a>\n"
- + "\t <b>\n\t <x>x3</x>\n\t </b>\n"
- + "\t <b>\n\t <y>y4</y>\n\t </b>\n"
- + "\t </a>\n"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/a");
- rr.addField("a", "/root/a/b/x", true);
- rr.addField("b", "/root/a/b/y", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(2, l.size());
- assertNull(((List) l.get(1).get("a")).get(1));
- assertNull(((List) l.get(1).get("b")).get(0));
- }
-
- @Test
- public void testElems2LevelEmpty() {
- String xml="<root>\n"
- + "\t<a>\n"
- + "\t <b>\n\t <x>x0</x>\n"
- + "\t <y>y0</y>\n"
- + "\t </b>\n"
- + "\t <b>\n\t <x></x>\n" // empty
- + "\t <y>y1</y>\n"
- + "\t </b>\n"
- + "\t</a>\n"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/a");
- rr.addField("a", "/root/a/b/x", true);
- rr.addField("b", "/root/a/b/y", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(1, l.size());
- assertEquals("x0",((List) l.get(0).get("a")).get(0));
- assertEquals("y0",((List) l.get(0).get("b")).get(0));
- assertEquals("",((List) l.get(0).get("a")).get(1));
- assertEquals("y1",((List) l.get(0).get("b")).get(1));
- }
-
- @Test
- public void testMixedContent() {
- String xml = "<xhtml:p xmlns:xhtml=\"http://xhtml.com/\" >This text is \n" +
- " <xhtml:b>bold</xhtml:b> and this text is \n" +
- " <xhtml:u>underlined</xhtml:u>!\n" +
- "</xhtml:p>";
- XPathRecordReader rr = new XPathRecordReader("/p");
- rr.addField("p", "/p", true);
- rr.addField("b", "/p/b", true);
- rr.addField("u", "/p/u", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- Map<String, Object> row = l.get(0);
-
- assertEquals("bold", ((List) row.get("b")).get(0));
- assertEquals("underlined", ((List) row.get("u")).get(0));
- String p = (String) ((List) row.get("p")).get(0);
- assertTrue(p.contains("This text is"));
- assertTrue(p.contains("and this text is"));
- assertTrue(p.contains("!"));
- // Should not contain content from child elements
- assertFalse(p.contains("bold"));
- }
-
- @Test
- public void testMixedContentFlattened() {
- String xml = "<xhtml:p xmlns:xhtml=\"http://xhtml.com/\" >This text is \n" +
- " <xhtml:b>bold</xhtml:b> and this text is \n" +
- " <xhtml:u>underlined</xhtml:u>!\n" +
- "</xhtml:p>";
- XPathRecordReader rr = new XPathRecordReader("/p");
- rr.addField("p", "/p", false, XPathRecordReader.FLATTEN);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- Map<String, Object> row = l.get(0);
- assertEquals("This text is \n" +
- " bold and this text is \n" +
- " underlined!", ((String)row.get("p")).trim() );
- }
-
- @Test
- public void testElems2LevelWithAttrib() {
- String xml = "<root>\n\t<a>\n\t <b k=\"x\">\n"
- + "\t <x>x0</x>\n"
- + "\t <y></y>\n" // empty
- + "\t </b>\n"
- + "\t <b k=\"y\">\n"
- + "\t <x></x>\n" // empty
- + "\t <y>y1</y>\n"
- + "\t </b>\n"
- + "\t <b k=\"z\">\n"
- + "\t <x>x2</x>\n"
- + "\t <y>y2</y>\n"
- + "\t </b>\n"
- + "\t </a>\n"
- + "\t <a>\n\t <b>\n"
- + "\t <x>x3</x>\n"
- + "\t </b>\n"
- + "\t <b>\n"
- + "\t <y>y4</y>\n"
- + "\t </b>\n"
- + "\t </a>\n"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/a");
- rr.addField("x", "/root/a/b[@k]/x", true);
- rr.addField("y", "/root/a/b[@k]/y", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(2, l.size());
- assertEquals(3, ((List) l.get(0).get("x")).size());
- assertEquals(3, ((List) l.get(0).get("y")).size());
- assertEquals("x0", ((List) l.get(0).get("x")).get(0));
- assertEquals("", ((List) l.get(0).get("y")).get(0));
- assertEquals("", ((List) l.get(0).get("x")).get(1));
- assertEquals("y1", ((List) l.get(0).get("y")).get(1));
- assertEquals("x2", ((List) l.get(0).get("x")).get(2));
- assertEquals("y2", ((List) l.get(0).get("y")).get(2));
- assertEquals(0, l.get(1).size());
- }
-
- @Test
- public void testElems2LevelWithAttribMultiple() {
- String xml="<root>\n"
- + "\t<a>\n\t <b k=\"x\" m=\"n\" >\n"
- + "\t <x>x0</x>\n"
- + "\t <y>y0</y>\n"
- + "\t </b>\n"
- + "\t <b k=\"y\" m=\"p\">\n"
- + "\t <x>x1</x>\n"
- + "\t <y>y1</y>\n"
- + "\t </b>\n"
- + "\t </a>\n"
- + "\t<a>\n\t <b k=\"x\">\n"
- + "\t <x>x3</x>\n"
- + "\t </b>\n"
- + "\t <b m=\"n\">\n"
- + "\t <y>y4</y>\n"
- + "\t </b>\n"
- + "\t </a>\n"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/a");
- rr.addField("x", "/root/a/b[@k][@m='n']/x", true);
- rr.addField("y", "/root/a/b[@k][@m='n']/y", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(2, l.size());
- assertEquals(1, ((List) l.get(0).get("x")).size());
- assertEquals(1, ((List) l.get(0).get("y")).size());
- assertEquals(0, l.get(1).size());
- }
-
- @Test
- public void testElems2LevelWithAttribVal() {
- String xml="<root>\n\t<a>\n <b k=\"x\">\n"
- + "\t <x>x0</x>\n"
- + "\t <y>y0</y>\n"
- + "\t </b>\n"
- + "\t <b k=\"y\">\n"
- + "\t <x>x1</x>\n"
- + "\t <y>y1</y>\n"
- + "\t </b>\n"
- + "\t </a>\n"
- + "\t <a>\n <b><x>x3</x></b>\n"
- + "\t <b><y>y4</y></b>\n"
- + "\t</a>\n" + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/a");
- rr.addField("x", "/root/a/b[@k='x']/x", true);
- rr.addField("y", "/root/a/b[@k='x']/y", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(2, l.size());
- assertEquals(1, ((List) l.get(0).get("x")).size());
- assertEquals(1, ((List) l.get(0).get("y")).size());
- assertEquals(0, l.get(1).size());
- }
-
- @Test
- public void testAttribValWithSlash() {
- String xml = "<root><b>\n" +
- " <a x=\"a/b\" h=\"hello-A\"/> \n" +
- "</b></root>";
- XPathRecordReader rr = new XPathRecordReader("/root/b");
- rr.addField("x", "/root/b/a[@x='a/b']/@h", false);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(1, l.size());
- Map<String, Object> m = l.get(0);
- assertEquals("hello-A", m.get("x"));
- }
-
- @Test
- public void testUnsupported_Xpaths() {
- String xml = "<root><b><a x=\"a/b\" h=\"hello-A\"/> </b></root>";
- XPathRecordReader rr=null;
- try {
- rr = new XPathRecordReader("//b");
- fail("A RuntimeException was expected: //b forEach cannot begin with '//'.");
- }
- catch (RuntimeException ex) { }
- try {
- rr.addField("bold" ,"b", false);
- fail("A RuntimeException was expected: 'b' xpaths must begin with '/'.");
- }
- catch (RuntimeException ex) { }
- }
-
- @Test
- public void testAny_decendent_from_root() {
- XPathRecordReader rr = new XPathRecordReader("/anyd/contenido");
- rr.addField("descdend", "//boo", true);
- rr.addField("inr_descd","//boo/i", false);
- rr.addField("cont", "/anyd/contenido", false);
- rr.addField("id", "/anyd/contenido/@id", false);
- rr.addField("status", "/anyd/status", false);
- rr.addField("title", "/anyd/contenido/titulo", false,XPathRecordReader.FLATTEN);
- rr.addField("resume", "/anyd/contenido/resumen",false);
- rr.addField("text", "/anyd/contenido/texto", false);
-
- String xml="<anyd>\n"
- + " this <boo>top level</boo> is ignored because it is external to the forEach\n"
- + " <status>as is <boo>this element</boo></status>\n"
- + " <contenido id=\"10097\" idioma=\"cat\">\n"
- + " This one is <boo>not ignored as it's</boo> inside a forEach\n"
- + " <antetitulo><i> big <boo>antler</boo></i></antetitulo>\n"
- + " <titulo> My <i>flattened <boo>title</boo></i> </titulo>\n"
- + " <resumen> My summary <i>skip this!</i> </resumen>\n"
- + " <texto> <boo>Within the body of</boo>My text</texto>\n"
- + " <p>Access <boo>inner <i>sub clauses</i> as well</boo></p>\n"
- + " </contenido>\n"
- + "</anyd>";
-
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(1, l.size());
- Map<String, Object> m = l.get(0);
- assertEquals("This one is inside a forEach", m.get("cont").toString().trim());
- assertEquals("10097" ,m.get("id"));
- assertEquals("My flattened title" ,m.get("title").toString().trim());
- assertEquals("My summary" ,m.get("resume").toString().trim());
- assertEquals("My text" ,m.get("text").toString().trim());
- assertEquals("not ignored as it's",(String) ((List) m.get("descdend")).get(0) );
- assertEquals("antler" ,(String) ((List) m.get("descdend")).get(1) );
- assertEquals("Within the body of" ,(String) ((List) m.get("descdend")).get(2) );
- assertEquals("inner as well" ,(String) ((List) m.get("descdend")).get(3) );
- assertEquals("sub clauses" ,m.get("inr_descd").toString().trim());
- }
-
- @Test
- public void testAny_decendent_of_a_child1() {
- XPathRecordReader rr = new XPathRecordReader("/anycd");
- rr.addField("descdend", "/anycd//boo", true);
-
- // same test string as above but checking to see if *all* //boo's are collected
- String xml="<anycd>\n"
- + " this <boo>top level</boo> is ignored because it is external to the forEach\n"
- + " <status>as is <boo>this element</boo></status>\n"
- + " <contenido id=\"10097\" idioma=\"cat\">\n"
- + " This one is <boo>not ignored as it's</boo> inside a forEach\n"
- + " <antetitulo><i> big <boo>antler</boo></i></antetitulo>\n"
- + " <titulo> My <i>flattened <boo>title</boo></i> </titulo>\n"
- + " <resumen> My summary <i>skip this!</i> </resumen>\n"
- + " <texto> <boo>Within the body of</boo>My text</texto>\n"
- + " <p>Access <boo>inner <i>sub clauses</i> as well</boo></p>\n"
- + " </contenido>\n"
- + "</anycd>";
-
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(1, l.size());
- Map<String, Object> m = l.get(0);
- assertEquals("top level" ,(String) ((List) m.get("descdend")).get(0) );
- assertEquals("this element" ,(String) ((List) m.get("descdend")).get(1) );
- assertEquals("not ignored as it's",(String) ((List) m.get("descdend")).get(2) );
- assertEquals("antler" ,(String) ((List) m.get("descdend")).get(3) );
- assertEquals("title" ,(String) ((List) m.get("descdend")).get(4) );
- assertEquals("Within the body of" ,(String) ((List) m.get("descdend")).get(5) );
- assertEquals("inner as well" ,(String) ((List) m.get("descdend")).get(6) );
- }
-
- @Test
- public void testAny_decendent_of_a_child2() {
- XPathRecordReader rr = new XPathRecordReader("/anycd");
- rr.addField("descdend", "/anycd/contenido//boo", true);
-
- // same test string as above but checking to see if *some* //boo's are collected
- String xml="<anycd>\n"
- + " this <boo>top level</boo> is ignored because it is external to the forEach\n"
- + " <status>as is <boo>this element</boo></status>\n"
- + " <contenido id=\"10097\" idioma=\"cat\">\n"
- + " This one is <boo>not ignored as it's</boo> inside a forEach\n"
- + " <antetitulo><i> big <boo>antler</boo></i></antetitulo>\n"
- + " <titulo> My <i>flattened <boo>title</boo></i> </titulo>\n"
- + " <resumen> My summary <i>skip this!</i> </resumen>\n"
- + " <texto> <boo>Within the body of</boo>My text</texto>\n"
- + " <p>Access <boo>inner <i>sub clauses</i> as well</boo></p>\n"
- + " </contenido>\n"
- + "</anycd>";
-
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(1, l.size());
- Map<String, Object> m = l.get(0);
- assertEquals("not ignored as it's",((List) m.get("descdend")).get(0) );
- assertEquals("antler" ,((List) m.get("descdend")).get(1) );
- assertEquals("title" ,((List) m.get("descdend")).get(2) );
- assertEquals("Within the body of" ,((List) m.get("descdend")).get(3) );
- assertEquals("inner as well" ,((List) m.get("descdend")).get(4) );
- }
-
- @Test
- public void testAnother() {
- String xml="<root>\n"
- + " <contenido id=\"10097\" idioma=\"cat\">\n"
- + " <antetitulo></antetitulo>\n"
- + " <titulo> This is my title </titulo>\n"
- + " <resumen> This is my summary </resumen>\n"
- + " <texto> This is the body of my text </texto>\n"
- + " </contenido>\n"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/contenido");
- rr.addField("id", "/root/contenido/@id", false);
- rr.addField("title", "/root/contenido/titulo", false);
- rr.addField("resume","/root/contenido/resumen",false);
- rr.addField("text", "/root/contenido/texto", false);
-
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(1, l.size());
- Map<String, Object> m = l.get(0);
- assertEquals("10097", m.get("id"));
- assertEquals("This is my title", m.get("title").toString().trim());
- assertEquals("This is my summary", m.get("resume").toString().trim());
- assertEquals("This is the body of my text", m.get("text").toString()
- .trim());
- }
-
- @Test
- public void testSameForEachAndXpath(){
- String xml="<root>\n" +
- " <cat>\n" +
- " <name>hello</name>\n" +
- " </cat>\n" +
- " <item name=\"item name\"/>\n" +
- "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/cat/name");
- rr.addField("catName", "/root/cat/name",false);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals("hello",l.get(0).get("catName"));
- }
-
- @Test
- public void testPutNullTest(){
- String xml = "<root>\n" +
- " <i>\n" +
- " <x>\n" +
- " <a>A.1.1</a>\n" +
- " <b>B.1.1</b>\n" +
- " </x>\n" +
- " <x>\n" +
- " <b>B.1.2</b>\n" +
- " <c>C.1.2</c>\n" +
- " </x>\n" +
- " </i>\n" +
- " <i>\n" +
- " <x>\n" +
- " <a>A.2.1</a>\n" +
- " <c>C.2.1</c>\n" +
- " </x>\n" +
- " <x>\n" +
- " <b>B.2.2</b>\n" +
- " <c>C.2.2</c>\n" +
- " </x>\n" +
- " </i>\n" +
- "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/i");
- rr.addField("a", "/root/i/x/a", true);
- rr.addField("b", "/root/i/x/b", true);
- rr.addField("c", "/root/i/x/c", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- Map<String, Object> map = l.get(0);
- List<String> a = (List<String>) map.get("a");
- List<String> b = (List<String>) map.get("b");
- List<String> c = (List<String>) map.get("c");
-
- assertEquals("A.1.1",a.get(0));
- assertEquals("B.1.1",b.get(0));
- assertNull(c.get(0));
-
- assertNull(a.get(1));
- assertEquals("B.1.2",b.get(1));
- assertEquals("C.1.2",c.get(1));
-
- map = l.get(1);
- a = (List<String>) map.get("a");
- b = (List<String>) map.get("b");
- c = (List<String>) map.get("c");
- assertEquals("A.2.1",a.get(0));
- assertNull(b.get(0));
- assertEquals("C.2.1",c.get(0));
-
- assertNull(a.get(1));
- assertEquals("B.2.2",b.get(1));
- assertEquals("C.2.2",c.get(1));
- }
-
-
- @Test
- public void testError(){
- String malformedXml = "<root>\n" +
- " <node>\n" +
- " <id>1</id>\n" +
- " <desc>test1</desc>\n" +
- " </node>\n" +
- " <node>\n" +
- " <id>2</id>\n" +
- " <desc>test2</desc>\n" +
- " </node>\n" +
- " <node>\n" +
- " <id/>3</id>\n" + // invalid XML
- " <desc>test3</desc>\n" +
- " </node>\n" +
- "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/node");
- rr.addField("id", "/root/node/id", true);
- rr.addField("desc", "/root/node/desc", true);
- try {
- rr.getAllRecords(new StringReader(malformedXml));
- fail("A RuntimeException was expected: the input XML is invalid.");
- } catch (Exception e) { }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
deleted file mode 100644
index c8727d0..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.lang.invoke.MethodHandles;
-
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-import org.apache.solr.cloud.AbstractZkTestCase;
-import org.apache.solr.cloud.ZkTestServer;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.SuppressForbidden;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.request.LocalSolrQueryRequest;
-import org.apache.solr.request.SolrQueryRequest;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestZKPropertiesWriter extends AbstractDataImportHandlerTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- protected static ZkTestServer zkServer;
-
- protected static String zkDir;
-
- private static CoreContainer cc;
-
- private String dateFormat = "yyyy-MM-dd HH:mm:ss.SSSSSS";
-
- @BeforeClass
- public static void dihZk_beforeClass() throws Exception {
- zkDir = createTempDir("zkData").toFile().getAbsolutePath();
- zkServer = new ZkTestServer(zkDir);
- zkServer.run();
-
- System.setProperty("solrcloud.skip.autorecovery", "true");
- System.setProperty("zkHost", zkServer.getZkAddress());
- System.setProperty("jetty.port", "0000");
-
- AbstractZkTestCase.buildZooKeeper(zkServer.getZkHost(), zkServer.getZkAddress(), getFile("dih/solr"),
- "dataimport-solrconfig.xml", "dataimport-schema.xml");
-
- //initCore("solrconfig.xml", "schema.xml", getFile("dih/solr").getAbsolutePath());
- cc = createDefaultCoreContainer(getFile("dih/solr").toPath());
- }
-
- @Before
- public void beforeDihZKTest() throws Exception {
-
- }
-
- @After
- public void afterDihZkTest() throws Exception {
- MockDataSource.clearCache();
- }
-
-
- @AfterClass
- public static void dihZk_afterClass() throws Exception {
- cc.shutdown();
-
- zkServer.shutdown();
-
- zkServer = null;
- zkDir = null;
- cc = null;
- }
-
- @SuppressForbidden(reason = "Needs currentTimeMillis to construct date stamps")
- @Test
- public void testZKPropertiesWriter() throws Exception {
- // test using ZooKeeper
- assertTrue("Not using ZooKeeper", h.getCoreContainer().isZooKeeperAware());
-
- // for the really slow/busy computer, we wait to make sure we have a leader before starting
- h.getCoreContainer().getZkController().getZkStateReader().getLeaderUrl("collection1", "shard1", 30000);
-
- assertQ("test query on empty index", request("qlkciyopsbgzyvkylsjhchghjrdf"),
- "//result[@numFound='0']");
-
- SimpleDateFormat errMsgFormat = new SimpleDateFormat(dateFormat, Locale.ROOT);
-
- delQ("*:*");
- commit();
- SimpleDateFormat df = new SimpleDateFormat(dateFormat, Locale.ROOT);
- Date oneSecondAgo = new Date(System.currentTimeMillis() - 1000);
-
- Map<String, String> init = new HashMap<>();
- init.put("dateFormat", dateFormat);
- ZKPropertiesWriter spw = new ZKPropertiesWriter();
- spw.init(new DataImporter(h.getCore(), "dataimport"), init);
- Map<String, Object> props = new HashMap<>();
- props.put("SomeDates.last_index_time", oneSecondAgo);
- props.put("last_index_time", oneSecondAgo);
- spw.persist(props);
-
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "year_s", "2013"));
- MockDataSource.setIterator("select " + df.format(oneSecondAgo) + " from dummy", rows.iterator());
-
- h.query("/dataimport", lrf.makeRequest("command", "full-import", "dataConfig",
- generateConfig(), "clean", "true", "commit", "true", "synchronous",
- "true", "indent", "true"));
- props = spw.readIndexerProperties();
- Date entityDate = df.parse((String) props.get("SomeDates.last_index_time"));
- Date docDate = df.parse((String) props.get("last_index_time"));
-
- Assert.assertTrue("This date: " + errMsgFormat.format(oneSecondAgo) + " should be prior to the document date: " + errMsgFormat.format(docDate), docDate.getTime() - oneSecondAgo.getTime() > 0);
- Assert.assertTrue("This date: " + errMsgFormat.format(oneSecondAgo) + " should be prior to the entity date: " + errMsgFormat.format(entityDate), entityDate.getTime() - oneSecondAgo.getTime() > 0);
- assertQ(request("*:*"), "//*[@numFound='1']", "//doc/str[@name=\"year_s\"]=\"2013\"");
-
- }
-
- public SolrQueryRequest request(String... q) {
- LocalSolrQueryRequest req = lrf.makeRequest(q);
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.add(req.getParams());
- params.set("distrib", true);
- req.setParams(params);
- return req;
- }
-
- protected String generateConfig() {
- StringBuilder sb = new StringBuilder();
- sb.append("<dataConfig> \n");
- sb.append("<propertyWriter dateFormat=\"" + dateFormat + "\" type=\"ZKPropertiesWriter\" />\n");
- sb.append("<dataSource name=\"mock\" type=\"MockDataSource\"/>\n");
- sb.append("<document name=\"TestSimplePropertiesWriter\"> \n");
- sb.append("<entity name=\"SomeDates\" processor=\"SqlEntityProcessor\" dataSource=\"mock\" ");
- sb.append("query=\"select ${dih.last_index_time} from dummy\" >\n");
- sb.append("<field column=\"AYEAR_S\" name=\"year_s\" /> \n");
- sb.append("</entity>\n");
- sb.append("</document> \n");
- sb.append("</dataConfig> \n");
- String config = sb.toString();
- log.debug(config);
- return config;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TripleThreatTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TripleThreatTransformer.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TripleThreatTransformer.java
deleted file mode 100644
index 2d0aadb..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TripleThreatTransformer.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * This transformer does 3 things
- * <ul>
- * <li>It turns every row into 3 rows,
- * modifying any "id" column to ensure duplicate entries in the index
- * <li>The 2nd Row has 2x values for every column,
- * with the added one being backwards of the original
- * <li>The 3rd Row has an added static value
- * </ul>
- *
- * Also, this does not extend Transformer.
- */
-public class TripleThreatTransformer {
- public Object transformRow(Map<String, Object> row) {
- List<Map<String, Object>> rows = new ArrayList<>(3);
- rows.add(row);
- rows.add(addDuplicateBackwardsValues(row));
- rows.add(new LinkedHashMap<>(row));
- rows.get(2).put("AddAColumn_s", "Added");
- modifyIdColumn(rows.get(1), 1);
- modifyIdColumn(rows.get(2), 2);
- return rows;
- }
- private LinkedHashMap<String,Object> addDuplicateBackwardsValues(Map<String, Object> row) {
- LinkedHashMap<String,Object> n = new LinkedHashMap<>();
- for(Map.Entry<String,Object> entry : row.entrySet()) {
- String key = entry.getKey();
- if(!"id".equalsIgnoreCase(key)) {
- String[] vals = new String[2];
- vals[0] = entry.getValue()==null ? "null" : entry.getValue().toString();
- vals[1] = new StringBuilder(vals[0]).reverse().toString();
- n.put(key, Arrays.asList(vals));
- } else {
- n.put(key, entry.getValue());
- }
- }
- return n;
- }
-
- private void modifyIdColumn(Map<String, Object> row, int num) {
- Object o = row.remove("ID");
- if(o==null) {
- o = row.remove("id");
- }
- if(o!=null) {
- String id = o.toString();
- id = "TripleThreat-" + num + "-" + id;
- row.put("id", id);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/contentstream-solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/contentstream-solrconfig.xml b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/contentstream-solrconfig.xml
new file mode 100644
index 0000000..a07ab78
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/contentstream-solrconfig.xml
@@ -0,0 +1,293 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<config>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+ <indexConfig>
+ <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+ </indexConfig>
+
+ <!-- Used to specify an alternate directory to hold all index data
+ other than the default ./data under the Solr home.
+ If replication is in use, this should match the replication configuration. -->
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <!-- the default high-performance update handler -->
+ <updateHandler class="solr.DirectUpdateHandler2">
+
+ <!-- A prefix of "solr." for class names is an alias that
+ causes solr to search appropriate packages, including
+ org.apache.solr.(search|update|request|core|analysis)
+ -->
+
+ <!-- Limit the number of deletions Solr will buffer during doc updating.
+
+ Setting this lower can help bound memory use during indexing.
+ -->
+ <maxPendingDeletes>100000</maxPendingDeletes>
+
+ </updateHandler>
+
+
+ <query>
+ <!-- Maximum number of clauses in a boolean query... can affect
+ range or prefix queries that expand to big boolean
+ queries. An exception is thrown if exceeded. -->
+ <maxBooleanClauses>1024</maxBooleanClauses>
+
+
+ <!-- Cache used by SolrIndexSearcher for filters (DocSets),
+ unordered sets of *all* documents that match a query.
+ When a new searcher is opened, its caches may be prepopulated
+ or "autowarmed" using data from caches in the old searcher.
+ autowarmCount is the number of items to prepopulate. For LRUCache,
+ the autowarmed items will be the most recently accessed items.
+ Parameters:
+ class - the SolrCache implementation (currently only LRUCache)
+ size - the maximum number of entries in the cache
+ initialSize - the initial capacity (number of entries) of
+ the cache. (seel java.util.HashMap)
+ autowarmCount - the number of entries to prepopulate from
+ and old cache.
+ -->
+ <filterCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="256"/>
+
+ <!-- queryResultCache caches results of searches - ordered lists of
+ document ids (DocList) based on a query, a sort, and the range
+ of documents requested. -->
+ <queryResultCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="256"/>
+
+ <!-- documentCache caches Lucene Document objects (the stored fields for each document).
+ Since Lucene internal document ids are transient, this cache will not be autowarmed. -->
+ <documentCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+
+ <!-- If true, stored fields that are not requested will be loaded lazily.
+
+ This can result in a significant speed improvement if the usual case is to
+ not load all stored fields, especially if the skipped fields are large compressed
+ text fields.
+ -->
+ <enableLazyFieldLoading>true</enableLazyFieldLoading>
+
+ <!-- Example of a generic cache. These caches may be accessed by name
+ through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert().
+ The purpose is to enable easy caching of user/application level data.
+ The regenerator argument should be specified as an implementation
+ of solr.search.CacheRegenerator if autowarming is desired. -->
+ <!--
+ <cache name="myUserCache"
+ class="solr.LRUCache"
+ size="4096"
+ initialSize="1024"
+ autowarmCount="1024"
+ regenerator="org.mycompany.mypackage.MyRegenerator"
+ />
+ -->
+
+ <!-- An optimization that attempts to use a filter to satisfy a search.
+ If the requested sort does not include score, then the filterCache
+ will be checked for a filter matching the query. If found, the filter
+ will be used as the source of document ids, and then the sort will be
+ applied to that.
+ <useFilterForSortedQuery>true</useFilterForSortedQuery>
+ -->
+
+ <!-- An optimization for use with the queryResultCache. When a search
+ is requested, a superset of the requested number of document ids
+ are collected. For example, if a search for a particular query
+ requests matching documents 10 through 19, and queryWindowSize is 50,
+ then documents 0 through 49 will be collected and cached. Any further
+ requests in that range can be satisfied via the cache. -->
+ <queryResultWindowSize>50</queryResultWindowSize>
+
+ <!-- Maximum number of documents to cache for any entry in the
+ queryResultCache. -->
+ <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
+
+ <!-- This entry enables an int hash representation for filters (DocSets)
+ when the number of items in the set is less than maxSize. For smaller
+ sets, this representation is more memory efficient, more efficient to
+ iterate over, and faster to take intersections. -->
+ <HashDocSet maxSize="3000" loadFactor="0.75"/>
+
+ <!-- a newSearcher event is fired whenever a new searcher is being prepared
+ and there is a current searcher handling requests (aka registered). -->
+ <!-- QuerySenderListener takes an array of NamedList and executes a
+ local query request for each NamedList in sequence. -->
+ <!--<listener event="newSearcher" class="solr.QuerySenderListener">-->
+ <!--<arr name="queries">-->
+ <!--<lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
+ <!--<lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
+ <!--<lst><str name="q">static newSearcher warming query from solrconfig.xml</str></lst>-->
+ <!--</arr>-->
+ <!--</listener>-->
+
+ <!-- a firstSearcher event is fired whenever a new searcher is being
+ prepared but there is no current registered searcher to handle
+ requests or to gain autowarming data from. -->
+ <!--<listener event="firstSearcher" class="solr.QuerySenderListener">-->
+ <!--<arr name="queries">-->
+ <!--</arr>-->
+ <!--</listener>-->
+
+ <!-- If a search request comes in and there is no current registered searcher,
+ then immediately register the still warming searcher and use it. If
+ "false" then all requests will block until the first searcher is done
+ warming. -->
+ <useColdSearcher>false</useColdSearcher>
+
+ <!-- Maximum number of searchers that may be warming in the background
+ concurrently. An error is returned if this limit is exceeded. Recommend
+ 1-2 for read-only slaves, higher for masters w/o cache warming. -->
+ <maxWarmingSearchers>4</maxWarmingSearchers>
+
+ </query>
+
+ <requestDispatcher>
+ <!--Make sure your system has some authentication before enabling remote streaming!
+ <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="-1" />
+ -->
+
+ <!-- Set HTTP caching related parameters (for proxy caches and clients).
+
+ To get the behaviour of Solr 1.2 (ie: no caching related headers)
+ use the never304="true" option and do not specify a value for
+ <cacheControl>
+ -->
+ <httpCaching never304="true">
+ <!--httpCaching lastModifiedFrom="openTime"
+ etagSeed="Solr"-->
+ <!-- lastModFrom="openTime" is the default, the Last-Modified value
+ (and validation against If-Modified-Since requests) will all be
+ relative to when the current Searcher was opened.
+ You can change it to lastModFrom="dirLastMod" if you want the
+ value to exactly corrispond to when the physical index was last
+ modified.
+
+ etagSeed="..." is an option you can change to force the ETag
+ header (and validation against If-None-Match requests) to be
+ differnet even if the index has not changed (ie: when making
+ significant changes to your config file)
+
+ lastModifiedFrom and etagSeed are both ignored if you use the
+ never304="true" option.
+ -->
+ <!-- If you include a <cacheControl> directive, it will be used to
+ generate a Cache-Control header, as well as an Expires header
+ if the value contains "max-age="
+
+ By default, no Cache-Control header is generated.
+
+ You can use the <cacheControl> option even if you have set
+ never304="true"
+ -->
+ <!-- <cacheControl>max-age=30, public</cacheControl> -->
+ </httpCaching>
+ </requestDispatcher>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <!-- default values for query parameters -->
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <!--
+ <int name="rows">10</int>
+ <str name="fl">*</str>
+ <str name="version">2.1</str>
+ -->
+ </lst>
+ </requestHandler>
+
+ <requestHandler name="/dataimport" class="org.apache.solr.handler.dataimport.DataImportHandler">
+ <lst name="defaults">
+ <str name="config">data-config.xml</str>
+
+ </lst>
+ </requestHandler>
+
+ <!--
+
+ Search components are registered to SolrCore and used by Search Handlers
+
+ By default, the following components are avaliable:
+
+ <searchComponent name="query" class="org.apache.solr.handler.component.QueryComponent" />
+ <searchComponent name="facet" class="org.apache.solr.handler.component.FacetComponent" />
+ <searchComponent name="mlt" class="org.apache.solr.handler.component.MoreLikeThisComponent" />
+ <searchComponent name="highlight" class="org.apache.solr.handler.component.HighlightComponent" />
+ <searchComponent name="debug" class="org.apache.solr.handler.component.DebugComponent" />
+
+ If you register a searchComponent to one of the standard names, that will be used instead.
+
+ -->
+
+ <requestHandler name="/search" class="org.apache.solr.handler.component.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ </lst>
+ <!--
+ By default, this will register the following components:
+
+ <arr name="components">
+ <str>query</str>
+ <str>facet</str>
+ <str>mlt</str>
+ <str>highlight</str>
+ <str>debug</str>
+ </arr>
+
+ To insert handlers before or after the 'standard' components, use:
+
+ <arr name="first-components">
+ <str>first</str>
+ </arr>
+
+ <arr name="last-components">
+ <str>last</str>
+ </arr>
+
+ -->
+ </requestHandler>
+
+ <!-- config for the admin interface -->
+ <admin>
+ <defaultQuery>*:*</defaultQuery>
+ </admin>
+
+ <updateRequestProcessorChain key="contentstream" default="true">
+ <processor class="org.apache.solr.handler.dataimport.AbstractDataImportHandlerTestCase$TestUpdateRequestProcessorFactory"/>
+ <processor class="solr.RunUpdateProcessorFactory"/>
+ <processor class="solr.LogUpdateProcessorFactory"/>
+ </updateRequestProcessorChain>
+
+</config>
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-end-to-end.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-end-to-end.xml b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-end-to-end.xml
new file mode 100644
index 0000000..a582112
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-end-to-end.xml
@@ -0,0 +1,41 @@
+<dataConfig>
+ <dataSource name="hsqldb" driver="org.hsqldb.jdbcDriver" url="jdbc:hsqldb:mem:." />
+ <document name="dih_end_to_end">
+ <entity
+ name="People"
+ processor="SqlEntityProcessor"
+ dataSource="hsqldb"
+ query="SELECT ID, NAME, COUNTRY_CODES FROM PEOPLE"
+ transformer="RegexTransformer"
+ >
+ <field column="ID" name="id" />
+ <field column="COUNTRY_CODE" sourceColName="COUNTRY_CODES" splitBy="," />
+
+<!--
+ Instead of using 'cachePk'/'cacheLookup' as done below, we could have done:
+ where="CODE=People.COUNTRY_CODE"
+-->
+ <entity
+ name="Countries"
+ processor="SqlEntityProcessor"
+ dataSource="hsqldb"
+ cacheImpl="SortedMapBackedCache"
+ cacheKey="CODE"
+ cacheLookup="People.COUNTRY_CODE"
+
+ query="SELECT CODE, COUNTRY_NAME FROM COUNTRIES"
+ >
+ <field column="CODE" name="DO_NOT_INDEX" />
+ </entity>
+
+ <entity
+ name="Sports"
+ processor="SqlEntityProcessor"
+ dataSource="hsqldb"
+ query="SELECT PERSON_ID, SPORT_NAME FROM PEOPLE_SPORTS WHERE PERSON_ID=${People.ID}"
+ />
+
+ </entity>
+ </document>
+</dataConfig>
+
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-with-datasource.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-with-datasource.xml b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-with-datasource.xml
new file mode 100644
index 0000000..46a6603
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-with-datasource.xml
@@ -0,0 +1,9 @@
+<dataConfig>
+ <dataSource type="MockDataSource" />
+ <document>
+ <entity name="x" query="select * from x">
+ <field column="id" />
+ <field column="desc" />
+ </entity>
+ </document>
+</dataConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-with-transformer.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-with-transformer.xml b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-with-transformer.xml
new file mode 100644
index 0000000..925e6c2
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-with-transformer.xml
@@ -0,0 +1,10 @@
+<dataConfig>
+ <dataSource type="MockDataSource" />
+ <dataSource name="mockDs" type="TestDocBuilder2$MockDataSource2" />
+ <document>
+ <entity name="x" query="select * from x" transformer="TestDocBuilder2$MockTransformer">
+ <field column="id" />
+ <field column="desc" />
+ </entity>
+ </document>
+</dataConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataconfig-contentstream.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataconfig-contentstream.xml b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataconfig-contentstream.xml
new file mode 100644
index 0000000..7520e74
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataconfig-contentstream.xml
@@ -0,0 +1,10 @@
+<dataConfig>
+ <dataSource type="ContentStreamDataSource" name="c"/>
+ <document>
+ <entity name="b" dataSource="c" processor="XPathEntityProcessor"
+ forEach="/root/b">
+ <field column="desc" xpath="/root/b/c"/>
+ <field column="id" xpath="/root/b/id"/>
+ </entity>
+ </document>
+</dataConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml
new file mode 100644
index 0000000..6754f9e
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml
@@ -0,0 +1,285 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<config>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+ <!-- Used to specify an alternate directory to hold all index data
+ other than the default ./data under the Solr home.
+ If replication is in use, this should match the replication configuration. -->
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <indexConfig>
+ <lockType>single</lockType>
+ <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+ </indexConfig>
+
+ <!-- the default high-performance update handler -->
+ <updateHandler class="solr.DirectUpdateHandler2">
+
+ <!-- A prefix of "solr." for class names is an alias that
+ causes solr to search appropriate packages, including
+ org.apache.solr.(search|update|request|core|analysis)
+ -->
+
+ <!-- Limit the number of deletions Solr will buffer during doc updating.
+
+ Setting this lower can help bound memory use during indexing.
+ -->
+ <maxPendingDeletes>100000</maxPendingDeletes>
+
+ </updateHandler>
+
+
+ <query>
+ <!-- Maximum number of clauses in a boolean query... can affect
+ range or prefix queries that expand to big boolean
+ queries. An exception is thrown if exceeded. -->
+ <maxBooleanClauses>1024</maxBooleanClauses>
+
+
+ <!-- Cache used by SolrIndexSearcher for filters (DocSets),
+ unordered sets of *all* documents that match a query.
+ When a new searcher is opened, its caches may be prepopulated
+ or "autowarmed" using data from caches in the old searcher.
+ autowarmCount is the number of items to prepopulate. For LRUCache,
+ the autowarmed items will be the most recently accessed items.
+ Parameters:
+ class - the SolrCache implementation (currently only LRUCache)
+ size - the maximum number of entries in the cache
+ initialSize - the initial capacity (number of entries) of
+ the cache. (seel java.util.HashMap)
+ autowarmCount - the number of entries to prepopulate from
+ and old cache.
+ -->
+ <filterCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="256"/>
+
+ <!-- queryResultCache caches results of searches - ordered lists of
+ document ids (DocList) based on a query, a sort, and the range
+ of documents requested. -->
+ <queryResultCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="256"/>
+
+ <!-- documentCache caches Lucene Document objects (the stored fields for each document).
+ Since Lucene internal document ids are transient, this cache will not be autowarmed. -->
+ <documentCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+
+ <!-- If true, stored fields that are not requested will be loaded lazily.
+
+ This can result in a significant speed improvement if the usual case is to
+ not load all stored fields, especially if the skipped fields are large compressed
+ text fields.
+ -->
+ <enableLazyFieldLoading>true</enableLazyFieldLoading>
+
+ <!-- Example of a generic cache. These caches may be accessed by name
+ through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert().
+ The purpose is to enable easy caching of user/application level data.
+ The regenerator argument should be specified as an implementation
+ of solr.search.CacheRegenerator if autowarming is desired. -->
+ <!--
+ <cache name="myUserCache"
+ class="solr.LRUCache"
+ size="4096"
+ initialSize="1024"
+ autowarmCount="1024"
+ regenerator="org.mycompany.mypackage.MyRegenerator"
+ />
+ -->
+
+ <!-- An optimization that attempts to use a filter to satisfy a search.
+ If the requested sort does not include score, then the filterCache
+ will be checked for a filter matching the query. If found, the filter
+ will be used as the source of document ids, and then the sort will be
+ applied to that.
+ <useFilterForSortedQuery>true</useFilterForSortedQuery>
+ -->
+
+ <!-- An optimization for use with the queryResultCache. When a search
+ is requested, a superset of the requested number of document ids
+ are collected. For example, if a search for a particular query
+ requests matching documents 10 through 19, and queryWindowSize is 50,
+ then documents 0 through 49 will be collected and cached. Any further
+ requests in that range can be satisfied via the cache. -->
+ <queryResultWindowSize>50</queryResultWindowSize>
+
+ <!-- Maximum number of documents to cache for any entry in the
+ queryResultCache. -->
+ <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
+
+ <!-- This entry enables an int hash representation for filters (DocSets)
+ when the number of items in the set is less than maxSize. For smaller
+ sets, this representation is more memory efficient, more efficient to
+ iterate over, and faster to take intersections. -->
+ <HashDocSet maxSize="3000" loadFactor="0.75"/>
+
+ <!-- a newSearcher event is fired whenever a new searcher is being prepared
+ and there is a current searcher handling requests (aka registered). -->
+ <!-- QuerySenderListener takes an array of NamedList and executes a
+ local query request for each NamedList in sequence. -->
+ <!--<listener event="newSearcher" class="solr.QuerySenderListener">-->
+ <!--<arr name="queries">-->
+ <!--<lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
+ <!--<lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
+ <!--<lst><str name="q">static newSearcher warming query from solrconfig.xml</str></lst>-->
+ <!--</arr>-->
+ <!--</listener>-->
+
+ <!-- a firstSearcher event is fired whenever a new searcher is being
+ prepared but there is no current registered searcher to handle
+ requests or to gain autowarming data from. -->
+ <!--<listener event="firstSearcher" class="solr.QuerySenderListener">-->
+ <!--<arr name="queries">-->
+ <!--</arr>-->
+ <!--</listener>-->
+
+ <!-- If a search request comes in and there is no current registered searcher,
+ then immediately register the still warming searcher and use it. If
+ "false" then all requests will block until the first searcher is done
+ warming. -->
+ <useColdSearcher>false</useColdSearcher>
+
+ <!-- Maximum number of searchers that may be warming in the background
+ concurrently. An error is returned if this limit is exceeded. Recommend
+ 1-2 for read-only slaves, higher for masters w/o cache warming. -->
+ <maxWarmingSearchers>4</maxWarmingSearchers>
+
+ </query>
+
+ <requestDispatcher>
+ <!--Make sure your system has some authentication before enabling remote streaming!
+ <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="-1" />
+ -->
+
+ <!-- Set HTTP caching related parameters (for proxy caches and clients).
+
+ To get the behaviour of Solr 1.2 (ie: no caching related headers)
+ use the never304="true" option and do not specify a value for
+ <cacheControl>
+ -->
+ <httpCaching never304="true">
+ <!--httpCaching lastModifiedFrom="openTime"
+ etagSeed="Solr"-->
+ <!-- lastModFrom="openTime" is the default, the Last-Modified value
+ (and validation against If-Modified-Since requests) will all be
+ relative to when the current Searcher was opened.
+ You can change it to lastModFrom="dirLastMod" if you want the
+ value to exactly corrispond to when the physical index was last
+ modified.
+
+ etagSeed="..." is an option you can change to force the ETag
+ header (and validation against If-None-Match requests) to be
+ differnet even if the index has not changed (ie: when making
+ significant changes to your config file)
+
+ lastModifiedFrom and etagSeed are both ignored if you use the
+ never304="true" option.
+ -->
+ <!-- If you include a <cacheControl> directive, it will be used to
+ generate a Cache-Control header, as well as an Expires header
+ if the value contains "max-age="
+
+ By default, no Cache-Control header is generated.
+
+ You can use the <cacheControl> option even if you have set
+ never304="true"
+ -->
+ <!-- <cacheControl>max-age=30, public</cacheControl> -->
+ </httpCaching>
+ </requestDispatcher>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <!-- default values for query parameters -->
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <!--
+ <int name="rows">10</int>
+ <str name="fl">*</str>
+ <str name="version">2.1</str>
+ -->
+ </lst>
+ </requestHandler>
+
+ <requestHandler name="/dataimport" class="org.apache.solr.handler.dataimport.DataImportHandler">
+ </requestHandler>
+
+ <!--
+
+ Search components are registered to SolrCore and used by Search Handlers
+
+ By default, the following components are avaliable:
+
+ <searchComponent name="query" class="org.apache.solr.handler.component.QueryComponent" />
+ <searchComponent name="facet" class="org.apache.solr.handler.component.FacetComponent" />
+ <searchComponent name="mlt" class="org.apache.solr.handler.component.MoreLikeThisComponent" />
+ <searchComponent name="highlight" class="org.apache.solr.handler.component.HighlightComponent" />
+ <searchComponent name="debug" class="org.apache.solr.handler.component.DebugComponent" />
+
+ If you register a searchComponent to one of the standard names, that will be used instead.
+
+ -->
+
+ <requestHandler name="/search" class="org.apache.solr.handler.component.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ </lst>
+ <!--
+ By default, this will register the following components:
+
+ <arr name="components">
+ <str>query</str>
+ <str>facet</str>
+ <str>mlt</str>
+ <str>highlight</str>
+ <str>debug</str>
+ </arr>
+
+ To insert handlers before or after the 'standard' components, use:
+
+ <arr name="first-components">
+ <str>first</str>
+ </arr>
+
+ <arr name="last-components">
+ <str>last</str>
+ </arr>
+
+ -->
+ </requestHandler>
+
+ <!-- config for the admin interface -->
+ <admin>
+ <defaultQuery>*:*</defaultQuery>
+ </admin>
+
+</config>
+
[07/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java
new file mode 100644
index 0000000..0028564
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+import java.util.TimeZone;
+
+import org.apache.solr.util.DateMathParser;
+import org.junit.Test;
+
+/**
+ * <p>
+ * Test for VariableResolver
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestVariableResolver extends AbstractDataImportHandlerTestCase {
+
+ @Test
+ public void testSimpleNamespace() {
+ VariableResolver vri = new VariableResolver();
+ Map<String,Object> ns = new HashMap<>();
+ ns.put("world", "WORLD");
+ vri.addNamespace("hello", ns);
+ assertEquals("WORLD", vri.resolve("hello.world"));
+ }
+
+ @Test
+ public void testDefaults() {
+ // System.out.println(System.setProperty(TestVariableResolver.class.getName(),"hello"));
+ System.setProperty(TestVariableResolver.class.getName(), "hello");
+ // System.out.println("s.gP()"+
+ // System.getProperty(TestVariableResolver.class.getName()));
+
+ Properties p = new Properties();
+ p.put("hello", "world");
+ VariableResolver vri = new VariableResolver(p);
+ Object val = vri.resolve(TestVariableResolver.class.getName());
+ // System.out.println("val = " + val);
+ assertEquals("hello", val);
+ assertEquals("world", vri.resolve("hello"));
+ }
+
+ @Test
+ public void testNestedNamespace() {
+ VariableResolver vri = new VariableResolver();
+ Map<String,Object> ns = new HashMap<>();
+ ns.put("world", "WORLD");
+ vri.addNamespace("hello", ns);
+ ns = new HashMap<>();
+ ns.put("world1", "WORLD1");
+ vri.addNamespace("hello.my", ns);
+ assertEquals("WORLD1", vri.resolve("hello.my.world1"));
+ }
+
+ @Test
+ public void test3LevelNestedNamespace() {
+ VariableResolver vri = new VariableResolver();
+ Map<String,Object> ns = new HashMap<>();
+ ns.put("world", "WORLD");
+ vri.addNamespace("hello", ns);
+ ns = new HashMap<>();
+ ns.put("world1", "WORLD1");
+ vri.addNamespace("hello.my.new", ns);
+ assertEquals("WORLD1", vri.resolve("hello.my.new.world1"));
+ }
+
+ @Test
+ public void dateNamespaceWithValue() {
+ VariableResolver vri = new VariableResolver();
+ vri.setEvaluators(new DataImporter().getEvaluators(Collections
+ .<Map<String,String>> emptyList()));
+ Map<String,Object> ns = new HashMap<>();
+ Date d = new Date();
+ ns.put("dt", d);
+ vri.addNamespace("A", ns);
+ assertEquals(
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).format(d),
+ vri.replaceTokens("${dataimporter.functions.formatDate(A.dt,'yyyy-MM-dd HH:mm:ss')}"));
+ }
+
+ @Test
+ public void dateNamespaceWithExpr() throws Exception {
+ VariableResolver vri = new VariableResolver();
+ vri.setEvaluators(new DataImporter().getEvaluators(Collections
+ .<Map<String,String>> emptyList()));
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
+ format.setTimeZone(TimeZone.getTimeZone("UTC"));
+ DateMathParser dmp = new DateMathParser(TimeZone.getDefault());
+
+ String s = vri
+ .replaceTokens("${dataimporter.functions.formatDate('NOW/DAY','yyyy-MM-dd HH:mm')}");
+ assertEquals(
+ new SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.ROOT).format(dmp.parseMath("/DAY")),
+ s);
+ }
+
+ @Test
+ public void testDefaultNamespace() {
+ VariableResolver vri = new VariableResolver();
+ Map<String,Object> ns = new HashMap<>();
+ ns.put("world", "WORLD");
+ vri.addNamespace(null, ns);
+ assertEquals("WORLD", vri.resolve("world"));
+ }
+
+ @Test
+ public void testDefaultNamespace1() {
+ VariableResolver vri = new VariableResolver();
+ Map<String,Object> ns = new HashMap<>();
+ ns.put("world", "WORLD");
+ vri.addNamespace(null, ns);
+ assertEquals("WORLD", vri.resolve("world"));
+ }
+
+ @Test
+ public void testFunctionNamespace1() throws Exception {
+ VariableResolver resolver = new VariableResolver();
+ final List<Map<String,String>> l = new ArrayList<>();
+ Map<String,String> m = new HashMap<>();
+ m.put("name", "test");
+ m.put("class", E.class.getName());
+ l.add(m);
+ resolver.setEvaluators(new DataImporter().getEvaluators(l));
+ ContextImpl context = new ContextImpl(null, resolver, null,
+ Context.FULL_DUMP, Collections.EMPTY_MAP, null, null);
+
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
+ format.setTimeZone(TimeZone.getTimeZone("UTC"));
+ DateMathParser dmp = new DateMathParser(TimeZone.getDefault());
+
+ String s = resolver
+ .replaceTokens("${dataimporter.functions.formatDate('NOW/DAY','yyyy-MM-dd HH:mm')}");
+ assertEquals(
+ new SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.ROOT).format(dmp.parseMath("/DAY")),
+ s);
+ assertEquals("Hello World",
+ resolver.replaceTokens("${dataimporter.functions.test('TEST')}"));
+ }
+
+ public static class E extends Evaluator {
+ @Override
+ public String evaluate(String expression, Context context) {
+ return "Hello World";
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
new file mode 100644
index 0000000..8ee6878
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.lang.invoke.MethodHandles;
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import junit.framework.Assert;
+
+import org.apache.solr.request.SolrQueryRequest;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestVariableResolverEndToEnd extends AbstractDIHJdbcTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @Test
+ public void test() throws Exception {
+ h.query("/dataimport", generateRequest());
+ SolrQueryRequest req = null;
+ try {
+ req = req("q", "*:*", "wt", "json", "indent", "true");
+ String response = h.query(req);
+ log.debug(response);
+ response = response.replaceAll("\\s","");
+ Assert.assertTrue(response.contains("\"numFound\":1"));
+ Pattern p = Pattern.compile("[\"]second1_s[\"][:][\"](.*?)[\"]");
+ Matcher m = p.matcher(response);
+ Assert.assertTrue(m.find());
+ String yearStr = m.group(1);
+ Assert.assertTrue(response.contains("\"second1_s\":\"" + yearStr + "\""));
+ Assert.assertTrue(response.contains("\"second2_s\":\"" + yearStr + "\""));
+ Assert.assertTrue(response.contains("\"second3_s\":\"" + yearStr + "\""));
+ Assert.assertTrue(response.contains("\"PORK_s\":\"GRILL\""));
+ Assert.assertTrue(response.contains("\"FISH_s\":\"FRY\""));
+ Assert.assertTrue(response.contains("\"BEEF_CUTS_mult_s\":[\"ROUND\",\"SIRLOIN\"]"));
+ } catch(Exception e) {
+ throw e;
+ } finally {
+ req.close();
+ }
+ }
+
+ @Override
+ protected String generateConfig() {
+ String thirdLocaleParam = random().nextBoolean() ? "" : (", '" + Locale.getDefault().toLanguageTag() + "'");
+ StringBuilder sb = new StringBuilder();
+ sb.append("<dataConfig> \n");
+ sb.append("<dataSource name=\"hsqldb\" driver=\"${dataimporter.request.dots.in.hsqldb.driver}\" url=\"jdbc:hsqldb:mem:.\" /> \n");
+ sb.append("<document name=\"TestEvaluators\"> \n");
+ sb.append("<entity name=\"FIRST\" processor=\"SqlEntityProcessor\" dataSource=\"hsqldb\" ");
+ sb.append(" query=\"" +
+ "select " +
+ " 1 as id, " +
+ " 'SELECT' as SELECT_KEYWORD, " +
+ " {ts '2017-02-18 12:34:56'} as FIRST_TS " +
+ "from DUAL \" >\n");
+ sb.append(" <field column=\"SELECT_KEYWORD\" name=\"select_keyword_s\" /> \n");
+ sb.append(" <entity name=\"SECOND\" processor=\"SqlEntityProcessor\" dataSource=\"hsqldb\" transformer=\"TemplateTransformer\" ");
+ sb.append(" query=\"" +
+ "${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
+ " 1 as SORT, " +
+ " {ts '2017-02-18 12:34:56'} as SECOND_TS, " +
+ " '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam + ")}' as SECOND1_S, " +
+ " 'PORK' AS MEAT, " +
+ " 'GRILL' AS METHOD, " +
+ " 'ROUND' AS CUTS, " +
+ " 'BEEF_CUTS' AS WHATKIND " +
+ "from DUAL " +
+ "WHERE 1=${FIRST.ID} " +
+ "UNION " +
+ "${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
+ " 2 as SORT, " +
+ " {ts '2017-02-18 12:34:56'} as SECOND_TS, " +
+ " '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam + ")}' as SECOND1_S, " +
+ " 'FISH' AS MEAT, " +
+ " 'FRY' AS METHOD, " +
+ " 'SIRLOIN' AS CUTS, " +
+ " 'BEEF_CUTS' AS WHATKIND " +
+ "from DUAL " +
+ "WHERE 1=${FIRST.ID} " +
+ "ORDER BY SORT \"" +
+ ">\n");
+ sb.append(" <field column=\"SECOND_S\" name=\"second_s\" /> \n");
+ sb.append(" <field column=\"SECOND1_S\" name=\"second1_s\" /> \n");
+ sb.append(" <field column=\"second2_s\" template=\"${dataimporter.functions.formatDate(SECOND.SECOND_TS, 'yyyy'" + thirdLocaleParam + ")}\" /> \n");
+ sb.append(" <field column=\"second3_s\" template=\"${dih.functions.formatDate(SECOND.SECOND_TS, 'yyyy'" + thirdLocaleParam + ")}\" /> \n");
+ sb.append(" <field column=\"METHOD\" name=\"${SECOND.MEAT}_s\"/>\n");
+ sb.append(" <field column=\"CUTS\" name=\"${SECOND.WHATKIND}_mult_s\"/>\n");
+ sb.append(" </entity>\n");
+ sb.append("</entity>\n");
+ sb.append("</document> \n");
+ sb.append("</dataConfig> \n");
+ String config = sb.toString();
+ log.info(config);
+ return config;
+ }
+ @Override
+ protected void populateData(Connection conn) throws Exception {
+ Statement s = null;
+ try {
+ s = conn.createStatement();
+ s.executeUpdate("create table dual(dual char(1) not null)");
+ s.executeUpdate("insert into dual values('Y')");
+ conn.commit();
+ } catch (Exception e) {
+ throw e;
+ } finally {
+ try {
+ s.close();
+ } catch (Exception ex) {}
+ try {
+ conn.close();
+ } catch (Exception ex) {}
+ }
+ }
+ @Override
+ protected Database setAllowedDatabases() {
+ return Database.HSQLDB;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestWriterImpl.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestWriterImpl.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestWriterImpl.java
new file mode 100644
index 0000000..e5c2a94
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestWriterImpl.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.*;
+
+/**
+ * <p>
+ * Test for writerImpl paramater (to provide own SolrWriter)
+ * </p>
+ *
+ *
+ * @since solr 4.0
+ */
+public class TestWriterImpl extends AbstractDataImportHandlerTestCase {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("dataimport-nodatasource-solrconfig.xml", "dataimport-schema.xml");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testDataConfigWithDataSource() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ rows.add(createMap("id", "2", "desc", "two"));
+ rows.add(createMap("id", "3", "desc", "break"));
+ rows.add(createMap("id", "4", "desc", "four"));
+
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ Map extraParams = createMap("writerImpl", TestSolrWriter.class.getName(),
+ "commit", "true");
+ runFullImport(loadDataConfig("data-config-with-datasource.xml"),
+ extraParams);
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='1']");
+ assertQ(req("id:3"), "//*[@numFound='0']");
+ assertQ(req("id:4"), "//*[@numFound='1']");
+ }
+
+ public static class TestSolrWriter extends SolrWriter {
+
+ public TestSolrWriter(UpdateRequestProcessor processor, SolrQueryRequest req) {
+ super(processor, req);
+ }
+
+ @Override
+ public boolean upload(SolrInputDocument doc) {
+ if (doc.getField("desc").getFirstValue().equals("break")) {
+ return false;
+ }
+ return super.upload(doc);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
new file mode 100644
index 0000000..72da77a
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
@@ -0,0 +1,491 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.File;
+import java.io.Reader;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Test;
+
+/**
+ * <p>
+ * Test for XPathEntityProcessor
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestXPathEntityProcessor extends AbstractDataImportHandlerTestCase {
+ boolean simulateSlowReader;
+ boolean simulateSlowResultProcessor;
+ int rowsToRead = -1;
+
+ @Test
+ public void withFieldsAndXpath() throws Exception {
+ File tmpdir = createTempDir().toFile();
+
+ createFile(tmpdir, "x.xsl", xsl.getBytes(StandardCharsets.UTF_8), false);
+ Map entityAttrs = createMap("name", "e", "url", "cd.xml",
+ XPathEntityProcessor.FOR_EACH, "/catalog/cd");
+ List fields = new ArrayList();
+ fields.add(createMap("column", "title", "xpath", "/catalog/cd/title"));
+ fields.add(createMap("column", "artist", "xpath", "/catalog/cd/artist"));
+ fields.add(createMap("column", "year", "xpath", "/catalog/cd/year"));
+ Context c = getContext(null,
+ new VariableResolver(), getDataSource(cdData), Context.FULL_DUMP, fields, entityAttrs);
+ XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
+ xPathEntityProcessor.init(c);
+ List<Map<String, Object>> result = new ArrayList<>();
+ while (true) {
+ Map<String, Object> row = xPathEntityProcessor.nextRow();
+ if (row == null)
+ break;
+ result.add(row);
+ }
+ assertEquals(3, result.size());
+ assertEquals("Empire Burlesque", result.get(0).get("title"));
+ assertEquals("Bonnie Tyler", result.get(1).get("artist"));
+ assertEquals("1982", result.get(2).get("year"));
+ }
+
+ @Test
+ public void testMultiValued() throws Exception {
+ Map entityAttrs = createMap("name", "e", "url", "testdata.xml",
+ XPathEntityProcessor.FOR_EACH, "/root");
+ List fields = new ArrayList();
+ fields.add(createMap("column", "a", "xpath", "/root/a", DataImporter.MULTI_VALUED, "true"));
+ Context c = getContext(null,
+ new VariableResolver(), getDataSource(testXml), Context.FULL_DUMP, fields, entityAttrs);
+ XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
+ xPathEntityProcessor.init(c);
+ List<Map<String, Object>> result = new ArrayList<>();
+ while (true) {
+ Map<String, Object> row = xPathEntityProcessor.nextRow();
+ if (row == null)
+ break;
+ result.add(row);
+ }
+ List l = (List)result.get(0).get("a");
+ assertEquals(3, l.size());
+ assertEquals("1", l.get(0));
+ assertEquals("2", l.get(1));
+ assertEquals("ü", l.get(2));
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ @Test
+ public void testMultiValuedWithMultipleDocuments() throws Exception {
+ Map entityAttrs = createMap("name", "e", "url", "testdata.xml", XPathEntityProcessor.FOR_EACH, "/documents/doc");
+ List fields = new ArrayList();
+ fields.add(createMap("column", "id", "xpath", "/documents/doc/id", DataImporter.MULTI_VALUED, "false"));
+ fields.add(createMap("column", "a", "xpath", "/documents/doc/a", DataImporter.MULTI_VALUED, "true"));
+ fields.add(createMap("column", "s1dataA", "xpath", "/documents/doc/sec1/s1dataA", DataImporter.MULTI_VALUED, "true"));
+ fields.add(createMap("column", "s1dataB", "xpath", "/documents/doc/sec1/s1dataB", DataImporter.MULTI_VALUED, "true"));
+ fields.add(createMap("column", "s1dataC", "xpath", "/documents/doc/sec1/s1dataC", DataImporter.MULTI_VALUED, "true"));
+
+ Context c = getContext(null,
+ new VariableResolver(), getDataSource(textMultipleDocuments), Context.FULL_DUMP, fields, entityAttrs);
+ XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
+ xPathEntityProcessor.init(c);
+ List<Map<String, Object>> result = new ArrayList<>();
+ while (true) {
+ Map<String, Object> row = xPathEntityProcessor.nextRow();
+ if (row == null)
+ break;
+ result.add(row);
+ }
+ {
+ assertEquals("1", result.get(0).get("id"));
+ List a = (List)result.get(0).get("a");
+ List s1dataA = (List)result.get(0).get("s1dataA");
+ List s1dataB = (List)result.get(0).get("s1dataB");
+ List s1dataC = (List)result.get(0).get("s1dataC");
+ assertEquals(2, a.size());
+ assertEquals("id1-a1", a.get(0));
+ assertEquals("id1-a2", a.get(1));
+ assertEquals(3, s1dataA.size());
+ assertEquals("id1-s1dataA-1", s1dataA.get(0));
+ assertNull(s1dataA.get(1));
+ assertEquals("id1-s1dataA-3", s1dataA.get(2));
+ assertEquals(3, s1dataB.size());
+ assertEquals("id1-s1dataB-1", s1dataB.get(0));
+ assertEquals("id1-s1dataB-2", s1dataB.get(1));
+ assertEquals("id1-s1dataB-3", s1dataB.get(2));
+ assertEquals(3, s1dataC.size());
+ assertNull(s1dataC.get(0));
+ assertNull(s1dataC.get(1));
+ assertNull(s1dataC.get(2));
+ }
+ {
+ assertEquals("2", result.get(1).get("id"));
+ List a = (List)result.get(1).get("a");
+ List s1dataA = (List)result.get(1).get("s1dataA");
+ List s1dataB = (List)result.get(1).get("s1dataB");
+ List s1dataC = (List)result.get(1).get("s1dataC");
+ assertTrue(a==null || a.size()==0);
+ assertEquals(1, s1dataA.size());
+ assertNull(s1dataA.get(0));
+ assertEquals(1, s1dataB.size());
+ assertEquals("id2-s1dataB-1", s1dataB.get(0));
+ assertEquals(1, s1dataC.size());
+ assertNull(s1dataC.get(0));
+ }
+ {
+ assertEquals("3", result.get(2).get("id"));
+ List a = (List)result.get(2).get("a");
+ List s1dataA = (List)result.get(2).get("s1dataA");
+ List s1dataB = (List)result.get(2).get("s1dataB");
+ List s1dataC = (List)result.get(2).get("s1dataC");
+ assertTrue(a==null || a.size()==0);
+ assertEquals(1, s1dataA.size());
+ assertEquals("id3-s1dataA-1", s1dataA.get(0));
+ assertEquals(1, s1dataB.size());
+ assertNull(s1dataB.get(0));
+ assertEquals(1, s1dataC.size());
+ assertNull(s1dataC.get(0));
+ }
+ {
+ assertEquals("4", result.get(3).get("id"));
+ List a = (List)result.get(3).get("a");
+ List s1dataA = (List)result.get(3).get("s1dataA");
+ List s1dataB = (List)result.get(3).get("s1dataB");
+ List s1dataC = (List)result.get(3).get("s1dataC");
+ assertTrue(a==null || a.size()==0);
+ assertEquals(1, s1dataA.size());
+ assertEquals("id4-s1dataA-1", s1dataA.get(0));
+ assertEquals(1, s1dataB.size());
+ assertEquals("id4-s1dataB-1", s1dataB.get(0));
+ assertEquals(1, s1dataC.size());
+ assertEquals("id4-s1dataC-1", s1dataC.get(0));
+ }
+ {
+ assertEquals("5", result.get(4).get("id"));
+ List a = (List)result.get(4).get("a");
+ List s1dataA = (List)result.get(4).get("s1dataA");
+ List s1dataB = (List)result.get(4).get("s1dataB");
+ List s1dataC = (List)result.get(4).get("s1dataC");
+ assertTrue(a==null || a.size()==0);
+ assertEquals(1, s1dataA.size());
+ assertNull(s1dataA.get(0));
+ assertEquals(1, s1dataB.size());
+ assertNull(s1dataB.get(0));
+ assertEquals(1, s1dataC.size());
+ assertEquals("id5-s1dataC-1", s1dataC.get(0));
+ }
+ {
+ assertEquals("6", result.get(5).get("id"));
+ List a = (List)result.get(5).get("a");
+ List s1dataA = (List)result.get(5).get("s1dataA");
+ List s1dataB = (List)result.get(5).get("s1dataB");
+ List s1dataC = (List)result.get(5).get("s1dataC");
+ assertTrue(a==null || a.size()==0);
+ assertEquals(3, s1dataA.size());
+ assertEquals("id6-s1dataA-1", s1dataA.get(0));
+ assertEquals("id6-s1dataA-2", s1dataA.get(1));
+ assertNull(s1dataA.get(2));
+ assertEquals(3, s1dataB.size());
+ assertEquals("id6-s1dataB-1", s1dataB.get(0));
+ assertEquals("id6-s1dataB-2", s1dataB.get(1));
+ assertEquals("id6-s1dataB-3", s1dataB.get(2));
+ assertEquals(3, s1dataC.size());
+ assertEquals("id6-s1dataC-1", s1dataC.get(0));
+ assertNull(s1dataC.get(1));
+ assertEquals("id6-s1dataC-3", s1dataC.get(2));
+ }
+ }
+
+ @Test
+ public void testMultiValuedFlatten() throws Exception {
+ Map entityAttrs = createMap("name", "e", "url", "testdata.xml",
+ XPathEntityProcessor.FOR_EACH, "/root");
+ List fields = new ArrayList();
+ fields.add(createMap("column", "a", "xpath", "/root/a" ,"flatten","true"));
+ Context c = getContext(null,
+ new VariableResolver(), getDataSource(testXmlFlatten), Context.FULL_DUMP, fields, entityAttrs);
+ XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
+ xPathEntityProcessor.init(c);
+ Map<String, Object> result = null;
+ while (true) {
+ Map<String, Object> row = xPathEntityProcessor.nextRow();
+ if (row == null)
+ break;
+ result = row;
+ }
+ assertEquals("1B2", result.get("a"));
+ }
+
+ @Test
+ public void withFieldsAndXpathStream() throws Exception {
+ final Object monitor = new Object();
+ final boolean[] done = new boolean[1];
+
+ Map entityAttrs = createMap("name", "e", "url", "cd.xml",
+ XPathEntityProcessor.FOR_EACH, "/catalog/cd", "stream", "true", "batchSize","1");
+ List fields = new ArrayList();
+ fields.add(createMap("column", "title", "xpath", "/catalog/cd/title"));
+ fields.add(createMap("column", "artist", "xpath", "/catalog/cd/artist"));
+ fields.add(createMap("column", "year", "xpath", "/catalog/cd/year"));
+ Context c = getContext(null,
+ new VariableResolver(), getDataSource(cdData), Context.FULL_DUMP, fields, entityAttrs);
+ XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor() {
+ private int count;
+
+ @Override
+ protected Map<String, Object> readRow(Map<String, Object> record,
+ String xpath) {
+ synchronized (monitor) {
+ if (simulateSlowReader && !done[0]) {
+ try {
+ monitor.wait(100);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ return super.readRow(record, xpath);
+ }
+ };
+
+ if (simulateSlowResultProcessor) {
+ xPathEntityProcessor.blockingQueueSize = 1;
+ }
+ xPathEntityProcessor.blockingQueueTimeOut = 1;
+ xPathEntityProcessor.blockingQueueTimeOutUnits = TimeUnit.MICROSECONDS;
+
+ xPathEntityProcessor.init(c);
+ List<Map<String, Object>> result = new ArrayList<>();
+ while (true) {
+ if (rowsToRead >= 0 && result.size() >= rowsToRead) {
+ Thread.currentThread().interrupt();
+ }
+ Map<String, Object> row = xPathEntityProcessor.nextRow();
+ if (row == null)
+ break;
+ result.add(row);
+ if (simulateSlowResultProcessor) {
+ synchronized (xPathEntityProcessor.publisherThread) {
+ if (xPathEntityProcessor.publisherThread.isAlive()) {
+ xPathEntityProcessor.publisherThread.wait(1000);
+ }
+ }
+ }
+ }
+
+ synchronized (monitor) {
+ done[0] = true;
+ monitor.notify();
+ }
+
+ // confirm that publisher thread stops.
+ xPathEntityProcessor.publisherThread.join(1000);
+ assertEquals("Expected thread to stop", false, xPathEntityProcessor.publisherThread.isAlive());
+
+ assertEquals(rowsToRead < 0 ? 3 : rowsToRead, result.size());
+
+ if (rowsToRead < 0) {
+ assertEquals("Empire Burlesque", result.get(0).get("title"));
+ assertEquals("Bonnie Tyler", result.get(1).get("artist"));
+ assertEquals("1982", result.get(2).get("year"));
+ }
+ }
+
+ @Test
+ public void withFieldsAndXpathStreamContinuesOnTimeout() throws Exception {
+ simulateSlowReader = true;
+ withFieldsAndXpathStream();
+ }
+
+ @Test
+ public void streamWritesMessageAfterBlockedAttempt() throws Exception {
+ simulateSlowResultProcessor = true;
+ withFieldsAndXpathStream();
+ }
+
+ @Test
+ public void streamStopsAfterInterrupt() throws Exception {
+ simulateSlowResultProcessor = true;
+ rowsToRead = 1;
+ withFieldsAndXpathStream();
+ }
+
+ @Test
+ public void withDefaultSolrAndXsl() throws Exception {
+ File tmpdir = createTempDir().toFile();
+ AbstractDataImportHandlerTestCase.createFile(tmpdir, "x.xsl", xsl.getBytes(StandardCharsets.UTF_8),
+ false);
+
+ Map entityAttrs = createMap("name", "e",
+ XPathEntityProcessor.USE_SOLR_ADD_SCHEMA, "true", "xsl", ""
+ + new File(tmpdir, "x.xsl").toURI(), "url", "cd.xml");
+ Context c = getContext(null,
+ new VariableResolver(), getDataSource(cdData), Context.FULL_DUMP, null, entityAttrs);
+ XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
+ xPathEntityProcessor.init(c);
+ List<Map<String, Object>> result = new ArrayList<>();
+ while (true) {
+ Map<String, Object> row = xPathEntityProcessor.nextRow();
+ if (row == null)
+ break;
+ result.add(row);
+ }
+ assertEquals(3, result.size());
+ assertEquals("Empire Burlesque", result.get(0).get("title"));
+ assertEquals("Bonnie Tyler", result.get(1).get("artist"));
+ assertEquals("1982", result.get(2).get("year"));
+ }
+
+ private DataSource<Reader> getDataSource(final String xml) {
+ return new DataSource<Reader>() {
+
+ @Override
+ public void init(Context context, Properties initProps) {
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public Reader getData(String query) {
+ return new StringReader(xml);
+ }
+ };
+ }
+
+ private static final String xsl = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ + "<xsl:stylesheet version=\"1.0\"\n"
+ + "xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\">\n"
+ + "<xsl:output version='1.0' method='xml' encoding='UTF-8' indent='yes'/>\n"
+ + "\n"
+ + "<xsl:template match=\"/\">\n"
+ + " <add> \n"
+ + " <xsl:for-each select=\"catalog/cd\">\n"
+ + " <doc>\n"
+ + " <field name=\"title\"><xsl:value-of select=\"title\"/></field>\n"
+ + " <field name=\"artist\"><xsl:value-of select=\"artist\"/></field>\n"
+ + " <field name=\"country\"><xsl:value-of select=\"country\"/></field>\n"
+ + " <field name=\"company\"><xsl:value-of select=\"company\"/></field> \n"
+ + " <field name=\"price\"><xsl:value-of select=\"price\"/></field>\n"
+ + " <field name=\"year\"><xsl:value-of select=\"year\"/></field> \n"
+ + " </doc>\n"
+ + " </xsl:for-each>\n"
+ + " </add> \n"
+ + "</xsl:template>\n" + "</xsl:stylesheet>";
+
+ private static final String cdData = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ + "<?xml-stylesheet type=\"text/xsl\" href=\"solr.xsl\"?>\n"
+ + "<catalog>\n"
+ + "\t<cd>\n"
+ + "\t\t<title>Empire Burlesque</title>\n"
+ + "\t\t<artist>Bob Dylan</artist>\n"
+ + "\t\t<country>USA</country>\n"
+ + "\t\t<company>Columbia</company>\n"
+ + "\t\t<price>10.90</price>\n"
+ + "\t\t<year>1985</year>\n"
+ + "\t</cd>\n"
+ + "\t<cd>\n"
+ + "\t\t<title>Hide your heart</title>\n"
+ + "\t\t<artist>Bonnie Tyler</artist>\n"
+ + "\t\t<country>UK</country>\n"
+ + "\t\t<company>CBS Records</company>\n"
+ + "\t\t<price>9.90</price>\n"
+ + "\t\t<year>1988</year>\n"
+ + "\t</cd>\n"
+ + "\t<cd>\n"
+ + "\t\t<title>Greatest Hits</title>\n"
+ + "\t\t<artist>Dolly Parton</artist>\n"
+ + "\t\t<country>USA</country>\n"
+ + "\t\t<company>RCA</company>\n"
+ + "\t\t<price>9.90</price>\n"
+ + "\t\t<year>1982</year>\n" + "\t</cd>\n" + "</catalog>\t";
+
+ private static final String testXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE root [\n<!ENTITY uuml \"ü\" >\n]>\n<root><a>1</a><a>2</a><a>ü</a></root>";
+
+ private static final String testXmlFlatten = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><root><a>1<b>B</b>2</a></root>";
+
+ private static final String textMultipleDocuments =
+ "<?xml version=\"1.0\" ?>" +
+ "<documents>" +
+ " <doc>" +
+ " <id>1</id>" +
+ " <a>id1-a1</a>" +
+ " <a>id1-a2</a>" +
+ " <sec1>" +
+ " <s1dataA>id1-s1dataA-1</s1dataA>" +
+ " <s1dataB>id1-s1dataB-1</s1dataB>" +
+ " </sec1>" +
+ " <sec1>" +
+ " <s1dataB>id1-s1dataB-2</s1dataB>" +
+ " </sec1>" +
+ " <sec1>" +
+ " <s1dataA>id1-s1dataA-3</s1dataA>" +
+ " <s1dataB>id1-s1dataB-3</s1dataB>" +
+ " </sec1>" +
+ " </doc>" +
+ " <doc>" +
+ " <id>2</id>" +
+ " <sec1>" +
+ " <s1dataB>id2-s1dataB-1</s1dataB>" +
+ " </sec1>" +
+ " </doc>" +
+ " <doc>" +
+ " <id>3</id>" +
+ " <sec1>" +
+ " <s1dataA>id3-s1dataA-1</s1dataA>" +
+ " </sec1>" +
+ " </doc>" +
+ " <doc>" +
+ " <id>4</id>" +
+ " <sec1>" +
+ " <s1dataA>id4-s1dataA-1</s1dataA>" +
+ " <s1dataB>id4-s1dataB-1</s1dataB>" +
+ " <s1dataC>id4-s1dataC-1</s1dataC>" +
+ " </sec1>" +
+ " </doc>" +
+ " <doc>" +
+ " <id>5</id>" +
+ " <sec1>" +
+ " <s1dataC>id5-s1dataC-1</s1dataC>" +
+ " </sec1>" +
+ " </doc>" +
+ " <doc>" +
+ " <id>6</id>" +
+ " <sec1>" +
+ " <s1dataA>id6-s1dataA-1</s1dataA>" +
+ " <s1dataB>id6-s1dataB-1</s1dataB>" +
+ " <s1dataC>id6-s1dataC-1</s1dataC>" +
+ " </sec1>" +
+ " <sec1>" +
+ " <s1dataA>id6-s1dataA-2</s1dataA>" +
+ " <s1dataB>id6-s1dataB-2</s1dataB>" +
+ " </sec1>" +
+ " <sec1>" +
+ " <s1dataB>id6-s1dataB-3</s1dataB>" +
+ " <s1dataC>id6-s1dataC-3</s1dataC>" +
+ " </sec1>" +
+ " </doc>" +
+ "</documents>"
+ ;
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathRecordReader.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathRecordReader.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathRecordReader.java
new file mode 100644
index 0000000..d8e3cbe
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathRecordReader.java
@@ -0,0 +1,598 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Test;
+
+/**
+ * <p> Test for XPathRecordReader </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestXPathRecordReader extends AbstractDataImportHandlerTestCase {
+ @Test
+ public void testBasic() {
+ String xml="<root>\n"
+ + " <b><c>Hello C1</c>\n"
+ + " <c>Hello C1</c>\n"
+ + " </b>\n"
+ + " <b><c>Hello C2</c>\n"
+ + " </b>\n"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/b");
+ rr.addField("c", "/root/b/c", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(2, l.size());
+ assertEquals(2, ((List) l.get(0).get("c")).size());
+ assertEquals(1, ((List) l.get(1).get("c")).size());
+ }
+
+ @Test
+ public void testAttributes() {
+ String xml="<root>\n"
+ + " <b a=\"x0\" b=\"y0\" />\n"
+ + " <b a=\"x1\" b=\"y1\" />\n"
+ + " <b a=\"x2\" b=\"y2\" />\n"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/b");
+ rr.addField("a", "/root/b/@a", false);
+ rr.addField("b", "/root/b/@b", false);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(3, l.size());
+ assertEquals("x0", l.get(0).get("a"));
+ assertEquals("x1", l.get(1).get("a"));
+ assertEquals("x2", l.get(2).get("a"));
+ assertEquals("y0", l.get(0).get("b"));
+ assertEquals("y1", l.get(1).get("b"));
+ assertEquals("y2", l.get(2).get("b"));
+ }
+
+ @Test
+ public void testAttrInRoot(){
+ String xml="<r>\n" +
+ "<merchantProduct id=\"814636051\" mid=\"189973\">\n" +
+ " <in_stock type=\"stock-4\" />\n" +
+ " <condition type=\"cond-0\" />\n" +
+ " <price>301.46</price>\n" +
+ " </merchantProduct>\n" +
+ "<merchantProduct id=\"814636052\" mid=\"189974\">\n" +
+ " <in_stock type=\"stock-5\" />\n" +
+ " <condition type=\"cond-1\" />\n" +
+ " <price>302.46</price>\n" +
+ " </merchantProduct>\n" +
+ "\n" +
+ "</r>";
+ XPathRecordReader rr = new XPathRecordReader("/r/merchantProduct");
+ rr.addField("id", "/r/merchantProduct/@id", false);
+ rr.addField("mid", "/r/merchantProduct/@mid", false);
+ rr.addField("price", "/r/merchantProduct/price", false);
+ rr.addField("conditionType", "/r/merchantProduct/condition/@type", false);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ Map<String, Object> m = l.get(0);
+ assertEquals("814636051", m.get("id"));
+ assertEquals("189973", m.get("mid"));
+ assertEquals("301.46", m.get("price"));
+ assertEquals("cond-0", m.get("conditionType"));
+
+ m = l.get(1);
+ assertEquals("814636052", m.get("id"));
+ assertEquals("189974", m.get("mid"));
+ assertEquals("302.46", m.get("price"));
+ assertEquals("cond-1", m.get("conditionType"));
+ }
+
+ @Test
+ public void testAttributes2Level() {
+ String xml="<root>\n"
+ + "<a>\n <b a=\"x0\" b=\"y0\" />\n"
+ + " <b a=\"x1\" b=\"y1\" />\n"
+ + " <b a=\"x2\" b=\"y2\" />\n"
+ + " </a>"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/a/b");
+ rr.addField("a", "/root/a/b/@a", false);
+ rr.addField("b", "/root/a/b/@b", false);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(3, l.size());
+ assertEquals("x0", l.get(0).get("a"));
+ assertEquals("y1", l.get(1).get("b"));
+ }
+
+ @Test
+ public void testAttributes2LevelHetero() {
+ String xml="<root>\n"
+ + "<a>\n <b a=\"x0\" b=\"y0\" />\n"
+ + " <b a=\"x1\" b=\"y1\" />\n"
+ + " <b a=\"x2\" b=\"y2\" />\n"
+ + " </a>"
+ + "<x>\n <b a=\"x4\" b=\"y4\" />\n"
+ + " <b a=\"x5\" b=\"y5\" />\n"
+ + " <b a=\"x6\" b=\"y6\" />\n"
+ + " </x>"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/a | /root/x");
+ rr.addField("a", "/root/a/b/@a", false);
+ rr.addField("b", "/root/a/b/@b", false);
+ rr.addField("a", "/root/x/b/@a", false);
+ rr.addField("b", "/root/x/b/@b", false);
+
+ final List<Map<String, Object>> a = new ArrayList<>();
+ final List<Map<String, Object>> x = new ArrayList<>();
+ rr.streamRecords(new StringReader(xml), (record, xpath) -> {
+ if (record == null) return;
+ if (xpath.equals("/root/a")) a.add(record);
+ if (xpath.equals("/root/x")) x.add(record);
+ });
+
+ assertEquals(1, a.size());
+ assertEquals(1, x.size());
+ }
+
+ @Test
+ public void testAttributes2LevelMissingAttrVal() {
+ String xml="<root>\n"
+ + "<a>\n <b a=\"x0\" b=\"y0\" />\n"
+ + " <b a=\"x1\" b=\"y1\" />\n"
+ + " </a>"
+ + "<a>\n <b a=\"x3\" />\n"
+ + " <b b=\"y4\" />\n"
+ + " </a>"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/a");
+ rr.addField("a", "/root/a/b/@a", true);
+ rr.addField("b", "/root/a/b/@b", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(2, l.size());
+ assertNull(((List) l.get(1).get("a")).get(1));
+ assertNull(((List) l.get(1).get("b")).get(0));
+ }
+
+ @Test
+ public void testElems2LevelMissing() {
+ String xml="<root>\n"
+ + "\t<a>\n"
+ + "\t <b>\n\t <x>x0</x>\n"
+ + "\t <y>y0</y>\n"
+ + "\t </b>\n"
+ + "\t <b>\n\t <x>x1</x>\n"
+ + "\t <y>y1</y>\n"
+ + "\t </b>\n"
+ + "\t </a>\n"
+ + "\t<a>\n"
+ + "\t <b>\n\t <x>x3</x>\n\t </b>\n"
+ + "\t <b>\n\t <y>y4</y>\n\t </b>\n"
+ + "\t </a>\n"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/a");
+ rr.addField("a", "/root/a/b/x", true);
+ rr.addField("b", "/root/a/b/y", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(2, l.size());
+ assertNull(((List) l.get(1).get("a")).get(1));
+ assertNull(((List) l.get(1).get("b")).get(0));
+ }
+
+ @Test
+ public void testElems2LevelEmpty() {
+ String xml="<root>\n"
+ + "\t<a>\n"
+ + "\t <b>\n\t <x>x0</x>\n"
+ + "\t <y>y0</y>\n"
+ + "\t </b>\n"
+ + "\t <b>\n\t <x></x>\n" // empty
+ + "\t <y>y1</y>\n"
+ + "\t </b>\n"
+ + "\t</a>\n"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/a");
+ rr.addField("a", "/root/a/b/x", true);
+ rr.addField("b", "/root/a/b/y", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(1, l.size());
+ assertEquals("x0",((List) l.get(0).get("a")).get(0));
+ assertEquals("y0",((List) l.get(0).get("b")).get(0));
+ assertEquals("",((List) l.get(0).get("a")).get(1));
+ assertEquals("y1",((List) l.get(0).get("b")).get(1));
+ }
+
+ @Test
+ public void testMixedContent() {
+ String xml = "<xhtml:p xmlns:xhtml=\"http://xhtml.com/\" >This text is \n" +
+ " <xhtml:b>bold</xhtml:b> and this text is \n" +
+ " <xhtml:u>underlined</xhtml:u>!\n" +
+ "</xhtml:p>";
+ XPathRecordReader rr = new XPathRecordReader("/p");
+ rr.addField("p", "/p", true);
+ rr.addField("b", "/p/b", true);
+ rr.addField("u", "/p/u", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ Map<String, Object> row = l.get(0);
+
+ assertEquals("bold", ((List) row.get("b")).get(0));
+ assertEquals("underlined", ((List) row.get("u")).get(0));
+ String p = (String) ((List) row.get("p")).get(0);
+ assertTrue(p.contains("This text is"));
+ assertTrue(p.contains("and this text is"));
+ assertTrue(p.contains("!"));
+ // Should not contain content from child elements
+ assertFalse(p.contains("bold"));
+ }
+
+ @Test
+ public void testMixedContentFlattened() {
+ String xml = "<xhtml:p xmlns:xhtml=\"http://xhtml.com/\" >This text is \n" +
+ " <xhtml:b>bold</xhtml:b> and this text is \n" +
+ " <xhtml:u>underlined</xhtml:u>!\n" +
+ "</xhtml:p>";
+ XPathRecordReader rr = new XPathRecordReader("/p");
+ rr.addField("p", "/p", false, XPathRecordReader.FLATTEN);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ Map<String, Object> row = l.get(0);
+ assertEquals("This text is \n" +
+ " bold and this text is \n" +
+ " underlined!", ((String)row.get("p")).trim() );
+ }
+
+ @Test
+ public void testElems2LevelWithAttrib() {
+ String xml = "<root>\n\t<a>\n\t <b k=\"x\">\n"
+ + "\t <x>x0</x>\n"
+ + "\t <y></y>\n" // empty
+ + "\t </b>\n"
+ + "\t <b k=\"y\">\n"
+ + "\t <x></x>\n" // empty
+ + "\t <y>y1</y>\n"
+ + "\t </b>\n"
+ + "\t <b k=\"z\">\n"
+ + "\t <x>x2</x>\n"
+ + "\t <y>y2</y>\n"
+ + "\t </b>\n"
+ + "\t </a>\n"
+ + "\t <a>\n\t <b>\n"
+ + "\t <x>x3</x>\n"
+ + "\t </b>\n"
+ + "\t <b>\n"
+ + "\t <y>y4</y>\n"
+ + "\t </b>\n"
+ + "\t </a>\n"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/a");
+ rr.addField("x", "/root/a/b[@k]/x", true);
+ rr.addField("y", "/root/a/b[@k]/y", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(2, l.size());
+ assertEquals(3, ((List) l.get(0).get("x")).size());
+ assertEquals(3, ((List) l.get(0).get("y")).size());
+ assertEquals("x0", ((List) l.get(0).get("x")).get(0));
+ assertEquals("", ((List) l.get(0).get("y")).get(0));
+ assertEquals("", ((List) l.get(0).get("x")).get(1));
+ assertEquals("y1", ((List) l.get(0).get("y")).get(1));
+ assertEquals("x2", ((List) l.get(0).get("x")).get(2));
+ assertEquals("y2", ((List) l.get(0).get("y")).get(2));
+ assertEquals(0, l.get(1).size());
+ }
+
+ @Test
+ public void testElems2LevelWithAttribMultiple() {
+ String xml="<root>\n"
+ + "\t<a>\n\t <b k=\"x\" m=\"n\" >\n"
+ + "\t <x>x0</x>\n"
+ + "\t <y>y0</y>\n"
+ + "\t </b>\n"
+ + "\t <b k=\"y\" m=\"p\">\n"
+ + "\t <x>x1</x>\n"
+ + "\t <y>y1</y>\n"
+ + "\t </b>\n"
+ + "\t </a>\n"
+ + "\t<a>\n\t <b k=\"x\">\n"
+ + "\t <x>x3</x>\n"
+ + "\t </b>\n"
+ + "\t <b m=\"n\">\n"
+ + "\t <y>y4</y>\n"
+ + "\t </b>\n"
+ + "\t </a>\n"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/a");
+ rr.addField("x", "/root/a/b[@k][@m='n']/x", true);
+ rr.addField("y", "/root/a/b[@k][@m='n']/y", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(2, l.size());
+ assertEquals(1, ((List) l.get(0).get("x")).size());
+ assertEquals(1, ((List) l.get(0).get("y")).size());
+ assertEquals(0, l.get(1).size());
+ }
+
+ @Test
+ public void testElems2LevelWithAttribVal() {
+ String xml="<root>\n\t<a>\n <b k=\"x\">\n"
+ + "\t <x>x0</x>\n"
+ + "\t <y>y0</y>\n"
+ + "\t </b>\n"
+ + "\t <b k=\"y\">\n"
+ + "\t <x>x1</x>\n"
+ + "\t <y>y1</y>\n"
+ + "\t </b>\n"
+ + "\t </a>\n"
+ + "\t <a>\n <b><x>x3</x></b>\n"
+ + "\t <b><y>y4</y></b>\n"
+ + "\t</a>\n" + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/a");
+ rr.addField("x", "/root/a/b[@k='x']/x", true);
+ rr.addField("y", "/root/a/b[@k='x']/y", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(2, l.size());
+ assertEquals(1, ((List) l.get(0).get("x")).size());
+ assertEquals(1, ((List) l.get(0).get("y")).size());
+ assertEquals(0, l.get(1).size());
+ }
+
+ @Test
+ public void testAttribValWithSlash() {
+ String xml = "<root><b>\n" +
+ " <a x=\"a/b\" h=\"hello-A\"/> \n" +
+ "</b></root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/b");
+ rr.addField("x", "/root/b/a[@x='a/b']/@h", false);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(1, l.size());
+ Map<String, Object> m = l.get(0);
+ assertEquals("hello-A", m.get("x"));
+ }
+
+ @Test
+ public void testUnsupported_Xpaths() {
+ String xml = "<root><b><a x=\"a/b\" h=\"hello-A\"/> </b></root>";
+ XPathRecordReader rr=null;
+ try {
+ rr = new XPathRecordReader("//b");
+ fail("A RuntimeException was expected: //b forEach cannot begin with '//'.");
+ }
+ catch (RuntimeException ex) { }
+ try {
+ rr.addField("bold" ,"b", false);
+ fail("A RuntimeException was expected: 'b' xpaths must begin with '/'.");
+ }
+ catch (RuntimeException ex) { }
+ }
+
+ @Test
+ public void testAny_decendent_from_root() {
+ XPathRecordReader rr = new XPathRecordReader("/anyd/contenido");
+ rr.addField("descdend", "//boo", true);
+ rr.addField("inr_descd","//boo/i", false);
+ rr.addField("cont", "/anyd/contenido", false);
+ rr.addField("id", "/anyd/contenido/@id", false);
+ rr.addField("status", "/anyd/status", false);
+ rr.addField("title", "/anyd/contenido/titulo", false,XPathRecordReader.FLATTEN);
+ rr.addField("resume", "/anyd/contenido/resumen",false);
+ rr.addField("text", "/anyd/contenido/texto", false);
+
+ String xml="<anyd>\n"
+ + " this <boo>top level</boo> is ignored because it is external to the forEach\n"
+ + " <status>as is <boo>this element</boo></status>\n"
+ + " <contenido id=\"10097\" idioma=\"cat\">\n"
+ + " This one is <boo>not ignored as it's</boo> inside a forEach\n"
+ + " <antetitulo><i> big <boo>antler</boo></i></antetitulo>\n"
+ + " <titulo> My <i>flattened <boo>title</boo></i> </titulo>\n"
+ + " <resumen> My summary <i>skip this!</i> </resumen>\n"
+ + " <texto> <boo>Within the body of</boo>My text</texto>\n"
+ + " <p>Access <boo>inner <i>sub clauses</i> as well</boo></p>\n"
+ + " </contenido>\n"
+ + "</anyd>";
+
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(1, l.size());
+ Map<String, Object> m = l.get(0);
+ assertEquals("This one is inside a forEach", m.get("cont").toString().trim());
+ assertEquals("10097" ,m.get("id"));
+ assertEquals("My flattened title" ,m.get("title").toString().trim());
+ assertEquals("My summary" ,m.get("resume").toString().trim());
+ assertEquals("My text" ,m.get("text").toString().trim());
+ assertEquals("not ignored as it's",(String) ((List) m.get("descdend")).get(0) );
+ assertEquals("antler" ,(String) ((List) m.get("descdend")).get(1) );
+ assertEquals("Within the body of" ,(String) ((List) m.get("descdend")).get(2) );
+ assertEquals("inner as well" ,(String) ((List) m.get("descdend")).get(3) );
+ assertEquals("sub clauses" ,m.get("inr_descd").toString().trim());
+ }
+
+ @Test
+ public void testAny_decendent_of_a_child1() {
+ XPathRecordReader rr = new XPathRecordReader("/anycd");
+ rr.addField("descdend", "/anycd//boo", true);
+
+ // same test string as above but checking to see if *all* //boo's are collected
+ String xml="<anycd>\n"
+ + " this <boo>top level</boo> is ignored because it is external to the forEach\n"
+ + " <status>as is <boo>this element</boo></status>\n"
+ + " <contenido id=\"10097\" idioma=\"cat\">\n"
+ + " This one is <boo>not ignored as it's</boo> inside a forEach\n"
+ + " <antetitulo><i> big <boo>antler</boo></i></antetitulo>\n"
+ + " <titulo> My <i>flattened <boo>title</boo></i> </titulo>\n"
+ + " <resumen> My summary <i>skip this!</i> </resumen>\n"
+ + " <texto> <boo>Within the body of</boo>My text</texto>\n"
+ + " <p>Access <boo>inner <i>sub clauses</i> as well</boo></p>\n"
+ + " </contenido>\n"
+ + "</anycd>";
+
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(1, l.size());
+ Map<String, Object> m = l.get(0);
+ assertEquals("top level" ,(String) ((List) m.get("descdend")).get(0) );
+ assertEquals("this element" ,(String) ((List) m.get("descdend")).get(1) );
+ assertEquals("not ignored as it's",(String) ((List) m.get("descdend")).get(2) );
+ assertEquals("antler" ,(String) ((List) m.get("descdend")).get(3) );
+ assertEquals("title" ,(String) ((List) m.get("descdend")).get(4) );
+ assertEquals("Within the body of" ,(String) ((List) m.get("descdend")).get(5) );
+ assertEquals("inner as well" ,(String) ((List) m.get("descdend")).get(6) );
+ }
+
+ @Test
+ public void testAny_decendent_of_a_child2() {
+ XPathRecordReader rr = new XPathRecordReader("/anycd");
+ rr.addField("descdend", "/anycd/contenido//boo", true);
+
+ // same test string as above but checking to see if *some* //boo's are collected
+ String xml="<anycd>\n"
+ + " this <boo>top level</boo> is ignored because it is external to the forEach\n"
+ + " <status>as is <boo>this element</boo></status>\n"
+ + " <contenido id=\"10097\" idioma=\"cat\">\n"
+ + " This one is <boo>not ignored as it's</boo> inside a forEach\n"
+ + " <antetitulo><i> big <boo>antler</boo></i></antetitulo>\n"
+ + " <titulo> My <i>flattened <boo>title</boo></i> </titulo>\n"
+ + " <resumen> My summary <i>skip this!</i> </resumen>\n"
+ + " <texto> <boo>Within the body of</boo>My text</texto>\n"
+ + " <p>Access <boo>inner <i>sub clauses</i> as well</boo></p>\n"
+ + " </contenido>\n"
+ + "</anycd>";
+
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(1, l.size());
+ Map<String, Object> m = l.get(0);
+ assertEquals("not ignored as it's",((List) m.get("descdend")).get(0) );
+ assertEquals("antler" ,((List) m.get("descdend")).get(1) );
+ assertEquals("title" ,((List) m.get("descdend")).get(2) );
+ assertEquals("Within the body of" ,((List) m.get("descdend")).get(3) );
+ assertEquals("inner as well" ,((List) m.get("descdend")).get(4) );
+ }
+
+ @Test
+ public void testAnother() {
+ String xml="<root>\n"
+ + " <contenido id=\"10097\" idioma=\"cat\">\n"
+ + " <antetitulo></antetitulo>\n"
+ + " <titulo> This is my title </titulo>\n"
+ + " <resumen> This is my summary </resumen>\n"
+ + " <texto> This is the body of my text </texto>\n"
+ + " </contenido>\n"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/contenido");
+ rr.addField("id", "/root/contenido/@id", false);
+ rr.addField("title", "/root/contenido/titulo", false);
+ rr.addField("resume","/root/contenido/resumen",false);
+ rr.addField("text", "/root/contenido/texto", false);
+
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(1, l.size());
+ Map<String, Object> m = l.get(0);
+ assertEquals("10097", m.get("id"));
+ assertEquals("This is my title", m.get("title").toString().trim());
+ assertEquals("This is my summary", m.get("resume").toString().trim());
+ assertEquals("This is the body of my text", m.get("text").toString()
+ .trim());
+ }
+
+ @Test
+ public void testSameForEachAndXpath(){
+ String xml="<root>\n" +
+ " <cat>\n" +
+ " <name>hello</name>\n" +
+ " </cat>\n" +
+ " <item name=\"item name\"/>\n" +
+ "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/cat/name");
+ rr.addField("catName", "/root/cat/name",false);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals("hello",l.get(0).get("catName"));
+ }
+
+ @Test
+ public void testPutNullTest(){
+ String xml = "<root>\n" +
+ " <i>\n" +
+ " <x>\n" +
+ " <a>A.1.1</a>\n" +
+ " <b>B.1.1</b>\n" +
+ " </x>\n" +
+ " <x>\n" +
+ " <b>B.1.2</b>\n" +
+ " <c>C.1.2</c>\n" +
+ " </x>\n" +
+ " </i>\n" +
+ " <i>\n" +
+ " <x>\n" +
+ " <a>A.2.1</a>\n" +
+ " <c>C.2.1</c>\n" +
+ " </x>\n" +
+ " <x>\n" +
+ " <b>B.2.2</b>\n" +
+ " <c>C.2.2</c>\n" +
+ " </x>\n" +
+ " </i>\n" +
+ "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/i");
+ rr.addField("a", "/root/i/x/a", true);
+ rr.addField("b", "/root/i/x/b", true);
+ rr.addField("c", "/root/i/x/c", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ Map<String, Object> map = l.get(0);
+ List<String> a = (List<String>) map.get("a");
+ List<String> b = (List<String>) map.get("b");
+ List<String> c = (List<String>) map.get("c");
+
+ assertEquals("A.1.1",a.get(0));
+ assertEquals("B.1.1",b.get(0));
+ assertNull(c.get(0));
+
+ assertNull(a.get(1));
+ assertEquals("B.1.2",b.get(1));
+ assertEquals("C.1.2",c.get(1));
+
+ map = l.get(1);
+ a = (List<String>) map.get("a");
+ b = (List<String>) map.get("b");
+ c = (List<String>) map.get("c");
+ assertEquals("A.2.1",a.get(0));
+ assertNull(b.get(0));
+ assertEquals("C.2.1",c.get(0));
+
+ assertNull(a.get(1));
+ assertEquals("B.2.2",b.get(1));
+ assertEquals("C.2.2",c.get(1));
+ }
+
+
+ @Test
+ public void testError(){
+ String malformedXml = "<root>\n" +
+ " <node>\n" +
+ " <id>1</id>\n" +
+ " <desc>test1</desc>\n" +
+ " </node>\n" +
+ " <node>\n" +
+ " <id>2</id>\n" +
+ " <desc>test2</desc>\n" +
+ " </node>\n" +
+ " <node>\n" +
+ " <id/>3</id>\n" + // invalid XML
+ " <desc>test3</desc>\n" +
+ " </node>\n" +
+ "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/node");
+ rr.addField("id", "/root/node/id", true);
+ rr.addField("desc", "/root/node/desc", true);
+ try {
+ rr.getAllRecords(new StringReader(malformedXml));
+ fail("A RuntimeException was expected: the input XML is invalid.");
+ } catch (Exception e) { }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
new file mode 100644
index 0000000..c8727d0
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.lang.invoke.MethodHandles;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.solr.cloud.AbstractZkTestCase;
+import org.apache.solr.cloud.ZkTestServer;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.SuppressForbidden;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestZKPropertiesWriter extends AbstractDataImportHandlerTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ protected static ZkTestServer zkServer;
+
+ protected static String zkDir;
+
+ private static CoreContainer cc;
+
+ private String dateFormat = "yyyy-MM-dd HH:mm:ss.SSSSSS";
+
+ @BeforeClass
+ public static void dihZk_beforeClass() throws Exception {
+ zkDir = createTempDir("zkData").toFile().getAbsolutePath();
+ zkServer = new ZkTestServer(zkDir);
+ zkServer.run();
+
+ System.setProperty("solrcloud.skip.autorecovery", "true");
+ System.setProperty("zkHost", zkServer.getZkAddress());
+ System.setProperty("jetty.port", "0000");
+
+ AbstractZkTestCase.buildZooKeeper(zkServer.getZkHost(), zkServer.getZkAddress(), getFile("dih/solr"),
+ "dataimport-solrconfig.xml", "dataimport-schema.xml");
+
+ //initCore("solrconfig.xml", "schema.xml", getFile("dih/solr").getAbsolutePath());
+ cc = createDefaultCoreContainer(getFile("dih/solr").toPath());
+ }
+
+ @Before
+ public void beforeDihZKTest() throws Exception {
+
+ }
+
+ @After
+ public void afterDihZkTest() throws Exception {
+ MockDataSource.clearCache();
+ }
+
+
+ @AfterClass
+ public static void dihZk_afterClass() throws Exception {
+ cc.shutdown();
+
+ zkServer.shutdown();
+
+ zkServer = null;
+ zkDir = null;
+ cc = null;
+ }
+
+ @SuppressForbidden(reason = "Needs currentTimeMillis to construct date stamps")
+ @Test
+ public void testZKPropertiesWriter() throws Exception {
+ // test using ZooKeeper
+ assertTrue("Not using ZooKeeper", h.getCoreContainer().isZooKeeperAware());
+
+ // for the really slow/busy computer, we wait to make sure we have a leader before starting
+ h.getCoreContainer().getZkController().getZkStateReader().getLeaderUrl("collection1", "shard1", 30000);
+
+ assertQ("test query on empty index", request("qlkciyopsbgzyvkylsjhchghjrdf"),
+ "//result[@numFound='0']");
+
+ SimpleDateFormat errMsgFormat = new SimpleDateFormat(dateFormat, Locale.ROOT);
+
+ delQ("*:*");
+ commit();
+ SimpleDateFormat df = new SimpleDateFormat(dateFormat, Locale.ROOT);
+ Date oneSecondAgo = new Date(System.currentTimeMillis() - 1000);
+
+ Map<String, String> init = new HashMap<>();
+ init.put("dateFormat", dateFormat);
+ ZKPropertiesWriter spw = new ZKPropertiesWriter();
+ spw.init(new DataImporter(h.getCore(), "dataimport"), init);
+ Map<String, Object> props = new HashMap<>();
+ props.put("SomeDates.last_index_time", oneSecondAgo);
+ props.put("last_index_time", oneSecondAgo);
+ spw.persist(props);
+
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "year_s", "2013"));
+ MockDataSource.setIterator("select " + df.format(oneSecondAgo) + " from dummy", rows.iterator());
+
+ h.query("/dataimport", lrf.makeRequest("command", "full-import", "dataConfig",
+ generateConfig(), "clean", "true", "commit", "true", "synchronous",
+ "true", "indent", "true"));
+ props = spw.readIndexerProperties();
+ Date entityDate = df.parse((String) props.get("SomeDates.last_index_time"));
+ Date docDate = df.parse((String) props.get("last_index_time"));
+
+ Assert.assertTrue("This date: " + errMsgFormat.format(oneSecondAgo) + " should be prior to the document date: " + errMsgFormat.format(docDate), docDate.getTime() - oneSecondAgo.getTime() > 0);
+ Assert.assertTrue("This date: " + errMsgFormat.format(oneSecondAgo) + " should be prior to the entity date: " + errMsgFormat.format(entityDate), entityDate.getTime() - oneSecondAgo.getTime() > 0);
+ assertQ(request("*:*"), "//*[@numFound='1']", "//doc/str[@name=\"year_s\"]=\"2013\"");
+
+ }
+
+ public SolrQueryRequest request(String... q) {
+ LocalSolrQueryRequest req = lrf.makeRequest(q);
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add(req.getParams());
+ params.set("distrib", true);
+ req.setParams(params);
+ return req;
+ }
+
+ protected String generateConfig() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("<dataConfig> \n");
+ sb.append("<propertyWriter dateFormat=\"" + dateFormat + "\" type=\"ZKPropertiesWriter\" />\n");
+ sb.append("<dataSource name=\"mock\" type=\"MockDataSource\"/>\n");
+ sb.append("<document name=\"TestSimplePropertiesWriter\"> \n");
+ sb.append("<entity name=\"SomeDates\" processor=\"SqlEntityProcessor\" dataSource=\"mock\" ");
+ sb.append("query=\"select ${dih.last_index_time} from dummy\" >\n");
+ sb.append("<field column=\"AYEAR_S\" name=\"year_s\" /> \n");
+ sb.append("</entity>\n");
+ sb.append("</document> \n");
+ sb.append("</dataConfig> \n");
+ String config = sb.toString();
+ log.debug(config);
+ return config;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TripleThreatTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TripleThreatTransformer.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TripleThreatTransformer.java
new file mode 100644
index 0000000..2d0aadb
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TripleThreatTransformer.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This transformer does 3 things
+ * <ul>
+ * <li>It turns every row into 3 rows,
+ * modifying any "id" column to ensure duplicate entries in the index
+ * <li>The 2nd Row has 2x values for every column,
+ * with the added one being backwards of the original
+ * <li>The 3rd Row has an added static value
+ * </ul>
+ *
+ * Also, this does not extend Transformer.
+ */
+public class TripleThreatTransformer {
+ public Object transformRow(Map<String, Object> row) {
+ List<Map<String, Object>> rows = new ArrayList<>(3);
+ rows.add(row);
+ rows.add(addDuplicateBackwardsValues(row));
+ rows.add(new LinkedHashMap<>(row));
+ rows.get(2).put("AddAColumn_s", "Added");
+ modifyIdColumn(rows.get(1), 1);
+ modifyIdColumn(rows.get(2), 2);
+ return rows;
+ }
+ private LinkedHashMap<String,Object> addDuplicateBackwardsValues(Map<String, Object> row) {
+ LinkedHashMap<String,Object> n = new LinkedHashMap<>();
+ for(Map.Entry<String,Object> entry : row.entrySet()) {
+ String key = entry.getKey();
+ if(!"id".equalsIgnoreCase(key)) {
+ String[] vals = new String[2];
+ vals[0] = entry.getValue()==null ? "null" : entry.getValue().toString();
+ vals[1] = new StringBuilder(vals[0]).reverse().toString();
+ n.put(key, Arrays.asList(vals));
+ } else {
+ n.put(key, entry.getValue());
+ }
+ }
+ return n;
+ }
+
+ private void modifyIdColumn(Map<String, Object> row, int num) {
+ Object o = row.remove("ID");
+ if(o==null) {
+ o = row.remove("id");
+ }
+ if(o!=null) {
+ String id = o.toString();
+ id = "TripleThreat-" + num + "-" + id;
+ row.put("id", id);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java
deleted file mode 100644
index 9a0f3a7..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.Reader;
-import java.math.BigDecimal;
-import java.sql.Clob;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.sql.rowset.serial.SerialClob;
-
-import org.apache.solr.handler.dataimport.AbstractDataImportHandlerTestCase.TestContext;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-
-public class AbstractDIHCacheTestCase {
- protected static final Date Feb21_2011 = new Date(1298268000000l);
- protected final String[] fieldTypes = { "INTEGER", "BIGDECIMAL", "STRING", "STRING", "FLOAT", "DATE", "CLOB" };
- protected final String[] fieldNames = { "a_id", "PI", "letter", "examples", "a_float", "a_date", "DESCRIPTION" };
- protected List<ControlData> data = new ArrayList<>();
- protected Clob APPLE = null;
-
- @Before
- public void setup() {
- try {
- APPLE = new SerialClob("Apples grow on trees and they are good to eat.".toCharArray());
- } catch (SQLException sqe) {
- Assert.fail("Could not Set up Test");
- }
-
- // The first row needs to have all non-null fields,
- // otherwise we would have to always send the fieldTypes & fieldNames as CacheProperties when building.
- data = new ArrayList<>();
- data.add(new ControlData(new Object[] {1, new BigDecimal(Math.PI), "A", "Apple", 1.11f, Feb21_2011, APPLE }));
- data.add(new ControlData(new Object[] {2, new BigDecimal(Math.PI), "B", "Ball", 2.22f, Feb21_2011, null }));
- data.add(new ControlData(new Object[] {4, new BigDecimal(Math.PI), "D", "Dog", 4.44f, Feb21_2011, null }));
- data.add(new ControlData(new Object[] {3, new BigDecimal(Math.PI), "C", "Cookie", 3.33f, Feb21_2011, null }));
- data.add(new ControlData(new Object[] {4, new BigDecimal(Math.PI), "D", "Daisy", 4.44f, Feb21_2011, null }));
- data.add(new ControlData(new Object[] {4, new BigDecimal(Math.PI), "D", "Drawing", 4.44f, Feb21_2011, null }));
- data.add(new ControlData(new Object[] {5, new BigDecimal(Math.PI), "E",
- Arrays.asList("Eggplant", "Ear", "Elephant", "Engine"), 5.55f, Feb21_2011, null }));
- }
-
- @After
- public void teardown() {
- APPLE = null;
- data = null;
- }
-
- //A limitation of this test class is that the primary key needs to be the first one in the list.
- //DIHCaches, however, can handle any field being the primary key.
- static class ControlData implements Comparable<ControlData>, Iterable<Object> {
- Object[] data;
-
- ControlData(Object[] data) {
- this.data = data;
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public int compareTo(ControlData cd) {
- Comparable c1 = (Comparable) data[0];
- Comparable c2 = (Comparable) cd.data[0];
- return c1.compareTo(c2);
- }
-
- @Override
- public Iterator<Object> iterator() {
- return Arrays.asList(data).iterator();
- }
- }
-
- protected void loadData(DIHCache cache, List<ControlData> theData, String[] theFieldNames, boolean keepOrdered) {
- for (ControlData cd : theData) {
- cache.add(controlDataToMap(cd, theFieldNames, keepOrdered));
- }
- }
-
- protected List<ControlData> extractDataInKeyOrder(DIHCache cache, String[] theFieldNames) {
- List<Object[]> data = new ArrayList<>();
- Iterator<Map<String, Object>> cacheIter = cache.iterator();
- while (cacheIter.hasNext()) {
- data.add(mapToObjectArray(cacheIter.next(), theFieldNames));
- }
- return listToControlData(data);
- }
-
- //This method assumes that the Primary Keys are integers and that the first id=1.
- //It will look for id's sequentially until one is skipped, then will stop.
- protected List<ControlData> extractDataByKeyLookup(DIHCache cache, String[] theFieldNames) {
- int recId = 1;
- List<Object[]> data = new ArrayList<>();
- while (true) {
- Iterator<Map<String, Object>> listORecs = cache.iterator(recId);
- if (listORecs == null) {
- break;
- }
-
- while(listORecs.hasNext()) {
- data.add(mapToObjectArray(listORecs.next(), theFieldNames));
- }
- recId++;
- }
- return listToControlData(data);
- }
-
- protected List<ControlData> listToControlData(List<Object[]> data) {
- List<ControlData> returnData = new ArrayList<>(data.size());
- for (int i = 0; i < data.size(); i++) {
- returnData.add(new ControlData(data.get(i)));
- }
- return returnData;
- }
-
- protected Object[] mapToObjectArray(Map<String, Object> rec, String[] theFieldNames) {
- Object[] oos = new Object[theFieldNames.length];
- for (int i = 0; i < theFieldNames.length; i++) {
- oos[i] = rec.get(theFieldNames[i]);
- }
- return oos;
- }
-
- protected void compareData(List<ControlData> theControl, List<ControlData> test) {
- // The test data should come back primarily in Key order and secondarily in insertion order.
- List<ControlData> control = new ArrayList<>(theControl);
- Collections.sort(control);
-
- StringBuilder errors = new StringBuilder();
- if (test.size() != control.size()) {
- errors.append("-Returned data has " + test.size() + " records. expected: " + control.size() + "\n");
- }
- for (int i = 0; i < control.size() && i < test.size(); i++) {
- Object[] controlRec = control.get(i).data;
- Object[] testRec = test.get(i).data;
- if (testRec.length != controlRec.length) {
- errors.append("-Record indexAt=" + i + " has " + testRec.length + " data elements. extpected: " + controlRec.length + "\n");
- }
- for (int j = 0; j < controlRec.length && j < testRec.length; j++) {
- Object controlObj = controlRec[j];
- Object testObj = testRec[j];
- if (controlObj == null && testObj != null) {
- errors.append("-Record indexAt=" + i + ", Data Element indexAt=" + j + " is not NULL as expected.\n");
- } else if (controlObj != null && testObj == null) {
- errors.append("-Record indexAt=" + i + ", Data Element indexAt=" + j + " is NULL. Expected: " + controlObj + " (class="
- + controlObj.getClass().getName() + ")\n");
- } else if (controlObj != null && testObj != null && controlObj instanceof Clob) {
- String controlString = clobToString((Clob) controlObj);
- String testString = clobToString((Clob) testObj);
- if (!controlString.equals(testString)) {
- errors.append("-Record indexAt=" + i + ", Data Element indexAt=" + j + " has: " + testString + " (class=Clob) ... Expected: " + controlString
- + " (class=Clob)\n");
- }
- } else if (controlObj != null && !controlObj.equals(testObj)) {
- errors.append("-Record indexAt=" + i + ", Data Element indexAt=" + j + " has: " + testObj + " (class=" + testObj.getClass().getName()
- + ") ... Expected: " + controlObj + " (class=" + controlObj.getClass().getName() + ")\n");
- }
- }
- }
- if (errors.length() > 0) {
- Assert.fail(errors.toString());
- }
- }
-
- protected Map<String, Object> controlDataToMap(ControlData cd, String[] theFieldNames, boolean keepOrdered) {
- Map<String, Object> rec = null;
- if (keepOrdered) {
- rec = new LinkedHashMap<>();
- } else {
- rec = new HashMap<>();
- }
- for (int i = 0; i < cd.data.length; i++) {
- String fieldName = theFieldNames[i];
- Object data = cd.data[i];
- rec.put(fieldName, data);
- }
- return rec;
- }
-
- protected String stringArrayToCommaDelimitedList(String[] strs) {
- StringBuilder sb = new StringBuilder();
- for (String a : strs) {
- if (sb.length() > 0) {
- sb.append(",");
- }
- sb.append(a);
- }
- return sb.toString();
- }
-
- protected String clobToString(Clob cl) {
- StringBuilder sb = new StringBuilder();
- try {
- Reader in = cl.getCharacterStream();
- char[] cbuf = new char[1024];
- int numGot = -1;
- while ((numGot = in.read(cbuf)) != -1) {
- sb.append(String.valueOf(cbuf, 0, numGot));
- }
- } catch (Exception e) {
- Assert.fail(e.toString());
- }
- return sb.toString();
- }
-
- public static Context getContext(final Map<String, String> entityAttrs) {
- VariableResolver resolver = new VariableResolver();
- final Context delegate = new ContextImpl(null, resolver, null, null, new HashMap<String, Object>(), null, null);
- return new TestContext(entityAttrs, delegate, null, true);
- }
-
-}
[17/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachePropertyUtil.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachePropertyUtil.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachePropertyUtil.java
new file mode 100644
index 0000000..544761f
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachePropertyUtil.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+public class CachePropertyUtil {
+ public static String getAttributeValueAsString(Context context, String attr) {
+ Object o = context.getSessionAttribute(attr, Context.SCOPE_ENTITY);
+ if (o == null) {
+ o = context.getResolvedEntityAttribute(attr);
+ }
+ if (o == null && context.getRequestParameters() != null) {
+ o = context.getRequestParameters().get(attr);
+ }
+ if (o == null) {
+ return null;
+ }
+ return o.toString();
+ }
+
+ public static Object getAttributeValue(Context context, String attr) {
+ Object o = context.getSessionAttribute(attr, Context.SCOPE_ENTITY);
+ if (o == null) {
+ o = context.getResolvedEntityAttribute(attr);
+ }
+ if (o == null && context.getRequestParameters() != null) {
+ o = context.getRequestParameters().get(attr);
+ }
+ if (o == null) {
+ return null;
+ }
+ return o;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ClobTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ClobTransformer.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ClobTransformer.java
new file mode 100644
index 0000000..4bbe046
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ClobTransformer.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static org.apache.solr.handler.dataimport.HTMLStripTransformer.TRUE;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.sql.Clob;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * {@link Transformer} instance which converts a {@link Clob} to a {@link String}.
+ * <p>
+ * Refer to <a href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * <p>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @since solr 1.4
+ */
+public class ClobTransformer extends Transformer {
+ @Override
+ public Object transformRow(Map<String, Object> aRow, Context context) {
+ for (Map<String, String> map : context.getAllEntityFields()) {
+ if (!TRUE.equals(map.get(CLOB))) continue;
+ String column = map.get(DataImporter.COLUMN);
+ String srcCol = map.get(RegexTransformer.SRC_COL_NAME);
+ if (srcCol == null)
+ srcCol = column;
+ Object o = aRow.get(srcCol);
+ if (o instanceof List) {
+ List<Clob> inputs = (List<Clob>) o;
+ List<String> results = new ArrayList<>();
+ for (Object input : inputs) {
+ if (input instanceof Clob) {
+ Clob clob = (Clob) input;
+ results.add(readFromClob(clob));
+ }
+ }
+ aRow.put(column, results);
+ } else {
+ if (o instanceof Clob) {
+ Clob clob = (Clob) o;
+ aRow.put(column, readFromClob(clob));
+ }
+ }
+ }
+ return aRow;
+ }
+
+ private String readFromClob(Clob clob) {
+ Reader reader = FieldReaderDataSource.readCharStream(clob);
+ StringBuilder sb = new StringBuilder();
+ char[] buf = new char[1024];
+ int len;
+ try {
+ while ((len = reader.read(buf)) != -1) {
+ sb.append(buf, 0, len);
+ }
+ } catch (IOException e) {
+ DataImportHandlerException.wrapAndThrow(DataImportHandlerException.SEVERE, e);
+ }
+ return sb.toString();
+ }
+
+ public static final String CLOB = "clob";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ConfigParseUtil.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ConfigParseUtil.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ConfigParseUtil.java
new file mode 100644
index 0000000..179df23
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ConfigParseUtil.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class ConfigParseUtil {
+ public static String getStringAttribute(Element e, String name, String def) {
+ String r = e.getAttribute(name);
+ if (r == null || "".equals(r.trim()))
+ r = def;
+ return r;
+ }
+
+ public static HashMap<String, String> getAllAttributes(Element e) {
+ HashMap<String, String> m = new HashMap<>();
+ NamedNodeMap nnm = e.getAttributes();
+ for (int i = 0; i < nnm.getLength(); i++) {
+ m.put(nnm.item(i).getNodeName(), nnm.item(i).getNodeValue());
+ }
+ return m;
+ }
+
+ public static String getText(Node elem, StringBuilder buffer) {
+ if (elem.getNodeType() != Node.CDATA_SECTION_NODE) {
+ NodeList childs = elem.getChildNodes();
+ for (int i = 0; i < childs.getLength(); i++) {
+ Node child = childs.item(i);
+ short childType = child.getNodeType();
+ if (childType != Node.COMMENT_NODE
+ && childType != Node.PROCESSING_INSTRUCTION_NODE) {
+ getText(child, buffer);
+ }
+ }
+ } else {
+ buffer.append(elem.getNodeValue());
+ }
+
+ return buffer.toString();
+ }
+
+ public static List<Element> getChildNodes(Element e, String byName) {
+ List<Element> result = new ArrayList<>();
+ NodeList l = e.getChildNodes();
+ for (int i = 0; i < l.getLength(); i++) {
+ if (e.equals(l.item(i).getParentNode())
+ && byName.equals(l.item(i).getNodeName()))
+ result.add((Element) l.item(i));
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContentStreamDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContentStreamDataSource.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContentStreamDataSource.java
new file mode 100644
index 0000000..4482160
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContentStreamDataSource.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.common.util.ContentStream;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Properties;
+
+/**
+ * A DataSource implementation which reads from the ContentStream of a POST request
+ * <p>
+ * Refer to <a href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 1.4
+ */
+public class ContentStreamDataSource extends DataSource<Reader> {
+ private ContextImpl context;
+ private ContentStream contentStream;
+ private Reader reader;
+
+ @Override
+ public void init(Context context, Properties initProps) {
+ this.context = (ContextImpl) context;
+ }
+
+ @Override
+ public Reader getData(String query) {
+ contentStream = context.getDocBuilder().getReqParams().getContentStream();
+ if (contentStream == null)
+ throw new DataImportHandlerException(SEVERE, "No stream available. The request has no body");
+ try {
+ return reader = contentStream.getReader();
+ } catch (IOException e) {
+ DataImportHandlerException.wrapAndThrow(SEVERE, e);
+ return null;
+ }
+ }
+
+ @Override
+ public void close() {
+ if (contentStream != null) {
+ try {
+ if (reader == null) reader = contentStream.getReader();
+ reader.close();
+ } catch (IOException e) {
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java
new file mode 100644
index 0000000..4d0cb3a
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.core.SolrCore;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * <p>
+ * This abstract class gives access to all available objects. So any
+ * component implemented by a user can have the full power of DataImportHandler
+ * </p>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @since solr 1.3
+ */
+public abstract class Context {
+ public static final String FULL_DUMP = "FULL_DUMP", DELTA_DUMP = "DELTA_DUMP", FIND_DELTA = "FIND_DELTA";
+
+ /**
+ * An object stored in entity scope is valid only for the current entity for the current document only.
+ */
+ public static final String SCOPE_ENTITY = "entity";
+
+ /**
+ * An object stored in global scope is available for the current import only but across entities and documents.
+ */
+ public static final String SCOPE_GLOBAL = "global";
+
+ /**
+ * An object stored in document scope is available for the current document only but across entities.
+ */
+ public static final String SCOPE_DOC = "document";
+
+ /**
+ * An object stored in 'solrcore' scope is available across imports, entities and documents throughout the life of
+ * a solr core. A solr core unload or reload will destroy this data.
+ */
+ public static final String SCOPE_SOLR_CORE = "solrcore";
+
+ /**
+ * Get the value of any attribute put into this entity
+ *
+ * @param name name of the attribute eg: 'name'
+ * @return value of named attribute in entity
+ */
+ public abstract String getEntityAttribute(String name);
+
+ /**
+ * Get the value of any attribute put into this entity after resolving all variables found in the attribute value
+ * @param name name of the attribute
+ * @return value of the named attribute after resolving all variables
+ */
+ public abstract String getResolvedEntityAttribute(String name);
+
+ /**
+ * Returns all the fields put into an entity. each item (which is a map ) in
+ * the list corresponds to one field. each if the map contains the attribute
+ * names and values in a field
+ *
+ * @return all fields in an entity
+ */
+ public abstract List<Map<String, String>> getAllEntityFields();
+
+ /**
+ * Returns the VariableResolver used in this entity which can be used to
+ * resolve the tokens in ${<namespce.name>}
+ *
+ * @return a VariableResolver instance
+ * @see org.apache.solr.handler.dataimport.VariableResolver
+ */
+
+ public abstract VariableResolver getVariableResolver();
+
+ /**
+ * Gets the datasource instance defined for this entity. Do not close() this instance.
+ * Transformers should use the getDataSource(String name) method.
+ *
+ * @return a new DataSource instance as configured for the current entity
+ * @see org.apache.solr.handler.dataimport.DataSource
+ * @see #getDataSource(String)
+ */
+ public abstract DataSource getDataSource();
+
+ /**
+ * Gets a new DataSource instance with a name. Ensure that you close() this after use
+ * because this is created just for this method call.
+ *
+ * @param name Name of the dataSource as defined in the dataSource tag
+ * @return a new DataSource instance
+ * @see org.apache.solr.handler.dataimport.DataSource
+ */
+ public abstract DataSource getDataSource(String name);
+
+ /**
+ * Returns the instance of EntityProcessor used for this entity
+ *
+ * @return instance of EntityProcessor used for the current entity
+ * @see org.apache.solr.handler.dataimport.EntityProcessor
+ */
+ public abstract EntityProcessor getEntityProcessor();
+
+ /**
+ * Store values in a certain name and scope (entity, document,global)
+ *
+ * @param name the key
+ * @param val the value
+ * @param scope the scope in which the given key, value pair is to be stored
+ */
+ public abstract void setSessionAttribute(String name, Object val, String scope);
+
+ /**
+ * get a value by name in the given scope (entity, document,global)
+ *
+ * @param name the key
+ * @param scope the scope from which the value is to be retrieved
+ * @return the object stored in the given scope with the given key
+ */
+ public abstract Object getSessionAttribute(String name, String scope);
+
+ /**
+ * Get the context instance for the parent entity. works only in the full dump
+ * If the current entity is rootmost a null is returned
+ *
+ * @return parent entity's Context
+ */
+ public abstract Context getParentContext();
+
+ /**
+ * The request parameters passed over HTTP for this command the values in the
+ * map are either String(for single valued parameters) or List<String> (for
+ * multi-valued parameters)
+ *
+ * @return the request parameters passed in the URL to initiate this process
+ */
+ public abstract Map<String, Object> getRequestParameters();
+
+ /**
+ * Returns if the current entity is the root entity
+ *
+ * @return true if current entity is the root entity, false otherwise
+ */
+ public abstract boolean isRootEntity();
+
+ /**
+ * Returns the current process FULL_DUMP, DELTA_DUMP, FIND_DELTA
+ *
+ * @return the type of the current running process
+ */
+ public abstract String currentProcess();
+
+ /**
+ * Exposing the actual SolrCore to the components
+ *
+ * @return the core
+ */
+ public abstract SolrCore getSolrCore();
+
+ /**
+ * Makes available some basic running statistics such as "docCount",
+ * "deletedDocCount", "rowCount", "queryCount" and "skipDocCount"
+ *
+ * @return a Map containing running statistics of the current import
+ */
+ public abstract Map<String, Object> getStats();
+
+ /**
+ * Returns the text specified in the script tag in the data-config.xml
+ */
+ public abstract String getScript();
+
+ /**
+ * Returns the language of the script as specified in the script tag in data-config.xml
+ */
+ public abstract String getScriptLanguage();
+
+ /**delete a document by id
+ */
+ public abstract void deleteDoc(String id);
+
+ /**delete documents by query
+ */
+ public abstract void deleteDocByQuery(String query);
+
+ /**Use this directly to resolve variable
+ * @param var the variable name
+ * @return the resolved value
+ */
+ public abstract Object resolve(String var);
+
+ /** Resolve variables in a template
+ *
+ * @return The string w/ variables resolved
+ */
+ public abstract String replaceTokens(String template);
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java
new file mode 100644
index 0000000..20a2aad
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.dataimport.config.Script;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * <p>
+ * An implementation for the Context
+ * </p>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @since solr 1.3
+ */
+public class ContextImpl extends Context {
+ protected EntityProcessorWrapper epw;
+
+ private ContextImpl parent;
+
+ private VariableResolver resolver;
+
+ private DataSource ds;
+
+ private String currProcess;
+
+ private Map<String, Object> requestParams;
+
+ private DataImporter dataImporter;
+
+ private Map<String, Object> entitySession, globalSession;
+
+ private Exception lastException = null;
+
+ DocBuilder.DocWrapper doc;
+
+ DocBuilder docBuilder;
+
+
+
+ public ContextImpl(EntityProcessorWrapper epw, VariableResolver resolver,
+ DataSource ds, String currProcess,
+ Map<String, Object> global, ContextImpl parentContext, DocBuilder docBuilder) {
+ this.epw = epw;
+ this.docBuilder = docBuilder;
+ this.resolver = resolver;
+ this.ds = ds;
+ this.currProcess = currProcess;
+ if (docBuilder != null) {
+ this.requestParams = docBuilder.getReqParams().getRawParams();
+ dataImporter = docBuilder.dataImporter;
+ }
+ globalSession = global;
+ parent = parentContext;
+ }
+
+ @Override
+ public String getEntityAttribute(String name) {
+ return epw==null || epw.getEntity() == null ? null : epw.getEntity().getAllAttributes().get(name);
+ }
+
+ @Override
+ public String getResolvedEntityAttribute(String name) {
+ return epw==null || epw.getEntity() == null ? null : resolver.replaceTokens(epw.getEntity().getAllAttributes().get(name));
+ }
+
+ @Override
+ public List<Map<String, String>> getAllEntityFields() {
+ return epw==null || epw.getEntity() == null ? Collections.EMPTY_LIST : epw.getEntity().getAllFieldsList();
+ }
+
+ @Override
+ public VariableResolver getVariableResolver() {
+ return resolver;
+ }
+
+ @Override
+ public DataSource getDataSource() {
+ if (ds != null) return ds;
+ if(epw==null) { return null; }
+ if (epw!=null && epw.getDatasource() == null) {
+ epw.setDatasource(dataImporter.getDataSourceInstance(epw.getEntity(), epw.getEntity().getDataSourceName(), this));
+ }
+ if (epw!=null && epw.getDatasource() != null && docBuilder != null && docBuilder.verboseDebug &&
+ Context.FULL_DUMP.equals(currentProcess())) {
+ //debug is not yet implemented properly for deltas
+ epw.setDatasource(docBuilder.getDebugLogger().wrapDs(epw.getDatasource()));
+ }
+ return epw.getDatasource();
+ }
+
+ @Override
+ public DataSource getDataSource(String name) {
+ return dataImporter.getDataSourceInstance(epw==null ? null : epw.getEntity(), name, this);
+ }
+
+ @Override
+ public boolean isRootEntity() {
+ return epw==null ? false : epw.getEntity().isDocRoot();
+ }
+
+ @Override
+ public String currentProcess() {
+ return currProcess;
+ }
+
+ @Override
+ public Map<String, Object> getRequestParameters() {
+ return requestParams;
+ }
+
+ @Override
+ public EntityProcessor getEntityProcessor() {
+ return epw;
+ }
+
+ @Override
+ public void setSessionAttribute(String name, Object val, String scope) {
+ if(name == null) {
+ return;
+ }
+ if (Context.SCOPE_ENTITY.equals(scope)) {
+ if (entitySession == null) {
+ entitySession = new HashMap<>();
+ }
+ entitySession.put(name, val);
+ } else if (Context.SCOPE_GLOBAL.equals(scope)) {
+ if (globalSession != null) {
+ globalSession.put(name, val);
+ }
+ } else if (Context.SCOPE_DOC.equals(scope)) {
+ DocBuilder.DocWrapper doc = getDocument();
+ if (doc != null) {
+ doc.setSessionAttribute(name, val);
+ }
+ } else if (SCOPE_SOLR_CORE.equals(scope)){
+ if(dataImporter != null) {
+ dataImporter.putToCoreScopeSession(name, val);
+ }
+ }
+ }
+
+ @Override
+ public Object getSessionAttribute(String name, String scope) {
+ if (Context.SCOPE_ENTITY.equals(scope)) {
+ if (entitySession == null)
+ return null;
+ return entitySession.get(name);
+ } else if (Context.SCOPE_GLOBAL.equals(scope)) {
+ if (globalSession != null) {
+ return globalSession.get(name);
+ }
+ } else if (Context.SCOPE_DOC.equals(scope)) {
+ DocBuilder.DocWrapper doc = getDocument();
+ return doc == null ? null: doc.getSessionAttribute(name);
+ } else if (SCOPE_SOLR_CORE.equals(scope)){
+ return dataImporter == null ? null : dataImporter.getFromCoreScopeSession(name);
+ }
+ return null;
+ }
+
+ @Override
+ public Context getParentContext() {
+ return parent;
+ }
+
+ private DocBuilder.DocWrapper getDocument() {
+ ContextImpl c = this;
+ while (true) {
+ if (c.doc != null)
+ return c.doc;
+ if (c.parent != null)
+ c = c.parent;
+ else
+ return null;
+ }
+ }
+
+ void setDoc(DocBuilder.DocWrapper docWrapper) {
+ this.doc = docWrapper;
+ }
+
+
+ @Override
+ public SolrCore getSolrCore() {
+ return dataImporter == null ? null : dataImporter.getCore();
+ }
+
+
+ @Override
+ public Map<String, Object> getStats() {
+ return docBuilder != null ? docBuilder.importStatistics.getStatsSnapshot() : Collections.<String, Object>emptyMap();
+ }
+
+ @Override
+ public String getScript() {
+ if (dataImporter != null) {
+ Script script = dataImporter.getConfig().getScript();
+ return script == null ? null : script.getText();
+ }
+ return null;
+ }
+
+ @Override
+ public String getScriptLanguage() {
+ if (dataImporter != null) {
+ Script script = dataImporter.getConfig().getScript();
+ return script == null ? null : script.getLanguage();
+ }
+ return null;
+ }
+
+ @Override
+ public void deleteDoc(String id) {
+ if(docBuilder != null){
+ docBuilder.writer.deleteDoc(id);
+ }
+ }
+
+ @Override
+ public void deleteDocByQuery(String query) {
+ if(docBuilder != null){
+ docBuilder.writer.deleteByQuery(query);
+ }
+ }
+
+ DocBuilder getDocBuilder(){
+ return docBuilder;
+ }
+ @Override
+ public Object resolve(String var) {
+ return resolver.resolve(var);
+ }
+
+ @Override
+ public String replaceTokens(String template) {
+ return resolver.replaceTokens(template);
+ }
+
+ public Exception getLastException() { return lastException; }
+
+ public void setLastException(Exception lastException) {this.lastException = lastException; }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHCache.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHCache.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHCache.java
new file mode 100644
index 0000000..a67b3e4
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHCache.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * <p>
+ * A cache that allows a DIH entity's data to persist locally prior being joined
+ * to other data and/or indexed.
+ * </p>
+ *
+ * @lucene.experimental
+ */
+public interface DIHCache extends Iterable<Map<String,Object>> {
+
+ /**
+ * <p>
+ * Opens the cache using the specified properties. The {@link Context}
+ * includes any parameters needed by the cache impl. This must be called
+ * before any read/write operations are permitted.
+ */
+ void open(Context context);
+
+ /**
+ * <p>
+ * Releases resources used by this cache, if possible. The cache is flushed
+ * but not destroyed.
+ * </p>
+ */
+ void close();
+
+ /**
+ * <p>
+ * Persists any pending data to the cache
+ * </p>
+ */
+ void flush();
+
+ /**
+ * <p>
+ * Closes the cache, if open. Then removes all data, possibly removing the
+ * cache entirely from persistent storage.
+ * </p>
+ */
+ public void destroy();
+
+ /**
+ * <p>
+ * Adds a document. If a document already exists with the same key, both
+ * documents will exist in the cache, as the cache allows duplicate keys. To
+ * update a key's documents, first call delete(Object key).
+ * </p>
+ */
+ void add(Map<String, Object> rec);
+
+ /**
+ * <p>
+ * Returns an iterator, allowing callers to iterate through the entire cache
+ * in key, then insertion, order.
+ * </p>
+ */
+ @Override
+ Iterator<Map<String,Object>> iterator();
+
+ /**
+ * <p>
+ * Returns an iterator, allowing callers to iterate through all documents that
+ * match the given key in insertion order.
+ * </p>
+ */
+ Iterator<Map<String,Object>> iterator(Object key);
+
+ /**
+ * <p>
+ * Delete all documents associated with the given key
+ * </p>
+ */
+ void delete(Object key);
+
+ /**
+ * <p>
+ * Delete all data from the cache,leaving the empty cache intact.
+ * </p>
+ */
+ void deleteAll();
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java
new file mode 100644
index 0000000..2f3d957
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+
+import java.lang.invoke.MethodHandles;
+import java.lang.reflect.Constructor;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.solr.common.SolrException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DIHCacheSupport {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private String cacheForeignKey;
+ private String cacheImplName;
+ private Map<String,DIHCache> queryVsCache = new HashMap<>();
+ private Map<String,Iterator<Map<String,Object>>> queryVsCacheIterator;
+ private Iterator<Map<String,Object>> dataSourceRowCache;
+ private boolean cacheDoKeyLookup;
+
+ public DIHCacheSupport(Context context, String cacheImplName) {
+ this.cacheImplName = cacheImplName;
+
+ Relation r = new Relation(context);
+ cacheDoKeyLookup = r.doKeyLookup;
+ String cacheKey = r.primaryKey;
+ cacheForeignKey = r.foreignKey;
+
+ context.setSessionAttribute(DIHCacheSupport.CACHE_PRIMARY_KEY, cacheKey,
+ Context.SCOPE_ENTITY);
+ context.setSessionAttribute(DIHCacheSupport.CACHE_FOREIGN_KEY, cacheForeignKey,
+ Context.SCOPE_ENTITY);
+ context.setSessionAttribute(DIHCacheSupport.CACHE_DELETE_PRIOR_DATA,
+ "true", Context.SCOPE_ENTITY);
+ context.setSessionAttribute(DIHCacheSupport.CACHE_READ_ONLY, "false",
+ Context.SCOPE_ENTITY);
+ }
+
+ static class Relation{
+ protected final boolean doKeyLookup;
+ protected final String foreignKey;
+ protected final String primaryKey;
+
+ public Relation(Context context) {
+ String where = context.getEntityAttribute("where");
+ String cacheKey = context.getEntityAttribute(DIHCacheSupport.CACHE_PRIMARY_KEY);
+ String lookupKey = context.getEntityAttribute(DIHCacheSupport.CACHE_FOREIGN_KEY);
+ if (cacheKey != null && lookupKey == null) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "'cacheKey' is specified for the entity "
+ + context.getEntityAttribute("name")
+ + " but 'cacheLookup' is missing");
+
+ }
+ if (where == null && cacheKey == null) {
+ doKeyLookup = false;
+ primaryKey = null;
+ foreignKey = null;
+ } else {
+ if (where != null) {
+ String[] splits = where.split("=");
+ primaryKey = splits[0];
+ foreignKey = splits[1].trim();
+ } else {
+ primaryKey = cacheKey;
+ foreignKey = lookupKey;
+ }
+ doKeyLookup = true;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Relation "
+ + primaryKey + "="+foreignKey ;
+ }
+
+
+ }
+
+ private DIHCache instantiateCache(Context context) {
+ DIHCache cache = null;
+ try {
+ @SuppressWarnings("unchecked")
+ Class<DIHCache> cacheClass = DocBuilder.loadClass(cacheImplName, context
+ .getSolrCore());
+ Constructor<DIHCache> constr = cacheClass.getConstructor();
+ cache = constr.newInstance();
+ cache.open(context);
+ } catch (Exception e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Unable to load Cache implementation:" + cacheImplName, e);
+ }
+ return cache;
+ }
+
+ public void initNewParent(Context context) {
+ dataSourceRowCache = null;
+ queryVsCacheIterator = new HashMap<>();
+ for (Map.Entry<String,DIHCache> entry : queryVsCache.entrySet()) {
+ queryVsCacheIterator.put(entry.getKey(), entry.getValue().iterator());
+ }
+ }
+
+ public void destroyAll() {
+ if (queryVsCache != null) {
+ for (DIHCache cache : queryVsCache.values()) {
+ cache.destroy();
+ }
+ }
+ queryVsCache = null;
+ dataSourceRowCache = null;
+ cacheForeignKey = null;
+ }
+
+ /**
+ * <p>
+ * Get all the rows from the datasource for the given query and cache them
+ * </p>
+ */
+ public void populateCache(String query,
+ Iterator<Map<String,Object>> rowIterator) {
+ Map<String,Object> aRow = null;
+ DIHCache cache = queryVsCache.get(query);
+ while ((aRow = getNextFromCache(query, rowIterator)) != null) {
+ cache.add(aRow);
+ }
+ }
+
+ private Map<String,Object> getNextFromCache(String query,
+ Iterator<Map<String,Object>> rowIterator) {
+ try {
+ if (rowIterator == null) return null;
+ if (rowIterator.hasNext()) return rowIterator.next();
+ return null;
+ } catch (Exception e) {
+ SolrException.log(log, "getNextFromCache() failed for query '" + query
+ + "'", e);
+ wrapAndThrow(DataImportHandlerException.WARN, e);
+ return null;
+ }
+ }
+
+ public Map<String,Object> getCacheData(Context context, String query,
+ Iterator<Map<String,Object>> rowIterator) {
+ if (cacheDoKeyLookup) {
+ return getIdCacheData(context, query, rowIterator);
+ } else {
+ return getSimpleCacheData(context, query, rowIterator);
+ }
+ }
+
+ /**
+ * If the where clause is present the cache is sql Vs Map of key Vs List of
+ * Rows.
+ *
+ * @param query
+ * the query string for which cached data is to be returned
+ *
+ * @return the cached row corresponding to the given query after all variables
+ * have been resolved
+ */
+ protected Map<String,Object> getIdCacheData(Context context, String query,
+ Iterator<Map<String,Object>> rowIterator) {
+ Object key = context.resolve(cacheForeignKey);
+ if (key == null) {
+ throw new DataImportHandlerException(DataImportHandlerException.WARN,
+ "The cache lookup value : " + cacheForeignKey
+ + " is resolved to be null in the entity :"
+ + context.getEntityAttribute("name"));
+
+ }
+ if (dataSourceRowCache == null) {
+ DIHCache cache = queryVsCache.get(query);
+
+ if (cache == null) {
+ cache = instantiateCache(context);
+ queryVsCache.put(query, cache);
+ populateCache(query, rowIterator);
+ }
+ dataSourceRowCache = cache.iterator(key);
+ }
+ return getFromRowCacheTransformed();
+ }
+
+ /**
+ * If where clause is not present the cache is a Map of query vs List of Rows.
+ *
+ * @param query
+ * string for which cached row is to be returned
+ *
+ * @return the cached row corresponding to the given query
+ */
+ protected Map<String,Object> getSimpleCacheData(Context context,
+ String query, Iterator<Map<String,Object>> rowIterator) {
+ if (dataSourceRowCache == null) {
+ DIHCache cache = queryVsCache.get(query);
+ if (cache == null) {
+ cache = instantiateCache(context);
+ queryVsCache.put(query, cache);
+ populateCache(query, rowIterator);
+ queryVsCacheIterator.put(query, cache.iterator());
+ }
+ Iterator<Map<String,Object>> cacheIter = queryVsCacheIterator.get(query);
+ dataSourceRowCache = cacheIter;
+ }
+
+ return getFromRowCacheTransformed();
+ }
+
+ protected Map<String,Object> getFromRowCacheTransformed() {
+ if (dataSourceRowCache == null || !dataSourceRowCache.hasNext()) {
+ dataSourceRowCache = null;
+ return null;
+ }
+ Map<String,Object> r = dataSourceRowCache.next();
+ return r;
+ }
+
+ /**
+ * <p>
+ * Specify the class for the cache implementation
+ * </p>
+ */
+ public static final String CACHE_IMPL = "cacheImpl";
+
+ /**
+ * <p>
+ * If the cache supports persistent data, set to "true" to delete any prior
+ * persisted data before running the entity.
+ * </p>
+ */
+
+ public static final String CACHE_DELETE_PRIOR_DATA = "cacheDeletePriorData";
+ /**
+ * <p>
+ * Specify the Foreign Key from the parent entity to join on. Use if the cache
+ * is on a child entity.
+ * </p>
+ */
+ public static final String CACHE_FOREIGN_KEY = "cacheLookup";
+
+ /**
+ * <p>
+ * Specify the Primary Key field from this Entity to map the input records
+ * with
+ * </p>
+ */
+ public static final String CACHE_PRIMARY_KEY = "cacheKey";
+ /**
+ * <p>
+ * If true, a pre-existing cache is re-opened for read-only access.
+ * </p>
+ */
+ public static final String CACHE_READ_ONLY = "cacheReadOnly";
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHLogLevels.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHLogLevels.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHLogLevels.java
new file mode 100644
index 0000000..24732d1
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHLogLevels.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+public enum DIHLogLevels {
+ START_ENTITY, END_ENTITY, TRANSFORMED_ROW, ENTITY_META, PRE_TRANSFORMER_ROW, START_DOC, END_DOC, ENTITY_OUT, ROW_END, TRANSFORMER_EXCEPTION, ENTITY_EXCEPTION, DISABLE_LOGGING, ENABLE_LOGGING, NONE
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHProperties.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHProperties.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHProperties.java
new file mode 100644
index 0000000..f51ef07
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHProperties.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.Date;
+import java.util.Map;
+
+/**
+ * Implementations write out properties about the last data import
+ * for use by the next import. ex: to persist the last import timestamp
+ * so that future delta imports can know what needs to be updated.
+ *
+ * @lucene.experimental
+ */
+public abstract class DIHProperties {
+
+ public abstract void init(DataImporter dataImporter, Map<String, String> initParams);
+
+ public abstract boolean isWritable();
+
+ public abstract void persist(Map<String, Object> props);
+
+ public abstract Map<String, Object> readIndexerProperties();
+
+ public abstract String convertDateToString(Date d);
+
+ public Date getCurrentTimestamp() {
+ return new Date();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHWriter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHWriter.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHWriter.java
new file mode 100644
index 0000000..bdb988d
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHWriter.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.solr.common.SolrInputDocument;
+
+/**
+ * @lucene.experimental
+ *
+ */
+public interface DIHWriter {
+
+ /**
+ * <p>
+ * If this writer supports transactions or commit points, then commit any changes,
+ * optionally optimizing the data for read/write performance
+ * </p>
+ */
+ public void commit(boolean optimize);
+
+ /**
+ * <p>
+ * Release resources used by this writer. After calling close, reads & updates will throw exceptions.
+ * </p>
+ */
+ public void close();
+
+ /**
+ * <p>
+ * If this writer supports transactions or commit points, then roll back any uncommitted changes.
+ * </p>
+ */
+ public void rollback();
+
+ /**
+ * <p>
+ * Delete from the writer's underlying data store based the passed-in writer-specific query. (Optional Operation)
+ * </p>
+ */
+ public void deleteByQuery(String q);
+
+ /**
+ * <p>
+ * Delete everything from the writer's underlying data store
+ * </p>
+ */
+ public void doDeleteAll();
+
+ /**
+ * <p>
+ * Delete from the writer's underlying data store based on the passed-in Primary Key
+ * </p>
+ */
+ public void deleteDoc(Object key);
+
+
+
+ /**
+ * <p>
+ * Add a document to this writer's underlying data store.
+ * </p>
+ * @return true on success, false on failure
+ */
+ public boolean upload(SolrInputDocument doc);
+
+
+
+ /**
+ * <p>
+ * Provide context information for this writer. init() should be called before using the writer.
+ * </p>
+ */
+ public void init(Context context) ;
+
+
+ /**
+ * <p>
+ * Specify the keys to be modified by a delta update (required by writers that can store duplicate keys)
+ * </p>
+ */
+ public void setDeltaKeys(Set<Map<String, Object>> deltaKeys) ;
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHWriterBase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHWriterBase.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHWriterBase.java
new file mode 100644
index 0000000..43e92c3
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DIHWriterBase.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public abstract class DIHWriterBase implements DIHWriter {
+ protected String keyFieldName;
+ protected Set<Object> deltaKeys = null;
+
+ @Override
+ public void setDeltaKeys(Set<Map<String,Object>> passedInDeltaKeys) {
+ deltaKeys = new HashSet<>();
+ for (Map<String,Object> aMap : passedInDeltaKeys) {
+ if (aMap.size() > 0) {
+ Object key = null;
+ if (keyFieldName != null) {
+ key = aMap.get(keyFieldName);
+ } else {
+ key = aMap.entrySet().iterator().next();
+ }
+ if (key != null) {
+ deltaKeys.add(key);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java
new file mode 100644
index 0000000..71ee442
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.lang.invoke.MethodHandles;
+import java.lang.reflect.Constructor;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.metrics.MetricsMap;
+import org.apache.solr.metrics.SolrMetricManager;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.RawResponseWriter;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import org.apache.solr.update.processor.UpdateRequestProcessorChain;
+import org.apache.solr.util.plugin.SolrCoreAware;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.handler.dataimport.DataImporter.IMPORT_CMD;
+
+/**
+ * <p>
+ * Solr Request Handler for data import from databases and REST data sources.
+ * </p>
+ * <p>
+ * It is configured in solrconfig.xml
+ * </p>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @since solr 1.3
+ */
+public class DataImportHandler extends RequestHandlerBase implements
+ SolrCoreAware {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private DataImporter importer;
+
+ private boolean debugEnabled = true;
+
+ private String myName = "dataimport";
+
+ private MetricsMap metrics;
+
+ private static final String PARAM_WRITER_IMPL = "writerImpl";
+ private static final String DEFAULT_WRITER_NAME = "SolrWriter";
+
+ public DataImporter getImporter() {
+ return this.importer;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void init(NamedList args) {
+ super.init(args);
+ Map<String,String> macro = new HashMap<>();
+ macro.put("expandMacros", "false");
+ defaults = SolrParams.wrapDefaults(defaults, new MapSolrParams(macro));
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void inform(SolrCore core) {
+ try {
+ String name = getPluginInfo().name;
+ if (name.startsWith("/")) {
+ myName = name.substring(1);
+ }
+ // some users may have '/' in the handler name. replace with '_'
+ myName = myName.replaceAll("/", "_");
+ debugEnabled = StrUtils.parseBool((String)initArgs.get(ENABLE_DEBUG), true);
+ importer = new DataImporter(core, myName);
+ } catch (Exception e) {
+ log.error( DataImporter.MSG.LOAD_EXP, e);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, DataImporter.MSG.LOAD_EXP, e);
+ }
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp)
+ throws Exception {
+ rsp.setHttpCaching(false);
+
+ //TODO: figure out why just the first one is OK...
+ ContentStream contentStream = null;
+ Iterable<ContentStream> streams = req.getContentStreams();
+ if(streams != null){
+ for (ContentStream stream : streams) {
+ contentStream = stream;
+ break;
+ }
+ }
+ SolrParams params = req.getParams();
+ NamedList defaultParams = (NamedList) initArgs.get("defaults");
+ RequestInfo requestParams = new RequestInfo(req, getParamsMap(params), contentStream);
+ String command = requestParams.getCommand();
+
+ if (DataImporter.SHOW_CONF_CMD.equals(command)) {
+ String dataConfigFile = params.get("config");
+ String dataConfig = params.get("dataConfig");
+ if(dataConfigFile != null) {
+ dataConfig = SolrWriter.getResourceAsString(req.getCore().getResourceLoader().openResource(dataConfigFile));
+ }
+ if(dataConfig==null) {
+ rsp.add("status", DataImporter.MSG.NO_CONFIG_FOUND);
+ } else {
+ // Modify incoming request params to add wt=raw
+ ModifiableSolrParams rawParams = new ModifiableSolrParams(req.getParams());
+ rawParams.set(CommonParams.WT, "raw");
+ req.setParams(rawParams);
+ ContentStreamBase content = new ContentStreamBase.StringStream(dataConfig);
+ rsp.add(RawResponseWriter.CONTENT, content);
+ }
+ return;
+ }
+
+ rsp.add("initArgs", initArgs);
+ String message = "";
+
+ if (command != null) {
+ rsp.add("command", command);
+ }
+ // If importer is still null
+ if (importer == null) {
+ rsp.add("status", DataImporter.MSG.NO_INIT);
+ return;
+ }
+
+ if (command != null && DataImporter.ABORT_CMD.equals(command)) {
+ importer.runCmd(requestParams, null);
+ } else if (importer.isBusy()) {
+ message = DataImporter.MSG.CMD_RUNNING;
+ } else if (command != null) {
+ if (DataImporter.FULL_IMPORT_CMD.equals(command)
+ || DataImporter.DELTA_IMPORT_CMD.equals(command) ||
+ IMPORT_CMD.equals(command)) {
+ importer.maybeReloadConfiguration(requestParams, defaultParams);
+ UpdateRequestProcessorChain processorChain =
+ req.getCore().getUpdateProcessorChain(params);
+ UpdateRequestProcessor processor = processorChain.createProcessor(req, rsp);
+ SolrResourceLoader loader = req.getCore().getResourceLoader();
+ DIHWriter sw = getSolrWriter(processor, loader, requestParams, req);
+
+ if (requestParams.isDebug()) {
+ if (debugEnabled) {
+ // Synchronous request for the debug mode
+ importer.runCmd(requestParams, sw);
+ rsp.add("mode", "debug");
+ rsp.add("documents", requestParams.getDebugInfo().debugDocuments);
+ if (requestParams.getDebugInfo().debugVerboseOutput != null) {
+ rsp.add("verbose-output", requestParams.getDebugInfo().debugVerboseOutput);
+ }
+ } else {
+ message = DataImporter.MSG.DEBUG_NOT_ENABLED;
+ }
+ } else {
+ // Asynchronous request for normal mode
+ if(requestParams.getContentStream() == null && !requestParams.isSyncMode()){
+ importer.runAsync(requestParams, sw);
+ } else {
+ importer.runCmd(requestParams, sw);
+ }
+ }
+ } else if (DataImporter.RELOAD_CONF_CMD.equals(command)) {
+ if(importer.maybeReloadConfiguration(requestParams, defaultParams)) {
+ message = DataImporter.MSG.CONFIG_RELOADED;
+ } else {
+ message = DataImporter.MSG.CONFIG_NOT_RELOADED;
+ }
+ }
+ }
+ rsp.add("status", importer.isBusy() ? "busy" : "idle");
+ rsp.add("importResponse", message);
+ rsp.add("statusMessages", importer.getStatusMessages());
+ }
+
+ /** The value is converted to a String or {@code List<String>} if multi-valued. */
+ private Map<String, Object> getParamsMap(SolrParams params) {
+ Map<String, Object> result = new HashMap<>();
+ for (Map.Entry<String, String[]> pair : params){
+ String s = pair.getKey();
+ String[] val = pair.getValue();
+ if (val == null || val.length < 1)
+ continue;
+ if (val.length == 1)
+ result.put(s, val[0]);
+ else
+ result.put(s, Arrays.asList(val));
+ }
+ return result;
+ }
+
+ private DIHWriter getSolrWriter(final UpdateRequestProcessor processor,
+ final SolrResourceLoader loader, final RequestInfo requestParams,
+ SolrQueryRequest req) {
+ SolrParams reqParams = req.getParams();
+ String writerClassStr = null;
+ if (reqParams != null && reqParams.get(PARAM_WRITER_IMPL) != null) {
+ writerClassStr = (String) reqParams.get(PARAM_WRITER_IMPL);
+ }
+ DIHWriter writer;
+ if (writerClassStr != null
+ && !writerClassStr.equals(DEFAULT_WRITER_NAME)
+ && !writerClassStr.equals(DocBuilder.class.getPackage().getName() + "."
+ + DEFAULT_WRITER_NAME)) {
+ try {
+ @SuppressWarnings("unchecked")
+ Class<DIHWriter> writerClass = DocBuilder.loadClass(writerClassStr, req.getCore());
+ Constructor<DIHWriter> cnstr = writerClass.getConstructor(new Class[] {
+ UpdateRequestProcessor.class, SolrQueryRequest.class});
+ return cnstr.newInstance((Object) processor, (Object) req);
+ } catch (Exception e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Unable to load Writer implementation:" + writerClassStr, e);
+ }
+ } else {
+ return new SolrWriter(processor, req) {
+ @Override
+ public boolean upload(SolrInputDocument document) {
+ try {
+ return super.upload(document);
+ } catch (RuntimeException e) {
+ log.error("Exception while adding: " + document, e);
+ return false;
+ }
+ }
+ };
+ }
+ }
+
+ @Override
+ public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) {
+ super.initializeMetrics(manager, registryName, tag, scope);
+ metrics = new MetricsMap((detailed, map) -> {
+ if (importer != null) {
+ DocBuilder.Statistics cumulative = importer.cumulativeStatistics;
+
+ map.put("Status", importer.getStatus().toString());
+
+ if (importer.docBuilder != null) {
+ DocBuilder.Statistics running = importer.docBuilder.importStatistics;
+ map.put("Documents Processed", running.docCount);
+ map.put("Requests made to DataSource", running.queryCount);
+ map.put("Rows Fetched", running.rowsCount);
+ map.put("Documents Deleted", running.deletedDocCount);
+ map.put("Documents Skipped", running.skipDocCount);
+ }
+
+ map.put(DataImporter.MSG.TOTAL_DOC_PROCESSED, cumulative.docCount);
+ map.put(DataImporter.MSG.TOTAL_QUERIES_EXECUTED, cumulative.queryCount);
+ map.put(DataImporter.MSG.TOTAL_ROWS_EXECUTED, cumulative.rowsCount);
+ map.put(DataImporter.MSG.TOTAL_DOCS_DELETED, cumulative.deletedDocCount);
+ map.put(DataImporter.MSG.TOTAL_DOCS_SKIPPED, cumulative.skipDocCount);
+ }
+ });
+ manager.registerGauge(this, registryName, metrics, tag, true, "importer", getCategory().toString(), scope);
+ }
+
+ // //////////////////////SolrInfoMBeans methods //////////////////////
+
+ @Override
+ public String getDescription() {
+ return DataImporter.MSG.JMX_DESC;
+ }
+
+ public static final String ENABLE_DEBUG = "enableDebug";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java
new file mode 100644
index 0000000..e69b3fd
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+/**
+ * <p> Exception class for all DataImportHandler exceptions </p>
+ * <p>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @since solr 1.3
+ */
+public class DataImportHandlerException extends RuntimeException {
+ private int errCode;
+
+ public boolean debugged = false;
+
+ public static final int SEVERE = 500, WARN = 400, SKIP = 300, SKIP_ROW =301;
+
+ public DataImportHandlerException(int err) {
+ super();
+ errCode = err;
+ }
+
+ public DataImportHandlerException(int err, String message) {
+ super(message + (SolrWriter.getDocCount() == null ? "" : MSG + SolrWriter.getDocCount()));
+ errCode = err;
+ }
+
+ public DataImportHandlerException(int err, String message, Throwable cause) {
+ super(message + (SolrWriter.getDocCount() == null ? "" : MSG + SolrWriter.getDocCount()), cause);
+ errCode = err;
+ }
+
+ public DataImportHandlerException(int err, Throwable cause) {
+ super(cause);
+ errCode = err;
+ }
+
+ public int getErrCode() {
+ return errCode;
+ }
+
+ public static DataImportHandlerException wrapAndThrow(int err, Exception e) {
+ if (e instanceof DataImportHandlerException) {
+ throw (DataImportHandlerException) e;
+ } else {
+ throw new DataImportHandlerException(err, e);
+ }
+ }
+
+ public static DataImportHandlerException wrapAndThrow(int err, Exception e, String msg) {
+ if (e instanceof DataImportHandlerException) {
+ throw (DataImportHandlerException) e;
+ } else {
+ throw new DataImportHandlerException(err, msg, e);
+ }
+ }
+
+
+ public static final String MSG = " Processing Document # ";
+}
[15/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessor.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessor.java
new file mode 100644
index 0000000..8cfbed9
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessor.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.Map;
+
+/**
+ * <p>
+ * An instance of entity processor serves an entity. It is reused throughout the
+ * import process.
+ * </p>
+ * <p>
+ * Implementations of this abstract class must provide a public no-args constructor.
+ * </p>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 1.3
+ */
+public abstract class EntityProcessor {
+
+ /**
+ * This method is called when it starts processing an entity. When it comes
+ * back to the entity it is called again. So it can reset anything at that point.
+ * For a rootmost entity this is called only once for an ingestion. For sub-entities , this
+ * is called multiple once for each row from its parent entity
+ *
+ * @param context The current context
+ */
+ public abstract void init(Context context);
+
+ /**
+ * This method helps streaming the data for each row . The implementation
+ * would fetch as many rows as needed and gives one 'row' at a time. Only this
+ * method is used during a full import
+ *
+ * @return A 'row'. The 'key' for the map is the column name and the 'value'
+ * is the value of that column. If there are no more rows to be
+ * returned, return 'null'
+ */
+ public abstract Map<String, Object> nextRow();
+
+ /**
+ * This is used for delta-import. It gives the pks of the changed rows in this
+ * entity
+ *
+ * @return the pk vs value of all changed rows
+ */
+ public abstract Map<String, Object> nextModifiedRowKey();
+
+ /**
+ * This is used during delta-import. It gives the primary keys of the rows
+ * that are deleted from this entity. If this entity is the root entity, solr
+ * document is deleted. If this is a sub-entity, the Solr document is
+ * considered as 'changed' and will be recreated
+ *
+ * @return the pk vs value of all changed rows
+ */
+ public abstract Map<String, Object> nextDeletedRowKey();
+
+ /**
+ * This is used during delta-import. This gives the primary keys and their
+ * values of all the rows changed in a parent entity due to changes in this
+ * entity.
+ *
+ * @return the pk vs value of all changed rows in the parent entity
+ */
+ public abstract Map<String, Object> nextModifiedParentRowKey();
+
+ /**
+ * Invoked for each entity at the very end of the import to do any needed cleanup tasks.
+ *
+ */
+ public abstract void destroy();
+
+ /**
+ * Invoked after the transformers are invoked. EntityProcessors can add, remove or modify values
+ * added by Transformers in this method.
+ *
+ * @param r The transformed row
+ * @since solr 1.4
+ */
+ public void postTransform(Map<String, Object> r) {
+ }
+
+ /**
+ * Invoked when the Entity processor is destroyed towards the end of import.
+ *
+ * @since solr 1.4
+ */
+ public void close() {
+ //no-op
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
new file mode 100644
index 0000000..8311f36
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.common.SolrException;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.util.*;
+
+/**
+ * <p> Base class for all implementations of {@link EntityProcessor} </p> <p> Most implementations of {@link EntityProcessor}
+ * extend this base class which provides common functionality. </p>
+ * <p>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @since solr 1.3
+ */
+public class EntityProcessorBase extends EntityProcessor {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ protected boolean isFirstInit = true;
+
+ protected String entityName;
+
+ protected Context context;
+
+ protected Iterator<Map<String, Object>> rowIterator;
+
+ protected String query;
+
+ protected String onError = ABORT;
+
+ protected DIHCacheSupport cacheSupport = null;
+
+ private Zipper zipper;
+
+
+ @Override
+ public void init(Context context) {
+ this.context = context;
+ if (isFirstInit) {
+ firstInit(context);
+ }
+ if(zipper!=null){
+ zipper.onNewParent(context);
+ }else{
+ if(cacheSupport!=null) {
+ cacheSupport.initNewParent(context);
+ }
+ }
+ }
+
+ /**
+ * first time init call. do one-time operations here
+ * it's necessary to call it from the overridden method,
+ * otherwise it throws NPE on accessing zipper from nextRow()
+ */
+ protected void firstInit(Context context) {
+ entityName = context.getEntityAttribute("name");
+ String s = context.getEntityAttribute(ON_ERROR);
+ if (s != null) onError = s;
+
+ zipper = Zipper.createOrNull(context);
+
+ if(zipper==null){
+ initCache(context);
+ }
+ isFirstInit = false;
+ }
+
+ protected void initCache(Context context) {
+ String cacheImplName = context
+ .getResolvedEntityAttribute(DIHCacheSupport.CACHE_IMPL);
+
+ if (cacheImplName != null ) {
+ cacheSupport = new DIHCacheSupport(context, cacheImplName);
+ }
+ }
+
+ @Override
+ public Map<String, Object> nextModifiedRowKey() {
+ return null;
+ }
+
+ @Override
+ public Map<String, Object> nextDeletedRowKey() {
+ return null;
+ }
+
+ @Override
+ public Map<String, Object> nextModifiedParentRowKey() {
+ return null;
+ }
+
+ /**
+ * For a simple implementation, this is the only method that the sub-class should implement. This is intended to
+ * stream rows one-by-one. Return null to signal end of rows
+ *
+ * @return a row where the key is the name of the field and value can be any Object or a Collection of objects. Return
+ * null to signal end of rows
+ */
+ @Override
+ public Map<String, Object> nextRow() {
+ return null;// do not do anything
+ }
+
+ protected Map<String, Object> getNext() {
+ if(zipper!=null){
+ return zipper.supplyNextChild(rowIterator);
+ }else{
+ if(cacheSupport==null) {
+ try {
+ if (rowIterator == null)
+ return null;
+ if (rowIterator.hasNext())
+ return rowIterator.next();
+ query = null;
+ rowIterator = null;
+ return null;
+ } catch (Exception e) {
+ SolrException.log(log, "getNext() failed for query '" + query + "'", e);
+ query = null;
+ rowIterator = null;
+ wrapAndThrow(DataImportHandlerException.WARN, e);
+ return null;
+ }
+ } else {
+ return cacheSupport.getCacheData(context, query, rowIterator);
+ }
+ }
+ }
+
+
+ @Override
+ public void destroy() {
+ query = null;
+ if(cacheSupport!=null){
+ cacheSupport.destroyAll();
+ }
+ cacheSupport = null;
+ }
+
+
+
+ public static final String TRANSFORMER = "transformer";
+
+ public static final String TRANSFORM_ROW = "transformRow";
+
+ public static final String ON_ERROR = "onError";
+
+ public static final String ABORT = "abort";
+
+ public static final String CONTINUE = "continue";
+
+ public static final String SKIP = "skip";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java
new file mode 100644
index 0000000..8a76e11
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.dataimport.config.ConfigNameConstants;
+import org.apache.solr.handler.dataimport.config.Entity;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.*;
+import static org.apache.solr.handler.dataimport.EntityProcessorBase.*;
+import static org.apache.solr.handler.dataimport.EntityProcessorBase.SKIP;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A Wrapper over {@link EntityProcessor} instance which performs transforms and handles multi-row outputs correctly.
+ *
+ * @since solr 1.4
+ */
+public class EntityProcessorWrapper extends EntityProcessor {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private EntityProcessor delegate;
+ private Entity entity;
+ private DataSource datasource;
+ private List<EntityProcessorWrapper> children = new ArrayList<>();
+ private DocBuilder docBuilder;
+ private boolean initialized;
+ private String onError;
+ private Context context;
+ private VariableResolver resolver;
+ private String entityName;
+
+ protected List<Transformer> transformers;
+
+ protected List<Map<String, Object>> rowcache;
+
+ public EntityProcessorWrapper(EntityProcessor delegate, Entity entity, DocBuilder docBuilder) {
+ this.delegate = delegate;
+ this.entity = entity;
+ this.docBuilder = docBuilder;
+ }
+
+ @Override
+ public void init(Context context) {
+ rowcache = null;
+ this.context = context;
+ resolver = (VariableResolver) context.getVariableResolver();
+ if (entityName == null) {
+ onError = resolver.replaceTokens(context.getEntityAttribute(ON_ERROR));
+ if (onError == null) onError = ABORT;
+ entityName = context.getEntityAttribute(ConfigNameConstants.NAME);
+ }
+ delegate.init(context);
+
+ }
+
+ @SuppressWarnings("unchecked")
+ void loadTransformers() {
+ String transClasses = context.getEntityAttribute(TRANSFORMER);
+
+ if (transClasses == null) {
+ transformers = Collections.EMPTY_LIST;
+ return;
+ }
+
+ String[] transArr = transClasses.split(",");
+ transformers = new ArrayList<Transformer>() {
+ @Override
+ public boolean add(Transformer transformer) {
+ if (docBuilder != null && docBuilder.verboseDebug) {
+ transformer = docBuilder.getDebugLogger().wrapTransformer(transformer);
+ }
+ return super.add(transformer);
+ }
+ };
+ for (String aTransArr : transArr) {
+ String trans = aTransArr.trim();
+ if (trans.startsWith("script:")) {
+ // The script transformer is a potential vulnerability, esp. when the script is
+ // provided from an untrusted source. Check and don't proceed if source is untrusted.
+ checkIfTrusted(trans);
+ String functionName = trans.substring("script:".length());
+ ScriptTransformer scriptTransformer = new ScriptTransformer();
+ scriptTransformer.setFunctionName(functionName);
+ transformers.add(scriptTransformer);
+ continue;
+ }
+ try {
+ Class clazz = DocBuilder.loadClass(trans, context.getSolrCore());
+ if (Transformer.class.isAssignableFrom(clazz)) {
+ transformers.add((Transformer) clazz.newInstance());
+ } else {
+ Method meth = clazz.getMethod(TRANSFORM_ROW, Map.class);
+ transformers.add(new ReflectionTransformer(meth, clazz, trans));
+ }
+ } catch (NoSuchMethodException nsme){
+ String msg = "Transformer :"
+ + trans
+ + "does not implement Transformer interface or does not have a transformRow(Map<String.Object> m)method";
+ log.error(msg);
+ wrapAndThrow(SEVERE, nsme,msg);
+ } catch (Exception e) {
+ log.error("Unable to load Transformer: " + aTransArr, e);
+ wrapAndThrow(SEVERE, e,"Unable to load Transformer: " + trans);
+ }
+ }
+
+ }
+
+ private void checkIfTrusted(String trans) {
+ if (docBuilder != null) {
+ SolrCore core = docBuilder.dataImporter.getCore();
+ boolean trusted = (core != null)? core.getCoreDescriptor().isConfigSetTrusted(): true;
+ if (!trusted) {
+ Exception ex = new SolrException(ErrorCode.UNAUTHORIZED, "The configset for this collection was uploaded "
+ + "without any authentication in place,"
+ + " and this transformer is not available for collections with untrusted configsets. To use this transformer,"
+ + " re-upload the configset after enabling authentication and authorization.");
+ String msg = "Transformer: "
+ + trans
+ + ". " + ex.getMessage();
+ log.error(msg);
+ wrapAndThrow(SEVERE, ex, msg);
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ static class ReflectionTransformer extends Transformer {
+ final Method meth;
+
+ final Class clazz;
+
+ final String trans;
+
+ final Object o;
+
+ public ReflectionTransformer(Method meth, Class clazz, String trans)
+ throws Exception {
+ this.meth = meth;
+ this.clazz = clazz;
+ this.trans = trans;
+ o = clazz.newInstance();
+ }
+
+ @Override
+ public Object transformRow(Map<String, Object> aRow, Context context) {
+ try {
+ return meth.invoke(o, aRow);
+ } catch (Exception e) {
+ log.warn("method invocation failed on transformer : " + trans, e);
+ throw new DataImportHandlerException(WARN, e);
+ }
+ }
+ }
+
+ protected Map<String, Object> getFromRowCache() {
+ Map<String, Object> r = rowcache.remove(0);
+ if (rowcache.isEmpty())
+ rowcache = null;
+ return r;
+ }
+
+ @SuppressWarnings("unchecked")
+ protected Map<String, Object> applyTransformer(Map<String, Object> row) {
+ if(row == null) return null;
+ if (transformers == null)
+ loadTransformers();
+ if (transformers == Collections.EMPTY_LIST)
+ return row;
+ Map<String, Object> transformedRow = row;
+ List<Map<String, Object>> rows = null;
+ boolean stopTransform = checkStopTransform(row);
+ VariableResolver resolver = (VariableResolver) context.getVariableResolver();
+ for (Transformer t : transformers) {
+ if (stopTransform) break;
+ try {
+ if (rows != null) {
+ List<Map<String, Object>> tmpRows = new ArrayList<>();
+ for (Map<String, Object> map : rows) {
+ resolver.addNamespace(entityName, map);
+ Object o = t.transformRow(map, context);
+ if (o == null)
+ continue;
+ if (o instanceof Map) {
+ Map oMap = (Map) o;
+ stopTransform = checkStopTransform(oMap);
+ tmpRows.add((Map) o);
+ } else if (o instanceof List) {
+ tmpRows.addAll((List) o);
+ } else {
+ log.error("Transformer must return Map<String, Object> or a List<Map<String, Object>>");
+ }
+ }
+ rows = tmpRows;
+ } else {
+ resolver.addNamespace(entityName, transformedRow);
+ Object o = t.transformRow(transformedRow, context);
+ if (o == null)
+ return null;
+ if (o instanceof Map) {
+ Map oMap = (Map) o;
+ stopTransform = checkStopTransform(oMap);
+ transformedRow = (Map) o;
+ } else if (o instanceof List) {
+ rows = (List) o;
+ } else {
+ log.error("Transformer must return Map<String, Object> or a List<Map<String, Object>>");
+ }
+ }
+ } catch (Exception e) {
+ log.warn("transformer threw error", e);
+ if (ABORT.equals(onError)) {
+ wrapAndThrow(SEVERE, e);
+ } else if (SKIP.equals(onError)) {
+ wrapAndThrow(DataImportHandlerException.SKIP, e);
+ }
+ // onError = continue
+ }
+ }
+ if (rows == null) {
+ return transformedRow;
+ } else {
+ rowcache = rows;
+ return getFromRowCache();
+ }
+
+ }
+
+ private boolean checkStopTransform(Map oMap) {
+ return oMap.get("$stopTransform") != null
+ && Boolean.parseBoolean(oMap.get("$stopTransform").toString());
+ }
+
+ @Override
+ public Map<String, Object> nextRow() {
+ if (rowcache != null) {
+ return getFromRowCache();
+ }
+ while (true) {
+ Map<String, Object> arow = null;
+ try {
+ arow = delegate.nextRow();
+ } catch (Exception e) {
+ if(ABORT.equals(onError)){
+ wrapAndThrow(SEVERE, e);
+ } else {
+ //SKIP is not really possible. If this calls the nextRow() again the Entityprocessor would be in an inconisttent state
+ SolrException.log(log, "Exception in entity : "+ entityName, e);
+ return null;
+ }
+ }
+ if (arow == null) {
+ return null;
+ } else {
+ arow = applyTransformer(arow);
+ if (arow != null) {
+ delegate.postTransform(arow);
+ return arow;
+ }
+ }
+ }
+ }
+
+ @Override
+ public Map<String, Object> nextModifiedRowKey() {
+ Map<String, Object> row = delegate.nextModifiedRowKey();
+ row = applyTransformer(row);
+ rowcache = null;
+ return row;
+ }
+
+ @Override
+ public Map<String, Object> nextDeletedRowKey() {
+ Map<String, Object> row = delegate.nextDeletedRowKey();
+ row = applyTransformer(row);
+ rowcache = null;
+ return row;
+ }
+
+ @Override
+ public Map<String, Object> nextModifiedParentRowKey() {
+ return delegate.nextModifiedParentRowKey();
+ }
+
+ @Override
+ public void destroy() {
+ delegate.destroy();
+ }
+
+ public VariableResolver getVariableResolver() {
+ return (VariableResolver) context.getVariableResolver();
+ }
+
+ public Context getContext() {
+ return context;
+ }
+
+ @Override
+ public void close() {
+ delegate.close();
+ }
+
+ public Entity getEntity() {
+ return entity;
+ }
+
+ public List<EntityProcessorWrapper> getChildren() {
+ return children;
+ }
+
+ public DataSource getDatasource() {
+ return datasource;
+ }
+
+ public void setDatasource(DataSource datasource) {
+ this.datasource = datasource;
+ }
+
+ public boolean isInitialized() {
+ return initialized;
+ }
+
+ public void setInitialized(boolean initialized) {
+ this.initialized = initialized;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Evaluator.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Evaluator.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Evaluator.java
new file mode 100644
index 0000000..22282b9
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Evaluator.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+
+/**
+ * <p>
+ * Pluggable functions for resolving variables
+ * </p>
+ * <p>
+ * Implementations of this abstract class must provide a public no-arg constructor.
+ * </p>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 1.3
+ */
+public abstract class Evaluator {
+
+ /**
+ * Return a String after processing an expression and a {@link VariableResolver}
+ *
+ * @see VariableResolver
+ * @param expression string to be evaluated
+ * @param context instance
+ * @return the value of the given expression evaluated using the resolver
+ */
+ public abstract String evaluate(String expression, Context context);
+
+ /**
+ * Parses a string of expression into separate params. The values are separated by commas. each value will be
+ * translated into one of the following:
+ * <ol>
+ * <li>If it is in single quotes the value will be translated to a String</li>
+ * <li>If is is not in quotes and is a number a it will be translated into a Double</li>
+ * <li>else it is a variable which can be resolved and it will be put in as an instance of VariableWrapper</li>
+ * </ol>
+ *
+ * @param expression the expression to be parsed
+ * @param vr the VariableResolver instance for resolving variables
+ *
+ * @return a List of objects which can either be a string, number or a variable wrapper
+ */
+ protected List<Object> parseParams(String expression, VariableResolver vr) {
+ List<Object> result = new ArrayList<>();
+ expression = expression.trim();
+ String[] ss = expression.split(",");
+ for (int i = 0; i < ss.length; i++) {
+ ss[i] = ss[i].trim();
+ if (ss[i].startsWith("'")) {//a string param has started
+ StringBuilder sb = new StringBuilder();
+ while (true) {
+ sb.append(ss[i]);
+ if (ss[i].endsWith("'")) break;
+ i++;
+ if (i >= ss.length)
+ throw new DataImportHandlerException(SEVERE, "invalid string at " + ss[i - 1] + " in function params: " + expression);
+ sb.append(",");
+ }
+ String s = sb.substring(1, sb.length() - 1);
+ s = s.replaceAll("\\\\'", "'");
+ result.add(s);
+ } else {
+ if (Character.isDigit(ss[i].charAt(0))) {
+ try {
+ Double doub = Double.parseDouble(ss[i]);
+ result.add(doub);
+ } catch (NumberFormatException e) {
+ if (vr.resolve(ss[i]) == null) {
+ wrapAndThrow(
+ SEVERE, e, "Invalid number :" + ss[i] +
+ "in parameters " + expression);
+ }
+ }
+ } else {
+ result.add(getVariableWrapper(ss[i], vr));
+ }
+ }
+ }
+ return result;
+ }
+
+ protected VariableWrapper getVariableWrapper(String s, VariableResolver vr) {
+ return new VariableWrapper(s,vr);
+ }
+
+ static protected class VariableWrapper {
+ public final String varName;
+ public final VariableResolver vr;
+
+ public VariableWrapper(String s, VariableResolver vr) {
+ this.varName = s;
+ this.vr = vr;
+ }
+
+ public Object resolve() {
+ return vr.resolve(varName);
+ }
+
+ @Override
+ public String toString() {
+ Object o = vr.resolve(varName);
+ return o == null ? null : o.toString();
+ }
+ }
+
+ static Pattern IN_SINGLE_QUOTES = Pattern.compile("^'(.*?)'$");
+
+ public static final String DATE_FORMAT_EVALUATOR = "formatDate";
+
+ public static final String URL_ENCODE_EVALUATOR = "encodeUrl";
+
+ public static final String ESCAPE_SOLR_QUERY_CHARS = "escapeQueryChars";
+
+ public static final String SQL_ESCAPE_EVALUATOR = "escapeSql";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EventListener.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EventListener.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EventListener.java
new file mode 100644
index 0000000..0c43a0b
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EventListener.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+/**
+ * Event listener for DataImportHandler
+ *
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @since solr 1.4
+ */
+public interface EventListener {
+
+ /**
+ * Event callback
+ *
+ * @param ctx the Context in which this event was called
+ */
+ void onEvent(Context ctx);
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java
new file mode 100644
index 0000000..571c280
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.sql.Blob;
+import java.sql.Clob;
+import java.sql.SQLException;
+import java.util.Properties;
+
+/**
+ * This can be useful for users who have a DB field containing xml and wish to use a nested {@link XPathEntityProcessor}
+ * <p>
+ * The datasouce may be configured as follows
+ * <p>
+ * <datasource name="f1" type="FieldReaderDataSource" />
+ * <p>
+ * The entity which uses this datasource must keep the url value as the variable name url="field-name"
+ * <p>
+ * The fieldname must be resolvable from {@link VariableResolver}
+ * <p>
+ * This may be used with any {@link EntityProcessor} which uses a {@link DataSource}<{@link Reader}> eg: {@link XPathEntityProcessor}
+ * <p>
+ * Supports String, BLOB, CLOB data types and there is an extra field (in the entity) 'encoding' for BLOB types
+ *
+ * @since 1.4
+ */
+public class FieldReaderDataSource extends DataSource<Reader> {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ protected VariableResolver vr;
+ protected String dataField;
+ private String encoding;
+ private EntityProcessorWrapper entityProcessor;
+
+ @Override
+ public void init(Context context, Properties initProps) {
+ dataField = context.getEntityAttribute("dataField");
+ encoding = context.getEntityAttribute("encoding");
+ entityProcessor = (EntityProcessorWrapper) context.getEntityProcessor();
+ /*no op*/
+ }
+
+ @Override
+ public Reader getData(String query) {
+ Object o = entityProcessor.getVariableResolver().resolve(dataField);
+ if (o == null) {
+ throw new DataImportHandlerException (SEVERE, "No field available for name : " +dataField);
+ }
+ if (o instanceof String) {
+ return new StringReader((String) o);
+ } else if (o instanceof Clob) {
+ Clob clob = (Clob) o;
+ try {
+ //Most of the JDBC drivers have getCharacterStream defined as public
+ // so let us just check it
+ return readCharStream(clob);
+ } catch (Exception e) {
+ log.info("Unable to get data from CLOB");
+ return null;
+
+ }
+
+ } else if (o instanceof Blob) {
+ Blob blob = (Blob) o;
+ try {
+ return getReader(blob);
+ } catch (Exception e) {
+ log.info("Unable to get data from BLOB");
+ return null;
+
+ }
+ } else {
+ return new StringReader(o.toString());
+ }
+
+ }
+
+ static Reader readCharStream(Clob clob) {
+ try {
+ return clob.getCharacterStream();
+ } catch (Exception e) {
+ wrapAndThrow(SEVERE, e,"Unable to get reader from clob");
+ return null;//unreachable
+ }
+ }
+
+ private Reader getReader(Blob blob)
+ throws SQLException, UnsupportedEncodingException {
+ if (encoding == null) {
+ return (new InputStreamReader(blob.getBinaryStream(), StandardCharsets.UTF_8));
+ } else {
+ return (new InputStreamReader(blob.getBinaryStream(), encoding));
+ }
+ }
+
+ @Override
+ public void close() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FieldStreamDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FieldStreamDataSource.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FieldStreamDataSource.java
new file mode 100644
index 0000000..ba7ca5d
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FieldStreamDataSource.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.lang.invoke.MethodHandles;
+import java.sql.Blob;
+import java.sql.SQLException;
+import java.util.Properties;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * This can be useful for users who have a DB field containing BLOBs which may be Rich documents
+ * <p>
+ * The datasource may be configured as follows
+ * <p>
+ * <dataSource name="f1" type="FieldStreamDataSource" />
+ * <p>
+ * The entity which uses this datasource must keep and attribute dataField
+ * <p>
+ * The fieldname must be resolvable from {@link VariableResolver}
+ * <p>
+ * This may be used with any {@link EntityProcessor} which uses a {@link DataSource}<{@link InputStream}> eg: TikaEntityProcessor
+ *
+ * @since 3.1
+ */
+public class FieldStreamDataSource extends DataSource<InputStream> {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ protected VariableResolver vr;
+ protected String dataField;
+ private EntityProcessorWrapper wrapper;
+
+ @Override
+ public void init(Context context, Properties initProps) {
+ dataField = context.getEntityAttribute("dataField");
+ wrapper = (EntityProcessorWrapper) context.getEntityProcessor();
+ /*no op*/
+ }
+
+ @Override
+ public InputStream getData(String query) {
+ Object o = wrapper.getVariableResolver().resolve(dataField);
+ if (o == null) {
+ throw new DataImportHandlerException(SEVERE, "No field available for name : " + dataField);
+ } else if (o instanceof Blob) {
+ Blob blob = (Blob) o;
+ try {
+ return blob.getBinaryStream();
+ } catch (SQLException sqle) {
+ log.info("Unable to get data from BLOB");
+ return null;
+ }
+ } else if (o instanceof byte[]) {
+ byte[] bytes = (byte[]) o;
+ return new ByteArrayInputStream(bytes);
+ } else {
+ throw new RuntimeException("unsupported type : " + o.getClass());
+ }
+
+ }
+
+ @Override
+ public void close() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FileDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FileDataSource.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FileDataSource.java
new file mode 100644
index 0000000..920472e
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FileDataSource.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.*;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.util.Properties;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+
+/**
+ * <p>
+ * A {@link DataSource} which reads from local files
+ * </p>
+ * <p>
+ * The file is read with the default platform encoding. It can be overriden by
+ * specifying the encoding in solrconfig.xml
+ * </p>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 1.3
+ */
+public class FileDataSource extends DataSource<Reader> {
+ public static final String BASE_PATH = "basePath";
+
+ /**
+ * The basePath for this data source
+ */
+ protected String basePath;
+
+ /**
+ * The encoding using which the given file should be read
+ */
+ protected String encoding = null;
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @Override
+ public void init(Context context, Properties initProps) {
+ basePath = initProps.getProperty(BASE_PATH);
+ if (initProps.get(URLDataSource.ENCODING) != null)
+ encoding = initProps.getProperty(URLDataSource.ENCODING);
+ }
+
+ /**
+ * <p>
+ * Returns a reader for the given file.
+ * </p>
+ * <p>
+ * If the given file is not absolute, we try to construct an absolute path
+ * using basePath configuration. If that fails, then the relative path is
+ * tried. If file is not found a RuntimeException is thrown.
+ * </p>
+ * <p>
+ * <b>It is the responsibility of the calling method to properly close the
+ * returned Reader</b>
+ * </p>
+ */
+ @Override
+ public Reader getData(String query) {
+ File f = getFile(basePath,query);
+ try {
+ return openStream(f);
+ } catch (Exception e) {
+ wrapAndThrow(SEVERE,e,"Unable to open File : "+f.getAbsolutePath());
+ return null;
+ }
+ }
+
+ static File getFile(String basePath, String query) {
+ try {
+ File file = new File(query);
+
+ // If it's not an absolute path, try relative from basePath.
+ if (!file.isAbsolute()) {
+ // Resolve and correct basePath.
+ File basePathFile;
+ if (basePath == null) {
+ basePathFile = new File(".").getAbsoluteFile();
+ log.warn("FileDataSource.basePath is empty. " +
+ "Resolving to: " + basePathFile.getAbsolutePath());
+ } else {
+ basePathFile = new File(basePath);
+ if (!basePathFile.isAbsolute()) {
+ basePathFile = basePathFile.getAbsoluteFile();
+ log.warn("FileDataSource.basePath is not absolute. Resolving to: "
+ + basePathFile.getAbsolutePath());
+ }
+ }
+
+ file = new File(basePathFile, query).getAbsoluteFile();
+ }
+
+ if (file.isFile() && file.canRead()) {
+ log.debug("Accessing File: " + file.getAbsolutePath());
+ return file;
+ } else {
+ throw new FileNotFoundException("Could not find file: " + query +
+ " (resolved to: " + file.getAbsolutePath());
+ }
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Open a {@link java.io.Reader} for the given file name
+ *
+ * @param file a {@link java.io.File} instance
+ * @return a Reader on the given file
+ * @throws FileNotFoundException if the File does not exist
+ * @throws UnsupportedEncodingException if the encoding is unsupported
+ * @since solr 1.4
+ */
+ protected Reader openStream(File file) throws FileNotFoundException,
+ UnsupportedEncodingException {
+ if (encoding == null) {
+ return new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8);
+ } else {
+ return new InputStreamReader(new FileInputStream(file), encoding);
+ }
+ }
+
+ @Override
+ public void close() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java
new file mode 100644
index 0000000..a03354f
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.solr.util.DateMathParser;
+
+/**
+ * <p>
+ * An {@link EntityProcessor} instance which can stream file names found in a given base
+ * directory matching patterns and returning rows containing file information.
+ * </p>
+ * <p>
+ * It supports querying a give base directory by matching:
+ * <ul>
+ * <li>regular expressions to file names</li>
+ * <li>excluding certain files based on regular expression</li>
+ * <li>last modification date (newer or older than a given date or time)</li>
+ * <li>size (bigger or smaller than size given in bytes)</li>
+ * <li>recursively iterating through sub-directories</li>
+ * </ul>
+ * Its output can be used along with {@link FileDataSource} to read from files in file
+ * systems.
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 1.3
+ * @see Pattern
+ */
+public class FileListEntityProcessor extends EntityProcessorBase {
+ /**
+ * A regex pattern to identify files given in data-config.xml after resolving any variables
+ */
+ protected String fileName;
+
+ /**
+ * The baseDir given in data-config.xml after resolving any variables
+ */
+ protected String baseDir;
+
+ /**
+ * A Regex pattern of excluded file names as given in data-config.xml after resolving any variables
+ */
+ protected String excludes;
+
+ /**
+ * The newerThan given in data-config as a {@link java.util.Date}
+ * <p>
+ * <b>Note: </b> This variable is resolved just-in-time in the {@link #nextRow()} method.
+ * </p>
+ */
+ protected Date newerThan;
+
+ /**
+ * The newerThan given in data-config as a {@link java.util.Date}
+ */
+ protected Date olderThan;
+
+ /**
+ * The biggerThan given in data-config as a long value
+ * <p>
+ * <b>Note: </b> This variable is resolved just-in-time in the {@link #nextRow()} method.
+ * </p>
+ */
+ protected long biggerThan = -1;
+
+ /**
+ * The smallerThan given in data-config as a long value
+ * <p>
+ * <b>Note: </b> This variable is resolved just-in-time in the {@link #nextRow()} method.
+ * </p>
+ */
+ protected long smallerThan = -1;
+
+ /**
+ * The recursive given in data-config. Default value is false.
+ */
+ protected boolean recursive = false;
+
+ private Pattern fileNamePattern, excludesPattern;
+
+ @Override
+ public void init(Context context) {
+ super.init(context);
+ fileName = context.getEntityAttribute(FILE_NAME);
+ if (fileName != null) {
+ fileName = context.replaceTokens(fileName);
+ fileNamePattern = Pattern.compile(fileName);
+ }
+ baseDir = context.getEntityAttribute(BASE_DIR);
+ if (baseDir == null)
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "'baseDir' is a required attribute");
+ baseDir = context.replaceTokens(baseDir);
+ File dir = new File(baseDir);
+ if (!dir.isDirectory())
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "'baseDir' value: " + baseDir + " is not a directory");
+
+ String r = context.getEntityAttribute(RECURSIVE);
+ if (r != null)
+ recursive = Boolean.parseBoolean(r);
+ excludes = context.getEntityAttribute(EXCLUDES);
+ if (excludes != null) {
+ excludes = context.replaceTokens(excludes);
+ excludesPattern = Pattern.compile(excludes);
+ }
+ }
+
+ /**
+ * Get the Date object corresponding to the given string.
+ *
+ * @param dateStr the date string. It can be a DateMath string or it may have a evaluator function
+ * @return a Date instance corresponding to the input string
+ */
+ private Date getDate(String dateStr) {
+ if (dateStr == null)
+ return null;
+
+ Matcher m = PLACE_HOLDER_PATTERN.matcher(dateStr);
+ if (m.find()) {
+ Object o = context.resolve(m.group(1));
+ if (o instanceof Date) return (Date)o;
+ dateStr = (String) o;
+ } else {
+ dateStr = context.replaceTokens(dateStr);
+ }
+ m = Evaluator.IN_SINGLE_QUOTES.matcher(dateStr);
+ if (m.find()) {
+ String expr = m.group(1);
+ //TODO refactor DateMathParser.parseMath a bit to have a static method for this logic.
+ if (expr.startsWith("NOW")) {
+ expr = expr.substring("NOW".length());
+ }
+ try {
+ // DWS TODO: is this TimeZone the right default for us? Deserves explanation if so.
+ return new DateMathParser(TimeZone.getDefault()).parseMath(expr);
+ } catch (ParseException exp) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Invalid expression for date", exp);
+ }
+ }
+ try {
+ return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).parse(dateStr);
+ } catch (ParseException exp) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Invalid expression for date", exp);
+ }
+ }
+
+ /**
+ * Get the Long value for the given string after resolving any evaluator or variable.
+ *
+ * @param sizeStr the size as a string
+ * @return the Long value corresponding to the given string
+ */
+ private Long getSize(String sizeStr) {
+ if (sizeStr == null)
+ return null;
+
+ Matcher m = PLACE_HOLDER_PATTERN.matcher(sizeStr);
+ if (m.find()) {
+ Object o = context.resolve(m.group(1));
+ if (o instanceof Number) {
+ Number number = (Number) o;
+ return number.longValue();
+ }
+ sizeStr = (String) o;
+ } else {
+ sizeStr = context.replaceTokens(sizeStr);
+ }
+
+ return Long.parseLong(sizeStr);
+ }
+
+ @Override
+ public Map<String, Object> nextRow() {
+ if (rowIterator != null)
+ return getNext();
+ List<Map<String, Object>> fileDetails = new ArrayList<>();
+ File dir = new File(baseDir);
+
+ String dateStr = context.getEntityAttribute(NEWER_THAN);
+ newerThan = getDate(dateStr);
+ dateStr = context.getEntityAttribute(OLDER_THAN);
+ olderThan = getDate(dateStr);
+ String biggerThanStr = context.getEntityAttribute(BIGGER_THAN);
+ if (biggerThanStr != null)
+ biggerThan = getSize(biggerThanStr);
+ String smallerThanStr = context.getEntityAttribute(SMALLER_THAN);
+ if (smallerThanStr != null)
+ smallerThan = getSize(smallerThanStr);
+
+ getFolderFiles(dir, fileDetails);
+ rowIterator = fileDetails.iterator();
+ return getNext();
+ }
+
+ private void getFolderFiles(File dir, final List<Map<String, Object>> fileDetails) {
+ // Fetch an array of file objects that pass the filter, however the
+ // returned array is never populated; accept() always returns false.
+ // Rather we make use of the fileDetails array which is populated as
+ // a side affect of the accept method.
+ dir.list(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ File fileObj = new File(dir, name);
+ if (fileObj.isDirectory()) {
+ if (recursive) getFolderFiles(fileObj, fileDetails);
+ } else if (fileNamePattern == null) {
+ addDetails(fileDetails, dir, name);
+ } else if (fileNamePattern.matcher(name).find()) {
+ if (excludesPattern != null && excludesPattern.matcher(name).find())
+ return false;
+ addDetails(fileDetails, dir, name);
+ }
+ return false;
+ }
+ });
+ }
+
+ private void addDetails(List<Map<String, Object>> files, File dir, String name) {
+ Map<String, Object> details = new HashMap<>();
+ File aFile = new File(dir, name);
+ if (aFile.isDirectory()) return;
+ long sz = aFile.length();
+ Date lastModified = new Date(aFile.lastModified());
+ if (biggerThan != -1 && sz <= biggerThan)
+ return;
+ if (smallerThan != -1 && sz >= smallerThan)
+ return;
+ if (olderThan != null && lastModified.after(olderThan))
+ return;
+ if (newerThan != null && lastModified.before(newerThan))
+ return;
+ details.put(DIR, dir.getAbsolutePath());
+ details.put(FILE, name);
+ details.put(ABSOLUTE_FILE, aFile.getAbsolutePath());
+ details.put(SIZE, sz);
+ details.put(LAST_MODIFIED, lastModified);
+ files.add(details);
+ }
+
+ public static final Pattern PLACE_HOLDER_PATTERN = Pattern
+ .compile("\\$\\{(.*?)\\}");
+
+ public static final String DIR = "fileDir";
+
+ public static final String FILE = "file";
+
+ public static final String ABSOLUTE_FILE = "fileAbsolutePath";
+
+ public static final String SIZE = "fileSize";
+
+ public static final String LAST_MODIFIED = "fileLastModified";
+
+ public static final String FILE_NAME = "fileName";
+
+ public static final String BASE_DIR = "baseDir";
+
+ public static final String EXCLUDES = "excludes";
+
+ public static final String NEWER_THAN = "newerThan";
+
+ public static final String OLDER_THAN = "olderThan";
+
+ public static final String BIGGER_THAN = "biggerThan";
+
+ public static final String SMALLER_THAN = "smallerThan";
+
+ public static final String RECURSIVE = "recursive";
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java
new file mode 100644
index 0000000..e62c329
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.lucene.analysis.charfilter.HTMLStripCharFilter;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.BufferedReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A {@link Transformer} implementation which strip off HTML tags using {@link HTMLStripCharFilter} This is useful
+ * in case you don't need this HTML anyway.
+ *
+ * @see HTMLStripCharFilter
+ * @since solr 1.4
+ */
+public class HTMLStripTransformer extends Transformer {
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Object transformRow(Map<String, Object> row, Context context) {
+ List<Map<String, String>> fields = context.getAllEntityFields();
+ for (Map<String, String> field : fields) {
+ String col = field.get(DataImporter.COLUMN);
+ String splitHTML = context.replaceTokens(field.get(STRIP_HTML));
+ if (!TRUE.equals(splitHTML))
+ continue;
+ Object tmpVal = row.get(col);
+ if (tmpVal == null)
+ continue;
+
+ if (tmpVal instanceof List) {
+ List<String> inputs = (List<String>) tmpVal;
+ List results = new ArrayList();
+ for (String input : inputs) {
+ if (input == null)
+ continue;
+ Object o = stripHTML(input, col);
+ if (o != null)
+ results.add(o);
+ }
+ row.put(col, results);
+ } else {
+ String value = tmpVal.toString();
+ Object o = stripHTML(value, col);
+ if (o != null)
+ row.put(col, o);
+ }
+ }
+ return row;
+ }
+
+ private Object stripHTML(String value, String column) {
+ StringBuilder out = new StringBuilder();
+ StringReader strReader = new StringReader(value);
+ try {
+ HTMLStripCharFilter html = new HTMLStripCharFilter(strReader.markSupported() ? strReader : new BufferedReader(strReader));
+ char[] cbuf = new char[1024 * 10];
+ while (true) {
+ int count = html.read(cbuf);
+ if (count == -1)
+ break; // end of stream mark is -1
+ if (count > 0)
+ out.append(cbuf, 0, count);
+ }
+ html.close();
+ } catch (IOException e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Failed stripping HTML for column: " + column, e);
+ }
+ return out.toString();
+ }
+
+ public static final String STRIP_HTML = "stripHTML";
+
+ public static final String TRUE = "true";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/JdbcDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/JdbcDataSource.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/JdbcDataSource.java
new file mode 100644
index 0000000..a8eed55
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/JdbcDataSource.java
@@ -0,0 +1,593 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.util.CryptoKeys;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.naming.InitialContext;
+import javax.naming.NamingException;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.lang.invoke.MethodHandles;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.sql.*;
+import java.util.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * <p> A DataSource implementation which can fetch data using JDBC. </p> <p> Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a> for more
+ * details. </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 1.3
+ */
+public class JdbcDataSource extends
+ DataSource<Iterator<Map<String, Object>>> {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ protected Callable<Connection> factory;
+
+ private long connLastUsed = 0;
+
+ private Connection conn;
+
+ private ResultSetIterator resultSetIterator;
+
+ private Map<String, Integer> fieldNameVsType = new HashMap<>();
+
+ private boolean convertType = false;
+
+ private int batchSize = FETCH_SIZE;
+
+ private int maxRows = 0;
+
+ @Override
+ public void init(Context context, Properties initProps) {
+ resolveVariables(context, initProps);
+ initProps = decryptPwd(context, initProps);
+ Object o = initProps.get(CONVERT_TYPE);
+ if (o != null)
+ convertType = Boolean.parseBoolean(o.toString());
+
+ factory = createConnectionFactory(context, initProps);
+
+ String bsz = initProps.getProperty("batchSize");
+ if (bsz != null) {
+ bsz = context.replaceTokens(bsz);
+ try {
+ batchSize = Integer.parseInt(bsz);
+ if (batchSize == -1)
+ batchSize = Integer.MIN_VALUE;
+ } catch (NumberFormatException e) {
+ log.warn("Invalid batch size: " + bsz);
+ }
+ }
+
+ for (Map<String, String> map : context.getAllEntityFields()) {
+ String n = map.get(DataImporter.COLUMN);
+ String t = map.get(DataImporter.TYPE);
+ if ("sint".equals(t) || "integer".equals(t))
+ fieldNameVsType.put(n, Types.INTEGER);
+ else if ("slong".equals(t) || "long".equals(t))
+ fieldNameVsType.put(n, Types.BIGINT);
+ else if ("float".equals(t) || "sfloat".equals(t))
+ fieldNameVsType.put(n, Types.FLOAT);
+ else if ("double".equals(t) || "sdouble".equals(t))
+ fieldNameVsType.put(n, Types.DOUBLE);
+ else if ("date".equals(t))
+ fieldNameVsType.put(n, Types.DATE);
+ else if ("boolean".equals(t))
+ fieldNameVsType.put(n, Types.BOOLEAN);
+ else if ("binary".equals(t))
+ fieldNameVsType.put(n, Types.BLOB);
+ else
+ fieldNameVsType.put(n, Types.VARCHAR);
+ }
+ }
+
+ private Properties decryptPwd(Context context, Properties initProps) {
+ String encryptionKey = initProps.getProperty("encryptKeyFile");
+ if (initProps.getProperty("password") != null && encryptionKey != null) {
+ // this means the password is encrypted and use the file to decode it
+ try {
+ try (Reader fr = new InputStreamReader(new FileInputStream(encryptionKey), UTF_8)) {
+ char[] chars = new char[100];//max 100 char password
+ int len = fr.read(chars);
+ if (len < 6)
+ throw new DataImportHandlerException(SEVERE, "There should be a password of length 6 atleast " + encryptionKey);
+ Properties props = new Properties();
+ props.putAll(initProps);
+ String password = null;
+ try {
+ password = CryptoKeys.decodeAES(initProps.getProperty("password"), new String(chars, 0, len)).trim();
+ } catch (SolrException se) {
+ throw new DataImportHandlerException(SEVERE, "Error decoding password", se.getCause());
+ }
+ props.put("password", password);
+ initProps = props;
+ }
+ } catch (IOException e) {
+ throw new DataImportHandlerException(SEVERE, "Could not load encryptKeyFile " + encryptionKey);
+ }
+ }
+ return initProps;
+ }
+
+ protected Callable<Connection> createConnectionFactory(final Context context,
+ final Properties initProps) {
+// final VariableResolver resolver = context.getVariableResolver();
+ final String jndiName = initProps.getProperty(JNDI_NAME);
+ final String url = initProps.getProperty(URL);
+ final String driver = initProps.getProperty(DRIVER);
+
+ if (url == null && jndiName == null)
+ throw new DataImportHandlerException(SEVERE,
+ "JDBC URL or JNDI name has to be specified");
+
+ if (driver != null) {
+ try {
+ DocBuilder.loadClass(driver, context.getSolrCore());
+ } catch (ClassNotFoundException e) {
+ wrapAndThrow(SEVERE, e, "Could not load driver: " + driver);
+ }
+ } else {
+ if(jndiName == null){
+ throw new DataImportHandlerException(SEVERE, "One of driver or jndiName must be specified in the data source");
+ }
+ }
+
+ String s = initProps.getProperty("maxRows");
+ if (s != null) {
+ maxRows = Integer.parseInt(s);
+ }
+
+ return factory = new Callable<Connection>() {
+ @Override
+ public Connection call() throws Exception {
+ log.info("Creating a connection for entity "
+ + context.getEntityAttribute(DataImporter.NAME) + " with URL: "
+ + url);
+ long start = System.nanoTime();
+ Connection c = null;
+
+ if (jndiName != null) {
+ c = getFromJndi(initProps, jndiName);
+ } else if (url != null) {
+ try {
+ c = DriverManager.getConnection(url, initProps);
+ } catch (SQLException e) {
+ // DriverManager does not allow you to use a driver which is not loaded through
+ // the class loader of the class which is trying to make the connection.
+ // This is a workaround for cases where the user puts the driver jar in the
+ // solr.home/lib or solr.home/core/lib directories.
+ Driver d = (Driver) DocBuilder.loadClass(driver, context.getSolrCore()).newInstance();
+ c = d.connect(url, initProps);
+ }
+ }
+ if (c != null) {
+ try {
+ initializeConnection(c, initProps);
+ } catch (SQLException e) {
+ try {
+ c.close();
+ } catch (SQLException e2) {
+ log.warn("Exception closing connection during cleanup", e2);
+ }
+
+ throw new DataImportHandlerException(SEVERE, "Exception initializing SQL connection", e);
+ }
+ }
+ log.info("Time taken for getConnection(): "
+ + TimeUnit.MILLISECONDS.convert(System.nanoTime() - start, TimeUnit.NANOSECONDS));
+ return c;
+ }
+
+ private void initializeConnection(Connection c, final Properties initProps)
+ throws SQLException {
+ if (Boolean.parseBoolean(initProps.getProperty("readOnly"))) {
+ c.setReadOnly(true);
+ // Add other sane defaults
+ c.setAutoCommit(true);
+ c.setTransactionIsolation(Connection.TRANSACTION_READ_UNCOMMITTED);
+ c.setHoldability(ResultSet.CLOSE_CURSORS_AT_COMMIT);
+ }
+ if (!Boolean.parseBoolean(initProps.getProperty("autoCommit"))) {
+ c.setAutoCommit(false);
+ }
+ String transactionIsolation = initProps.getProperty("transactionIsolation");
+ if ("TRANSACTION_READ_UNCOMMITTED".equals(transactionIsolation)) {
+ c.setTransactionIsolation(Connection.TRANSACTION_READ_UNCOMMITTED);
+ } else if ("TRANSACTION_READ_COMMITTED".equals(transactionIsolation)) {
+ c.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED);
+ } else if ("TRANSACTION_REPEATABLE_READ".equals(transactionIsolation)) {
+ c.setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ);
+ } else if ("TRANSACTION_SERIALIZABLE".equals(transactionIsolation)) {
+ c.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE);
+ } else if ("TRANSACTION_NONE".equals(transactionIsolation)) {
+ c.setTransactionIsolation(Connection.TRANSACTION_NONE);
+ }
+ String holdability = initProps.getProperty("holdability");
+ if ("CLOSE_CURSORS_AT_COMMIT".equals(holdability)) {
+ c.setHoldability(ResultSet.CLOSE_CURSORS_AT_COMMIT);
+ } else if ("HOLD_CURSORS_OVER_COMMIT".equals(holdability)) {
+ c.setHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT);
+ }
+ }
+
+ private Connection getFromJndi(final Properties initProps, final String jndiName) throws NamingException,
+ SQLException {
+
+ Connection c = null;
+ InitialContext ctx = new InitialContext();
+ Object jndival = ctx.lookup(jndiName);
+ if (jndival instanceof javax.sql.DataSource) {
+ javax.sql.DataSource dataSource = (javax.sql.DataSource) jndival;
+ String user = (String) initProps.get("user");
+ String pass = (String) initProps.get("password");
+ if(user == null || user.trim().equals("")){
+ c = dataSource.getConnection();
+ } else {
+ c = dataSource.getConnection(user, pass);
+ }
+ } else {
+ throw new DataImportHandlerException(SEVERE,
+ "the jndi name : '"+jndiName +"' is not a valid javax.sql.DataSource");
+ }
+ return c;
+ }
+ };
+ }
+
+ private void resolveVariables(Context ctx, Properties initProps) {
+ for (Map.Entry<Object, Object> entry : initProps.entrySet()) {
+ if (entry.getValue() != null) {
+ entry.setValue(ctx.replaceTokens((String) entry.getValue()));
+ }
+ }
+ }
+
+ @Override
+ public Iterator<Map<String, Object>> getData(String query) {
+ if (resultSetIterator != null) {
+ resultSetIterator.close();
+ resultSetIterator = null;
+ }
+ resultSetIterator = createResultSetIterator(query);
+ return resultSetIterator.getIterator();
+ }
+
+ protected ResultSetIterator createResultSetIterator(String query) {
+ return new ResultSetIterator(query);
+ }
+
+ private void logError(String msg, Exception e) {
+ log.warn(msg, e);
+ }
+
+ protected List<String> readFieldNames(ResultSetMetaData metaData)
+ throws SQLException {
+ List<String> colNames = new ArrayList<>();
+ int count = metaData.getColumnCount();
+ for (int i = 0; i < count; i++) {
+ colNames.add(metaData.getColumnLabel(i + 1));
+ }
+ return colNames;
+ }
+
+ protected class ResultSetIterator {
+ private ResultSet resultSet;
+
+ private Statement stmt = null;
+
+ private List<String> colNames;
+
+ private Iterator<Map<String, Object>> rSetIterator;
+
+ public ResultSetIterator(String query) {
+
+ try {
+ Connection c = getConnection();
+ stmt = createStatement(c, batchSize, maxRows);
+ log.debug("Executing SQL: " + query);
+ long start = System.nanoTime();
+ resultSet = executeStatement(stmt, query);
+ log.trace("Time taken for sql :"
+ + TimeUnit.MILLISECONDS.convert(System.nanoTime() - start, TimeUnit.NANOSECONDS));
+ setColNames(resultSet);
+ } catch (Exception e) {
+ close();
+ wrapAndThrow(SEVERE, e, "Unable to execute query: " + query);
+ return;
+ }
+ if (resultSet == null) {
+ close();
+ rSetIterator = new ArrayList<Map<String, Object>>().iterator();
+ return;
+ }
+
+ rSetIterator = createIterator(convertType, fieldNameVsType);
+ }
+
+
+ protected Statement createStatement(final Connection c, final int batchSize, final int maxRows)
+ throws SQLException {
+ Statement statement = c.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ statement.setFetchSize(batchSize);
+ statement.setMaxRows(maxRows);
+ return statement;
+ }
+
+ protected ResultSet executeStatement(Statement statement, String query) throws SQLException {
+ boolean resultSetReturned = statement.execute(query);
+ return getNextResultSet(resultSetReturned, statement);
+ }
+
+ protected ResultSet getNextResultSet(final boolean initialResultSetAvailable, final Statement statement) throws SQLException {
+ boolean resultSetAvailable = initialResultSetAvailable;
+ while (!resultSetAvailable && statement.getUpdateCount() != -1) {
+ resultSetAvailable = statement.getMoreResults();
+ }
+ if (resultSetAvailable) {
+ return statement.getResultSet();
+ }
+ return null;
+ }
+
+ protected void setColNames(final ResultSet resultSet) throws SQLException {
+ if (resultSet != null) {
+ colNames = readFieldNames(resultSet.getMetaData());
+ } else {
+ colNames = Collections.emptyList();
+ }
+ }
+
+ protected Iterator<Map<String,Object>> createIterator(final boolean convertType,
+ final Map<String,Integer> fieldNameVsType) {
+ return new Iterator<Map<String,Object>>() {
+ @Override
+ public boolean hasNext() {
+ return hasnext();
+ }
+
+ @Override
+ public Map<String,Object> next() {
+ return getARow(convertType, fieldNameVsType);
+ }
+
+ @Override
+ public void remove() {/* do nothing */
+ }
+ };
+ }
+
+
+
+ protected Map<String,Object> getARow(boolean convertType, Map<String,Integer> fieldNameVsType) {
+ if (getResultSet() == null)
+ return null;
+ Map<String, Object> result = new HashMap<>();
+ for (String colName : getColNames()) {
+ try {
+ if (!convertType) {
+ // Use underlying database's type information except for BigDecimal and BigInteger
+ // which cannot be serialized by JavaBin/XML. See SOLR-6165
+ Object value = getResultSet().getObject(colName);
+ if (value instanceof BigDecimal || value instanceof BigInteger) {
+ result.put(colName, value.toString());
+ } else {
+ result.put(colName, value);
+ }
+ continue;
+ }
+
+ Integer type = fieldNameVsType.get(colName);
+ if (type == null)
+ type = Types.VARCHAR;
+ switch (type) {
+ case Types.INTEGER:
+ result.put(colName, getResultSet().getInt(colName));
+ break;
+ case Types.FLOAT:
+ result.put(colName, getResultSet().getFloat(colName));
+ break;
+ case Types.BIGINT:
+ result.put(colName, getResultSet().getLong(colName));
+ break;
+ case Types.DOUBLE:
+ result.put(colName, getResultSet().getDouble(colName));
+ break;
+ case Types.DATE:
+ result.put(colName, getResultSet().getTimestamp(colName));
+ break;
+ case Types.BOOLEAN:
+ result.put(colName, getResultSet().getBoolean(colName));
+ break;
+ case Types.BLOB:
+ result.put(colName, getResultSet().getBytes(colName));
+ break;
+ default:
+ result.put(colName, getResultSet().getString(colName));
+ break;
+ }
+ } catch (SQLException e) {
+ logError("Error reading data ", e);
+ wrapAndThrow(SEVERE, e, "Error reading data from database");
+ }
+ }
+ return result;
+ }
+
+ protected boolean hasnext() {
+ if (getResultSet() == null) {
+ close();
+ return false;
+ }
+ try {
+ if (getResultSet().next()) {
+ return true;
+ } else {
+ closeResultSet();
+ setResultSet(getNextResultSet(getStatement().getMoreResults(), getStatement()));
+ setColNames(getResultSet());
+ return hasnext();
+ }
+ } catch (SQLException e) {
+ close();
+ wrapAndThrow(SEVERE,e);
+ return false;
+ }
+ }
+
+ protected void close() {
+ closeResultSet();
+ try {
+ if (getStatement() != null)
+ getStatement().close();
+ } catch (Exception e) {
+ logError("Exception while closing statement", e);
+ } finally {
+ setStatement(null);
+ }
+ }
+
+ protected void closeResultSet() {
+ try {
+ if (getResultSet() != null) {
+ getResultSet().close();
+ }
+ } catch (Exception e) {
+ logError("Exception while closing result set", e);
+ } finally {
+ setResultSet(null);
+ }
+ }
+
+ protected final Iterator<Map<String,Object>> getIterator() {
+ return rSetIterator;
+ }
+
+
+ protected final Statement getStatement() {
+ return stmt;
+ }
+
+ protected final void setStatement(Statement stmt) {
+ this.stmt = stmt;
+ }
+
+ protected final ResultSet getResultSet() {
+ return resultSet;
+ }
+
+ protected final void setResultSet(ResultSet resultSet) {
+ this.resultSet = resultSet;
+ }
+
+ protected final List<String> getColNames() {
+ return colNames;
+ }
+
+ protected final void setColNames(List<String> colNames) {
+ this.colNames = colNames;
+ }
+
+ }
+
+ protected Connection getConnection() throws Exception {
+ long currTime = System.nanoTime();
+ if (currTime - connLastUsed > CONN_TIME_OUT) {
+ synchronized (this) {
+ Connection tmpConn = factory.call();
+ closeConnection();
+ connLastUsed = System.nanoTime();
+ return conn = tmpConn;
+ }
+
+ } else {
+ connLastUsed = currTime;
+ return conn;
+ }
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+ try {
+ if(!isClosed){
+ log.error("JdbcDataSource was not closed prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!");
+ close();
+ }
+ } finally {
+ super.finalize();
+ }
+ }
+
+ private boolean isClosed = false;
+
+ @Override
+ public void close() {
+ if (resultSetIterator != null) {
+ resultSetIterator.close();
+ }
+ try {
+ closeConnection();
+ } finally {
+ isClosed = true;
+ }
+ }
+
+ private void closeConnection() {
+ try {
+ if (conn != null) {
+ try {
+ //SOLR-2045
+ conn.commit();
+ } catch(Exception ex) {
+ //ignore.
+ }
+ conn.close();
+ }
+ } catch (Exception e) {
+ log.error("Ignoring Error when closing connection", e);
+ }
+ }
+
+ private static final long CONN_TIME_OUT = TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS);
+
+ private static final int FETCH_SIZE = 500;
+
+ public static final String URL = "url";
+
+ public static final String JNDI_NAME = "jndiName";
+
+ public static final String DRIVER = "driver";
+
+ public static final String CONVERT_TYPE = "convertType";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/LineEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/LineEntityProcessor.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/LineEntityProcessor.java
new file mode 100644
index 0000000..0940cbd
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/LineEntityProcessor.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.*;
+import java.util.*;
+import java.util.regex.Pattern;
+
+import org.apache.commons.io.IOUtils;
+
+
+/**
+ * <p>
+ * An {@link EntityProcessor} instance which can stream lines of text read from a
+ * datasource. Options allow lines to be explicitly skipped or included in the index.
+ * </p>
+ * <p>
+ * Attribute summary
+ * <ul>
+ * <li>url is the required location of the input file. If this value is
+ * relative, it assumed to be relative to baseLoc.</li>
+ * <li>acceptLineRegex is an optional attribute that if present discards any
+ * line which does not match the regExp.</li>
+ * <li>skipLineRegex is an optional attribute that is applied after any
+ * acceptLineRegex and discards any line which matches this regExp.</li>
+ * </ul>
+ * <p>
+ * Although envisioned for reading lines from a file or url, LineEntityProcessor may also be useful
+ * for dealing with change lists, where each line contains filenames which can be used by subsequent entities
+ * to parse content from those files.
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 1.4
+ * @see Pattern
+ */
+public class LineEntityProcessor extends EntityProcessorBase {
+ private Pattern acceptLineRegex, skipLineRegex;
+ private String url;
+ private BufferedReader reader;
+
+ /**
+ * Parses each of the entity attributes.
+ */
+ @Override
+ public void init(Context context) {
+ super.init(context);
+ String s;
+
+ // init a regex to locate files from the input we want to index
+ s = context.getResolvedEntityAttribute(ACCEPT_LINE_REGEX);
+ if (s != null) {
+ acceptLineRegex = Pattern.compile(s);
+ }
+
+ // init a regex to locate files from the input to be skipped
+ s = context.getResolvedEntityAttribute(SKIP_LINE_REGEX);
+ if (s != null) {
+ skipLineRegex = Pattern.compile(s);
+ }
+
+ // the FileName is required.
+ url = context.getResolvedEntityAttribute(URL);
+ if (url == null) throw
+ new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "'"+ URL +"' is a required attribute");
+ }
+
+
+ /**
+ * Reads lines from the url till it finds a lines that matches the
+ * optional acceptLineRegex and does not match the optional skipLineRegex.
+ *
+ * @return A row containing a minimum of one field "rawLine" or null to signal
+ * end of file. The rawLine is the as line as returned by readLine()
+ * from the url. However transformers can be used to create as
+ * many other fields as required.
+ */
+ @Override
+ public Map<String, Object> nextRow() {
+ if (reader == null) {
+ reader = new BufferedReader((Reader) context.getDataSource().getData(url));
+ }
+
+ String line;
+
+ while ( true ) {
+ // read a line from the input file
+ try {
+ line = reader.readLine();
+ }
+ catch (IOException exp) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Problem reading from input", exp);
+ }
+
+ // end of input
+ if (line == null) {
+ closeResources();
+ return null;
+ }
+
+ // First scan whole line to see if we want it
+ if (acceptLineRegex != null && ! acceptLineRegex.matcher(line).find()) continue;
+ if (skipLineRegex != null && skipLineRegex.matcher(line).find()) continue;
+ // Contruct the 'row' of fields
+ Map<String, Object> row = new HashMap<>();
+ row.put("rawLine", line);
+ return row;
+ }
+ }
+
+ public void closeResources() {
+ if (reader != null) {
+ IOUtils.closeQuietly(reader);
+ }
+ reader= null;
+ }
+
+ @Override
+ public void destroy() {
+ closeResources();
+ super.destroy();
+ }
+
+ /**
+ * Holds the name of entity attribute that will be parsed to obtain
+ * the filename containing the changelist.
+ */
+ public static final String URL = "url";
+
+ /**
+ * Holds the name of entity attribute that will be parsed to obtain
+ * the pattern to be used when checking to see if a line should
+ * be returned.
+ */
+ public static final String ACCEPT_LINE_REGEX = "acceptLineRegex";
+
+ /**
+ * Holds the name of entity attribute that will be parsed to obtain
+ * the pattern to be used when checking to see if a line should
+ * be ignored.
+ */
+ public static final String SKIP_LINE_REGEX = "skipLineRegex";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/LogTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/LogTransformer.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/LogTransformer.java
new file mode 100644
index 0000000..66c525e
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/LogTransformer.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Map;
+
+/**
+ * A {@link Transformer} implementation which logs messages in a given template format.
+ * <p>
+ * Refer to <a href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 1.4
+ */
+public class LogTransformer extends Transformer {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @Override
+ public Object transformRow(Map<String, Object> row, Context ctx) {
+ String expr = ctx.getEntityAttribute(LOG_TEMPLATE);
+ String level = ctx.replaceTokens(ctx.getEntityAttribute(LOG_LEVEL));
+
+ if (expr == null || level == null) return row;
+
+ if ("info".equals(level)) {
+ if (log.isInfoEnabled())
+ log.info(ctx.replaceTokens(expr));
+ } else if ("trace".equals(level)) {
+ if (log.isTraceEnabled())
+ log.trace(ctx.replaceTokens(expr));
+ } else if ("warn".equals(level)) {
+ if (log.isWarnEnabled())
+ log.warn(ctx.replaceTokens(expr));
+ } else if ("error".equals(level)) {
+ if (log.isErrorEnabled())
+ log.error(ctx.replaceTokens(expr));
+ } else if ("debug".equals(level)) {
+ if (log.isDebugEnabled())
+ log.debug(ctx.replaceTokens(expr));
+ }
+
+ return row;
+ }
+
+ public static final String LOG_TEMPLATE = "logTemplate";
+ public static final String LOG_LEVEL = "logLevel";
+}
[04/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestHierarchicalDocBuilder.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestHierarchicalDocBuilder.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestHierarchicalDocBuilder.java
deleted file mode 100644
index 4a3ba72..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestHierarchicalDocBuilder.java
+++ /dev/null
@@ -1,479 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.join.BitSetProducer;
-import org.apache.lucene.search.join.QueryBitSetProducer;
-import org.apache.lucene.search.join.ScoreMode;
-import org.apache.lucene.search.join.ToParentBlockJoinQuery;
-import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.handler.dataimport.config.ConfigNameConstants;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.search.SolrIndexSearcher;
-import org.apache.solr.util.TestHarness;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Test for DocBuilder using the test harness.
- * <b> Documents are hierarchical in this test, i.e. each document have nested children documents.</b>
- */
-public class TestHierarchicalDocBuilder extends AbstractDataImportHandlerTestCase {
-
- private static final String FIELD_ID = "id";
- private int id = 0; //unique id
- private SolrQueryRequest req;
-
- /**
- * Holds the data related to randomly created index.
- * It is used for making assertions.
- */
- private static class ContextHolder {
- /** Overall documents number **/
- int counter = 0;
-
- /**
- * Each Hierarchy object represents nested documents with a parent at the root of hierarchy
- */
- List<Hierarchy> hierarchies = new ArrayList<Hierarchy>();
- }
-
- /**
- * Represents a hierarchical document structure
- */
- private static class Hierarchy {
-
- /**
- * Type of element, i.e. parent, child, grandchild, etc..
- */
- String elementType;
-
- /**
- * Fields of a current element
- */
- Map<String, Object> elementData = new HashMap<String,Object>();
-
- /**
- * Nested elements/documents hierarchies.
- */
- List<Hierarchy> elements = new ArrayList<Hierarchy>();
- }
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
- }
-
- @Before
- public void before() {
- req = req("*:*"); // don't really care about query
- MockDataSource.clearCache();
- }
-
- @After
- public void after() {
- req.close();
- MockDataSource.clearCache();
- }
-
- @Test
- public void testThreeLevelHierarchy() throws Exception {
- int parentsNum = 3; //fixed for simplicity of test
- int childrenNum = 0;
- int grandChildrenNum = 0;
-
- final String parentType = "parent";
- final String childType = "child";
- final String grandChildType = "grand_child";
-
- List<String> parentIds = createDataIterator("select * from PARENT", parentType, parentType, parentsNum);
- Collections.shuffle(parentIds, random());
- final String parentId1 = parentIds.get(0);
- String parentId2 = parentIds.get(1);
-
- //parent 1 children
- int firstParentChildrenNum = 3; //fixed for simplicity of test
- String select = "select * from CHILD where parent_id='" + parentId1 + "'";
- List<String> childrenIds = createDataIterator(select, childType, "child of first parent", firstParentChildrenNum);
- List<String> firstParentChildrenIds = new ArrayList<String>(childrenIds);
- childrenNum += childrenIds.size();
-
- // grand children of first parent first child
- final String childId = childrenIds.get(0);
- String description = "grandchild of first parent, child of " + childId + " child";
- select = "select * from GRANDCHILD where parent_id='" + childId + "'";
- List<String> grandChildrenIds = createDataIterator(select, grandChildType, description, atLeast(2));
- grandChildrenNum += grandChildrenIds.size();
-
- // grand children of first parent second child
- {
- String childId2 = childrenIds.get(1);
- description = "grandchild of first parent, child of " + childId2 + " child";
- select = "select * from GRANDCHILD where parent_id='" + childId2 + "'";
- }
- final List<String> grandChildrenIds2 = createDataIterator(select, grandChildType, description, atLeast(2));
- grandChildrenNum += grandChildrenIds2.size();
-
- List<String> allGrandChildrenIds = new ArrayList<>(grandChildrenIds);
- allGrandChildrenIds.addAll(grandChildrenIds2);
-
- // third children of first parent has no grand children
-
- // parent 2 children (no grand children)
- select = "select * from CHILD where parent_id='" + parentId2 + "'";
- childrenIds = createDataIterator(select, childType, "child of second parent", atLeast(2));
- childrenNum += childrenIds.size();
-
- // parent 3 has no children and grand children
-
- int totalDocsNum = parentsNum + childrenNum + grandChildrenNum;
-
- String resp = runFullImport(THREE_LEVEL_HIERARCHY_CONFIG);
- String xpath = "//arr[@name='documents']/lst[arr[@name='id']/str='"+parentId1+"']/"+
- "arr[@name='_childDocuments_']/lst[arr[@name='id']/str='"+childId+"']/"+
- "arr[@name='_childDocuments_']/lst[arr[@name='id']/str='"+grandChildrenIds.get(0)+"']";
- String results = TestHarness.validateXPath(resp,
- xpath);
- assertTrue("Debug documents does not contain child documents\n"+resp+"\n"+ xpath+
- "\n"+results, results == null);
-
- assertTrue("Update request processor processAdd was not called", TestUpdateRequestProcessor.processAddCalled);
- assertTrue("Update request processor processCommit was not callled", TestUpdateRequestProcessor.processCommitCalled);
- assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
-
- // very simple asserts to check that we at least have correct num of docs indexed
- assertQ(req("*:*"), "//*[@numFound='" + totalDocsNum + "']");
- assertQ(req("type_s:parent"), "//*[@numFound='" + parentsNum + "']");
- assertQ(req("type_s:child"), "//*[@numFound='" + childrenNum + "']");
- assertQ(req("type_s:grand_child"), "//*[@numFound='" + grandChildrenNum + "']");
-
- // let's check BlockJoin
- // get first parent by any grand children
- String randomGrandChildId = allGrandChildrenIds.get(random().nextInt(allGrandChildrenIds.size()));
- Query query = createToParentQuery(parentType, FIELD_ID, randomGrandChildId);
- assertSearch(query, FIELD_ID, parentId1);
-
- // get first parent by any children
- String randomChildId = firstParentChildrenIds.get(random().nextInt(firstParentChildrenIds.size()));
- query = createToParentQuery(parentType, FIELD_ID, randomChildId);
- assertSearch(query, FIELD_ID, parentId1);
-
- // get parent by children by grand children
- randomGrandChildId = grandChildrenIds.get(random().nextInt(grandChildrenIds.size()));
- ToParentBlockJoinQuery childBlockJoinQuery = createToParentQuery(childType, FIELD_ID, randomGrandChildId);
- ToParentBlockJoinQuery blockJoinQuery = new ToParentBlockJoinQuery(childBlockJoinQuery, createParentFilter(parentType), ScoreMode.Avg);
- assertSearch(blockJoinQuery, FIELD_ID, parentId1);
- }
-
- @Test
- public void testRandomDepthHierarchy() throws Exception {
- final String parentType = "parent";
-
- // Be aware that hierarchies grows exponentially, thus
- // numbers bigger than 6 may lead to significant memory usage
- // and cause OOME
- int parentsNum = 2 + random().nextInt(3);
- int depth = 2 + random().nextInt(3);
-
- ContextHolder holder = new ContextHolder();
-
- String config = createRandomizedConfig(depth, parentType, parentsNum, holder);
- runFullImport(config);
-
- assertTrue("Update request processor processAdd was not called", TestUpdateRequestProcessor.processAddCalled);
- assertTrue("Update request processor processCommit was not callled", TestUpdateRequestProcessor.processCommitCalled);
- assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
-
- assertQ(req("type_s:" + parentType), "//*[@numFound='" + parentsNum + "']");
- assertQ(req("-type_s:"+ parentType), "//*[@numFound='" + (holder.counter - parentsNum) + "']");
-
- // let's check BlockJoin
- Hierarchy randomHierarchy = holder.hierarchies.get(random().nextInt(holder.hierarchies.size()));
-
- Query deepestQuery = createBlockJoinQuery(randomHierarchy);
- assertSearch(deepestQuery, FIELD_ID, (String) randomHierarchy.elementData.get(FIELD_ID));
- }
-
- private Query createBlockJoinQuery(Hierarchy hierarchy) {
- List<Hierarchy> elements = hierarchy.elements;
- if (elements.isEmpty()) {
- BooleanQuery.Builder childQuery = new BooleanQuery.Builder();
- childQuery.add(new TermQuery(new Term(FIELD_ID, (String) hierarchy.elementData.get(FIELD_ID))), Occur.MUST);
- return childQuery.build();
- }
-
- Query childQuery = createBlockJoinQuery(elements.get(random().nextInt(elements.size())));
- return createToParentQuery(hierarchy.elementType, childQuery);
- }
-
- private ToParentBlockJoinQuery createToParentQuery(String parentType, String childField, String childFieldValue) {
- BooleanQuery.Builder childQuery = new BooleanQuery.Builder();
- childQuery.add(new TermQuery(new Term(childField, childFieldValue)), Occur.MUST);
- ToParentBlockJoinQuery result = createToParentQuery(parentType, childQuery.build());
-
- return result;
- }
-
- private ToParentBlockJoinQuery createToParentQuery(String parentType, Query childQuery) {
- ToParentBlockJoinQuery blockJoinQuery = new ToParentBlockJoinQuery(childQuery, createParentFilter(parentType), ScoreMode.Avg);
-
- return blockJoinQuery;
- }
-
- private void assertSearch(Query query, String field, String... values) throws IOException {
- /* The limit of search queue is doubled to catch the error in case when for some reason there are more docs than expected */
- SolrIndexSearcher searcher = req.getSearcher();
- TopDocs result = searcher.search(query, values.length * 2);
- assertEquals(values.length, result.totalHits.value);
- List<String> actualValues = new ArrayList<String>();
- for (int index = 0; index < values.length; ++index) {
- Document doc = searcher.doc(result.scoreDocs[index].doc);
- actualValues.add(doc.get(field));
- }
-
- for (String expectedValue: values) {
- boolean removed = actualValues.remove(expectedValue);
- if (!removed) {
- fail("Search result does not contain expected values");
- }
- }
- }
-
- @SuppressWarnings("unchecked")
- private List<String> createDataIterator(String query, String type, String description, int count) {
- List<Map<String, Object>> data = new ArrayList<Map<String, Object>>();
- List<String> ids = new ArrayList<String>(count);
- for (int index = 0; index < count; ++index) {
- String docId = nextId();
- ids.add(docId);
- Map<String, Object> doc = createMap(FIELD_ID, docId, "desc", docId + " " + description, "type_s", type);
- data.add(doc);
- }
- Collections.shuffle(data, random());
- MockDataSource.setIterator(query, data.iterator());
-
- return ids;
- }
-
- /**
- * Creates randomized configuration of a specified depth. Simple configuration example:
- *
- * <pre>
- *
- * <dataConfig>
- * <dataSource type="MockDataSource" />
- * <document>
- * <entity name="parent" query="SELECT * FROM parent">
- * <field column="id" />
- * <field column="desc" />
- * <field column="type_s" />
- * <entity child="true" name="parentChild0" query="select * from parentChild0 where parentChild0_parent_id='${parent.id}'">
- * <field column="id" />
- * <field column="desc" />
- * <field column="type_s" />
- * <entity child="true" name="parentChild0Child0" query="select * from parentChild0Child0 where parentChild0Child0_parent_id='${parentChild0.id}'">
- * <field column="id" />
- * <field column="desc" />
- * <field column="type_s" />
- * </entity>
- * <entity child="true" name="parentChild0Child1" query="select * from parentChild0Child1 where parentChild0Child1_parent_id='${parentChild0.id}'">
- * <field column="id" />
- * <field column="desc" />
- * <field column="type_s" />
- * </entity>
- * </entity>
- * <entity child="true" name="parentChild1" query="select * from parentChild1 where parentChild1_parent_id='${parent.id}'">
- * <field column="id" />
- * <field column="desc" />
- * <field column="type_s" />
- * <entity child="true" name="parentChild1Child0" query="select * from parentChild1Child0 where parentChild1Child0_parent_id='${parentChild1.id}'">
- * <field column="id" />
- * <field column="desc" />
- * <field column="type_s" />
- * </entity>
- * <entity child="true" name="parentChild1Child1" query="select * from parentChild1Child1 where parentChild1Child1_parent_id='${parentChild1.id}'">
- * <field column="id" />
- * <field column="desc" />
- * <field column="type_s" />
- * </entity>
- * </entity>
- * </entity>
- * </document>
- * </dataConfig>
- *
- * </pre>
- *
- * Internally configures MockDataSource.
- **/
- private String createRandomizedConfig(int depth, String parentType, int parentsNum, ContextHolder holder) {
- List<Hierarchy> parentData = createMockedIterator(parentType, "SELECT * FROM " + parentType, parentsNum, holder);
-
- holder.hierarchies = parentData;
-
- String children = createChildren(parentType, 0, depth, parentData, holder);
-
- String rootFields = createFieldsList(FIELD_ID, "desc", "type_s");
- String rootEntity = StrUtils.formatString(ROOT_ENTITY_TEMPLATE, parentType, "SELECT * FROM " + parentType, rootFields, children);
-
- String config = StrUtils.formatString(DATA_CONFIG_TEMPLATE, rootEntity);
- return config;
- }
-
- @SuppressWarnings("unchecked")
- private List<Hierarchy> createMockedIterator(String type, String query, int amount, ContextHolder holder) {
- List<Hierarchy> hierarchies = new ArrayList<Hierarchy>();
- List<Map<String, Object>> data = new ArrayList<Map<String, Object>>();
- for (int index = 0; index < amount; ++index) {
- holder.counter++;
- String idStr = String.valueOf(holder.counter);
- Map<String, Object> element = createMap(FIELD_ID, idStr, "desc", type + "_" + holder.counter, "type_s", type);
- data.add(element);
-
- Hierarchy hierarchy = new Hierarchy();
- hierarchy.elementType = type;
- hierarchy.elementData = element;
- hierarchies.add(hierarchy);
- }
-
- MockDataSource.setIterator(query, data.iterator());
-
- return hierarchies;
- }
-
- private List<Hierarchy> createMockedIterator(String type, List<Hierarchy> parentData, ContextHolder holder) {
- List<Hierarchy> result = new ArrayList<Hierarchy>();
- for (Hierarchy parentHierarchy: parentData) {
- Map<String, Object> data = parentHierarchy.elementData;
- String id = (String) data.get(FIELD_ID);
- String select = String.format(Locale.ROOT, "select * from %s where %s='%s'", type, type + "_parent_id", id);
-
- // Number of actual children documents
- int childrenNum = 1 + random().nextInt(3);
- List<Hierarchy> childHierarchies = createMockedIterator(type, select, childrenNum, holder);
- parentHierarchy.elements.addAll(childHierarchies);
- result.addAll(childHierarchies);
- }
- return result;
- }
-
- private String createChildren(String parentName, int currentLevel, int maxLevel,
- List<Hierarchy> parentData, ContextHolder holder) {
-
- if (currentLevel == maxLevel) { //recursion base
- return "";
- }
-
- // number of different children <b>types</b> of parent, i.e. parentChild0, parentChild1
- // @see #createMockedIterator for the actual number of each children type
- int childrenNumber = 2 + random().nextInt(3);
- StringBuilder builder = new StringBuilder();
- for (int childIndex = 0; childIndex < childrenNumber; ++childIndex) {
- String childName = parentName + "Child" + childIndex;
- String fields = createFieldsList(FIELD_ID, "desc", "type_s");
- String select = String.format(Locale.ROOT, "select * from %s where %s='%s'", childName, childName + "_parent_id", "${" + parentName + ".id}");
-
- //for each child entity create several iterators
- List<Hierarchy> childData = createMockedIterator(childName, parentData, holder);
-
- String subChildren = createChildren(childName, currentLevel + 1, maxLevel, childData, holder);
- String child = StrUtils.formatString(CHILD_ENTITY_TEMPLATE, childName, select, fields, subChildren);
- builder.append(child);
- builder.append('\n');
- }
-
- return builder.toString();
- }
-
- private String createFieldsList(String... fields) {
- StringBuilder builder = new StringBuilder();
- for (String field: fields) {
- String text = String.format(Locale.ROOT, "<field column='%s' />", field);
- builder.append(text);
- builder.append('\n');
- }
- return builder.toString();
- }
-
- private static final String THREE_LEVEL_HIERARCHY_CONFIG = "<dataConfig>\n" +
- " <dataSource type='MockDataSource' />\n" +
- " <document>\n" +
- " <entity name='PARENT' query='select * from PARENT'>\n" +
- " <field column='id' />\n" +
- " <field column='desc' />\n" +
- " <field column='type_s' />\n" +
- " <entity child='true' name='CHILD' query=\"select * from CHILD where parent_id='${PARENT.id}'\">\n" +
- " <field column='id' />\n" +
- " <field column='desc' />\n" +
- " <field column='type_s' />\n" +
- " <entity child='true' name='GRANDCHILD' query=\"select * from GRANDCHILD where parent_id='${CHILD.id}'\">\n" +
- " <field column='id' />\n" +
- " <field column='desc' />\n" +
- " <field column='type_s' />\n" +
- " </entity>\n" +
- " </entity>\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- /** {0} is rootEntity block **/
- private static final String DATA_CONFIG_TEMPLATE = "<dataConfig><dataSource type=\"MockDataSource\" />\n<document>\n {0}</document></dataConfig>";
-
- /**
- * {0} - entityName,
- * {1} - select query
- * {2} - fieldsList
- * {3} - childEntitiesList
- **/
- private static final String ROOT_ENTITY_TEMPLATE = "<entity name=\"{0}\" query=\"{1}\">\n{2} {3}\n</entity>\n";
-
- /**
- * {0} - entityName,
- * {1} - select query
- * {2} - fieldsList
- * {3} - childEntitiesList
- **/
- private static final String CHILD_ENTITY_TEMPLATE = "<entity " + ConfigNameConstants.CHILD + "=\"true\" name=\"{0}\" query=\"{1}\">\n {2} {3} </entity>\n";
-
- private BitSetProducer createParentFilter(String type) {
- BooleanQuery.Builder parentQuery = new BooleanQuery.Builder();
- parentQuery.add(new TermQuery(new Term("type_s", type)), Occur.MUST);
- return new QueryBitSetProducer(parentQuery.build());
- }
-
- private String nextId() {
- ++id;
- return String.valueOf(id);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestJdbcDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestJdbcDataSource.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestJdbcDataSource.java
deleted file mode 100644
index e44e7f9..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestJdbcDataSource.java
+++ /dev/null
@@ -1,662 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.sql.Connection;
-import java.sql.Driver;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import javax.sql.DataSource;
-
-import org.apache.solr.common.util.SuppressForbidden;
-import org.apache.solr.handler.dataimport.JdbcDataSource.ResultSetIterator;
-import static org.mockito.Mockito.*;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * <p>
- * Test for JdbcDataSource
- * </p>
- * <p>
- * Note: The tests are ignored for the lack of DB support for testing
- * </p>
- *
- *
- * @since solr 1.3
- */
-public class TestJdbcDataSource extends AbstractDataImportHandlerTestCase {
- private Driver driver;
- private DataSource dataSource;
- private Connection connection;
- private JdbcDataSource jdbcDataSource = new JdbcDataSource();
- List<Map<String, String>> fields = new ArrayList<>();
-
- Context context = AbstractDataImportHandlerTestCase.getContext(null, null,
- jdbcDataSource, Context.FULL_DUMP, fields, null);
-
- Properties props = new Properties();
-
- String sysProp = System.getProperty("java.naming.factory.initial");
-
- @BeforeClass
- public static void beforeClass() {
- assumeWorkingMockito();
- }
-
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- System.setProperty("java.naming.factory.initial",
- MockInitialContextFactory.class.getName());
-
- driver = mock(Driver.class);
- dataSource = mock(DataSource.class);
- connection = mock(Connection.class);
- props.clear();
- }
-
- @Override
- @After
- public void tearDown() throws Exception {
- if (sysProp == null) {
- System.getProperties().remove("java.naming.factory.initial");
- } else {
- System.setProperty("java.naming.factory.initial", sysProp);
- }
- super.tearDown();
- reset(driver, dataSource, connection);
- }
-
- @Test
- public void testRetrieveFromJndi() throws Exception {
- MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
-
- props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
-
- when(dataSource.getConnection()).thenReturn(connection);
-
- Connection conn = jdbcDataSource.createConnectionFactory(context, props)
- .call();
-
- verify(connection).setAutoCommit(false);
- verify(dataSource).getConnection();
-
- assertSame("connection", conn, connection);
- }
-
- @Test
- public void testRetrieveFromJndiWithCredentials() throws Exception {
- MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
-
- props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
- props.put("user", "Fred");
- props.put("password", "4r3d");
- props.put("holdability", "HOLD_CURSORS_OVER_COMMIT");
-
- when(dataSource.getConnection("Fred", "4r3d")).thenReturn(
- connection);
-
- Connection conn = jdbcDataSource.createConnectionFactory(context, props)
- .call();
-
- verify(connection).setAutoCommit(false);
- verify(connection).setHoldability(1);
- verify(dataSource).getConnection("Fred", "4r3d");
-
- assertSame("connection", conn, connection);
- }
-
- @Test
- public void testRetrieveFromJndiWithCredentialsEncryptedAndResolved() throws Exception {
- MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
-
- String user = "Fred";
- String plainPassword = "MyPassword";
- String encryptedPassword = "U2FsdGVkX18QMjY0yfCqlfBMvAB4d3XkwY96L7gfO2o=";
- String propsNamespace = "exampleNamespace";
-
- props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
-
- props.put("user", "${" +propsNamespace +".user}");
- props.put("encryptKeyFile", "${" +propsNamespace +".encryptKeyFile}");
- props.put("password", "${" +propsNamespace +".password}");
-
- when(dataSource.getConnection(user, plainPassword)).thenReturn(
- connection);
-
- Map<String,Object> values = new HashMap<>();
- values.put("user", user);
- values.put("encryptKeyFile", createEncryptionKeyFile());
- values.put("password", encryptedPassword);
- context.getVariableResolver().addNamespace(propsNamespace, values);
-
- jdbcDataSource.init(context, props);
- Connection conn = jdbcDataSource.getConnection();
-
- verify(connection).setAutoCommit(false);
- verify(dataSource).getConnection(user, plainPassword);
-
- assertSame("connection", conn, connection);
- }
-
- @Test
- public void testRetrieveFromJndiWithCredentialsWithEncryptedAndResolvedPwd() throws Exception {
- MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
-
- Properties properties = new Properties();
- properties.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
- properties.put("user", "Fred");
- properties.put("encryptKeyFile", "${foo.bar}");
- properties.put("password", "U2FsdGVkX18QMjY0yfCqlfBMvAB4d3XkwY96L7gfO2o=");
- when(dataSource.getConnection("Fred", "MyPassword")).thenReturn(
- connection);
-
- Map<String,Object> values = new HashMap<>();
- values.put("bar", createEncryptionKeyFile());
- context.getVariableResolver().addNamespace("foo", values);
-
- jdbcDataSource.init(context, properties);
- jdbcDataSource.getConnection();
-
- verify(connection).setAutoCommit(false);
- verify(dataSource).getConnection("Fred", "MyPassword");
- }
-
- @Test
- public void testRetrieveFromJndiFailureNotHidden() throws Exception {
- MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
-
- props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
-
- SQLException sqlException = new SQLException("fake");
- when(dataSource.getConnection()).thenThrow(sqlException);
-
- try {
- jdbcDataSource.createConnectionFactory(context, props).call();
- } catch (SQLException ex) {
- assertSame(sqlException, ex);
- }
-
- verify(dataSource).getConnection();
- }
-
- @Test
- public void testClosesConnectionWhenExceptionThrownOnSetAutocommit() throws Exception {
- MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
-
- props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
-
- SQLException sqlException = new SQLException("fake");
- when(dataSource.getConnection()).thenReturn(connection);
- doThrow(sqlException).when(connection).setAutoCommit(false);
-
- try {
- jdbcDataSource.createConnectionFactory(context, props).call();
- } catch (DataImportHandlerException ex) {
- assertSame(sqlException, ex.getCause());
- }
- verify(dataSource).getConnection();
- verify(connection).setAutoCommit(false);
- verify(connection).close();
- }
-
- @Test
- public void testClosesStatementWhenExceptionThrownOnExecuteQuery() throws Exception {
- MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
-
- props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
- when(dataSource.getConnection()).thenReturn(connection);
-
- jdbcDataSource.init(context, props);
-
- SQLException sqlException = new SQLException("fake");
- Statement statement = mock(Statement.class);
- when(connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY))
- .thenReturn(statement);
- when(statement.execute("query")).thenThrow(sqlException);
-
- try {
- jdbcDataSource.getData("query");
- fail("exception expected");
- } catch (DataImportHandlerException ex) {
- assertSame(sqlException, ex.getCause());
- }
-
- verify(dataSource).getConnection();
- verify(connection).setAutoCommit(false);
- verify(connection).createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- verify(statement).setFetchSize(500);
- verify(statement).setMaxRows(0);
- verify(statement).execute("query");
- verify(statement).close();
- }
-
- @Test
- public void testClosesStatementWhenResultSetNull() throws Exception {
- MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
-
- props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
- when(dataSource.getConnection()).thenReturn(connection);
-
- jdbcDataSource.init(context, props);
-
- Statement statement = mock(Statement.class);
- when(connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY))
- .thenReturn(statement);
- when(statement.execute("query")).thenReturn(false);
- when(statement.getUpdateCount()).thenReturn(-1);
-
- jdbcDataSource.getData("query");
-
- verify(dataSource).getConnection();
- verify(connection).setAutoCommit(false);
- verify(connection).createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- verify(statement).setFetchSize(500);
- verify(statement).setMaxRows(0);
- verify(statement).execute("query");
- verify(statement).getUpdateCount();
- verify(statement).close();
- }
-
- @Test
- public void testClosesStatementWhenHasNextCalledAndResultSetNull() throws Exception {
-
- MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
-
- props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
- when(dataSource.getConnection()).thenReturn(connection);
-
- jdbcDataSource.init(context, props);
-
- Statement statement = mock(Statement.class);
- when(connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY))
- .thenReturn(statement);
- when(statement.execute("query")).thenReturn(true);
- ResultSet resultSet = mock(ResultSet.class);
- when(statement.getResultSet()).thenReturn(resultSet);
- ResultSetMetaData metaData = mock(ResultSetMetaData.class);
- when(resultSet.getMetaData()).thenReturn(metaData);
- when(metaData.getColumnCount()).thenReturn(0);
-
- Iterator<Map<String,Object>> data = jdbcDataSource.getData("query");
-
- ResultSetIterator resultSetIterator = (ResultSetIterator) data.getClass().getDeclaredField("this$1").get(data);
- resultSetIterator.setResultSet(null);
-
- data.hasNext();
-
- verify(dataSource).getConnection();
- verify(connection).setAutoCommit(false);
- verify(connection).createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- verify(statement).setFetchSize(500);
- verify(statement).setMaxRows(0);
- verify(statement).execute("query");
- verify(statement).getResultSet();
- verify(statement).close();
- verify(resultSet).getMetaData();
- verify(metaData).getColumnCount();
- }
-
- @Test
- public void testClosesResultSetAndStatementWhenDataSourceIsClosed() throws Exception {
-
- MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
-
- props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
- when(dataSource.getConnection()).thenReturn(connection);
-
- jdbcDataSource.init(context, props);
-
- Statement statement = mock(Statement.class);
- when(connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY))
- .thenReturn(statement);
- when(statement.execute("query")).thenReturn(true);
- ResultSet resultSet = mock(ResultSet.class);
- when(statement.getResultSet()).thenReturn(resultSet);
- ResultSetMetaData metaData = mock(ResultSetMetaData.class);
- when(resultSet.getMetaData()).thenReturn(metaData);
- when(metaData.getColumnCount()).thenReturn(0);
-
- jdbcDataSource.getData("query");
- jdbcDataSource.close();
-
- verify(dataSource).getConnection();
- verify(connection).setAutoCommit(false);
- verify(connection).createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- verify(statement).setFetchSize(500);
- verify(statement).setMaxRows(0);
- verify(statement).execute("query");
- verify(statement).getResultSet();
- verify(resultSet).getMetaData();
- verify(metaData).getColumnCount();
- verify(resultSet).close();
- verify(statement).close();
- verify(connection).commit();
- verify(connection).close();
- }
-
- @Test
- public void testClosesCurrentResultSetIteratorWhenNewOneIsCreated() throws Exception {
-
- MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
-
- props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
- when(dataSource.getConnection()).thenReturn(connection);
-
- jdbcDataSource.init(context, props);
-
- Statement statement = mock(Statement.class);
- when(connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY))
- .thenReturn(statement);
- when(statement.execute("query")).thenReturn(true);
- ResultSet resultSet = mock(ResultSet.class);
- when(statement.getResultSet()).thenReturn(resultSet);
- ResultSetMetaData metaData = mock(ResultSetMetaData.class);
- when(resultSet.getMetaData()).thenReturn(metaData);
- when(metaData.getColumnCount()).thenReturn(0);
- when(statement.execute("other query")).thenReturn(false);
- when(statement.getUpdateCount()).thenReturn(-1);
-
- jdbcDataSource.getData("query");
- jdbcDataSource.getData("other query");
-
- verify(dataSource).getConnection();
- verify(connection).setAutoCommit(false);
- verify(connection, times(2)).createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- verify(statement, times(2)).setFetchSize(500);
- verify(statement, times(2)).setMaxRows(0);
- verify(statement).execute("query");
- verify(statement).getResultSet();
- verify(resultSet).getMetaData();
- verify(metaData).getColumnCount();
- verify(resultSet).close();
- verify(statement, times(2)).close();
- verify(statement).execute("other query");
- }
-
- @Test
- public void testMultipleResultsSets_UpdateCountUpdateCountResultSet() throws Exception {
- MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
- props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
- when(dataSource.getConnection()).thenReturn(connection);
- jdbcDataSource.init(context, props);
-
- Statement statement = mock(Statement.class);
- when(connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY))
- .thenReturn(statement);
- when(statement.execute("query")).thenReturn(false);
- when(statement.getUpdateCount()).thenReturn(1);
- when(statement.getMoreResults()).thenReturn(false).thenReturn(true);
- ResultSet resultSet = mock(ResultSet.class);
- when(statement.getResultSet()).thenReturn(resultSet);
- ResultSetMetaData metaData = mock(ResultSetMetaData.class);
- when(resultSet.getMetaData()).thenReturn(metaData);
- when(metaData.getColumnCount()).thenReturn(0);
-
- final ResultSetIterator resultSetIterator = jdbcDataSource.new ResultSetIterator("query");
- assertSame(resultSet, resultSetIterator.getResultSet());
-
- verify(connection).setAutoCommit(false);
- verify(connection).createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- verify(statement).setFetchSize(500);
- verify(statement).setMaxRows(0);
- verify(statement).execute("query");
- verify(statement, times(2)).getUpdateCount();
- verify(statement, times(2)).getMoreResults();
- }
-
- @Test
- public void testMultipleResultsSets_ResultSetResultSet() throws Exception {
- MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
- props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
- when(dataSource.getConnection()).thenReturn(connection);
- jdbcDataSource.init(context, props);
- connection.setAutoCommit(false);
-
- Statement statement = mock(Statement.class);
- when(connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY))
- .thenReturn(statement);
- when(statement.execute("query")).thenReturn(true);
- ResultSet resultSet1 = mock(ResultSet.class);
- ResultSet resultSet2 = mock(ResultSet.class);
- when(statement.getResultSet()).thenReturn(resultSet1).thenReturn(resultSet2).thenReturn(null);
- when(statement.getMoreResults()).thenReturn(true).thenReturn(false);
- ResultSetMetaData metaData1 = mock(ResultSetMetaData.class);
- when(resultSet1.getMetaData()).thenReturn(metaData1);
- when(metaData1.getColumnCount()).thenReturn(0);
- when(resultSet1.next()).thenReturn(false);
- ResultSetMetaData metaData2 = mock(ResultSetMetaData.class);
- when(resultSet2.getMetaData()).thenReturn(metaData2);
- when(metaData2.getColumnCount()).thenReturn(0);
- when(resultSet2.next()).thenReturn(true).thenReturn(false);
- when(statement.getUpdateCount()).thenReturn(-1);
-
- final ResultSetIterator resultSetIterator = jdbcDataSource.new ResultSetIterator("query");
- assertSame(resultSet1, resultSetIterator.getResultSet());
- assertTrue(resultSetIterator.hasnext());
- assertSame(resultSet2, resultSetIterator.getResultSet());
- assertFalse(resultSetIterator.hasnext());
-
- verify(dataSource).getConnection();
- verify(connection, times(2)).setAutoCommit(false);
- verify(connection).createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- verify(statement).setFetchSize(500);
- verify(statement).setMaxRows(0);
- verify(statement).execute("query");
- verify(statement, times(2)).getResultSet();
- verify(resultSet1).getMetaData();
- verify(metaData1).getColumnCount();
- verify(resultSet1).next();
- verify(resultSet1).close();
- verify(resultSet2).getMetaData();
- verify(metaData2).getColumnCount();
- verify(resultSet2, times(2)).next();
- verify(resultSet2).close();
- verify(statement, times(2)).getMoreResults();
- verify(statement).getUpdateCount();
- verify(statement).close();
- }
-
- @Test
- public void testRetrieveFromDriverManager() throws Exception {
- // we're not (directly) using a Mockito based mock class here because it won't have a consistent class name
- // that will work with DriverManager's class bindings
- MockDriver mockDriver = new MockDriver(connection);
- DriverManager.registerDriver(mockDriver);
- try {
- props.put(JdbcDataSource.DRIVER, MockDriver.class.getName());
- props.put(JdbcDataSource.URL, MockDriver.MY_JDBC_URL);
- props.put("holdability", "HOLD_CURSORS_OVER_COMMIT");
-
- Connection conn = jdbcDataSource.createConnectionFactory(context, props).call();
-
- verify(connection).setAutoCommit(false);
- verify(connection).setHoldability(1);
-
- assertSame("connection", conn, connection);
- } catch(Exception e) {
- throw e;
- } finally {
- DriverManager.deregisterDriver(mockDriver);
- }
- }
-
-
- @Test
- public void testEmptyResultSet() throws Exception {
- MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource);
-
- props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB");
- when(dataSource.getConnection()).thenReturn(connection);
-
- jdbcDataSource.init(context, props);
-
- Statement statement = mock(Statement.class);
- when(connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY))
- .thenReturn(statement);
- when(statement.execute("query")).thenReturn(true);
- ResultSet resultSet = mock(ResultSet.class);
- when(statement.getResultSet()).thenReturn(resultSet);
- ResultSetMetaData metaData = mock(ResultSetMetaData.class);
- when(resultSet.getMetaData()).thenReturn(metaData);
- when(metaData.getColumnCount()).thenReturn(0);
- when(resultSet.next()).thenReturn(false);
- when(statement.getMoreResults()).thenReturn(false);
- when(statement.getUpdateCount()).thenReturn(-1);
-
- Iterator<Map<String,Object>> resultSetIterator = jdbcDataSource.getData("query");
- resultSetIterator.hasNext();
- resultSetIterator.hasNext();
-
- verify(connection).setAutoCommit(false);
- verify(connection).createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- verify(statement).setFetchSize(500);
- verify(statement).setMaxRows(0);
- verify(statement).execute("query");
- verify(statement).getResultSet();
- verify(resultSet).getMetaData();
- verify(metaData).getColumnCount();
- verify(resultSet).next();
- verify(resultSet).close();
- verify(statement).getMoreResults();
- verify(statement).getUpdateCount();
- verify(statement).close();
- }
-
- @Test
- @Ignore("Needs a Mock database server to work")
- public void testBasic() throws Exception {
- JdbcDataSource dataSource = new JdbcDataSource();
- Properties p = new Properties();
- p.put("driver", "com.mysql.jdbc.Driver");
- p.put("url", "jdbc:mysql://127.0.0.1/autos");
- p.put("user", "root");
- p.put("password", "");
-
- List<Map<String, String>> flds = new ArrayList<>();
- Map<String, String> f = new HashMap<>();
- f.put("column", "trim_id");
- f.put("type", "long");
- flds.add(f);
- f = new HashMap<>();
- f.put("column", "msrp");
- f.put("type", "float");
- flds.add(f);
-
- Context c = getContext(null, null,
- dataSource, Context.FULL_DUMP, flds, null);
- dataSource.init(c, p);
- Iterator<Map<String, Object>> i = dataSource
- .getData("select make,model,year,msrp,trim_id from atrimlisting where make='Acura'");
- int count = 0;
- Object msrp = null;
- Object trim_id = null;
- while (i.hasNext()) {
- Map<String, Object> map = i.next();
- msrp = map.get("msrp");
- trim_id = map.get("trim_id");
- count++;
- }
- assertEquals(5, count);
- assertEquals(Float.class, msrp.getClass());
- assertEquals(Long.class, trim_id.getClass());
- }
-
- private String createEncryptionKeyFile() throws IOException {
- File tmpdir = createTempDir().toFile();
- byte[] content = "secret".getBytes(StandardCharsets.UTF_8);
- createFile(tmpdir, "enckeyfile.txt", content, false);
- return new File(tmpdir, "enckeyfile.txt").getAbsolutePath();
- }
-
- /**
- * A stub driver that returns our mocked connection for connection URL {@link #MY_JDBC_URL}.
- * <p>
- * This class is used instead of a Mockito mock because {@link DriverManager} uses the class
- * name to lookup the driver and also requires the driver to behave in a sane way, if other
- * drivers are registered in the runtime. A simple Mockito mock is likely to break
- * depending on JVM runtime version. So this class implements a full {@link Driver},
- * so {@code DriverManager} can do whatever it wants to find the correct driver for a URL.
- */
- public static final class MockDriver implements Driver {
- public static final String MY_JDBC_URL = "jdbc:fakedb";
- private final Connection conn;
-
- public MockDriver() throws SQLException {
- throw new AssertionError("The driver should never be directly instantiated by DIH's JdbcDataSource");
- }
-
- MockDriver(Connection conn) throws SQLException {
- this.conn = conn;
- }
-
- @Override
- public boolean acceptsURL(String url) throws java.sql.SQLException {
- return MY_JDBC_URL.equals(url);
- }
-
- @Override
- public Connection connect(String url, Properties info) throws java.sql.SQLException {
- return acceptsURL(url) ? conn : null;
- }
-
- @Override
- public int getMajorVersion() {
- return 1;
- }
-
- @Override
- public int getMinorVersion() {
- return 0;
- }
-
- @SuppressForbidden(reason="Required by JDBC")
- @Override
- public java.util.logging.Logger getParentLogger() throws java.sql.SQLFeatureNotSupportedException {
- throw new java.sql.SQLFeatureNotSupportedException();
- }
-
- @Override
- public java.sql.DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException {
- return new java.sql.DriverPropertyInfo[0];
- }
-
- @Override
- public boolean jdbcCompliant() {
- // we are not fully compliant:
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestJdbcDataSourceConvertType.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestJdbcDataSourceConvertType.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestJdbcDataSourceConvertType.java
deleted file mode 100644
index 94f437c..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestJdbcDataSourceConvertType.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Properties;
-
-@ThreadLeakAction({ThreadLeakAction.Action.WARN})
-@ThreadLeakLingering(linger = 0)
-@ThreadLeakZombies(ThreadLeakZombies.Consequence.CONTINUE)
-@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
-public class TestJdbcDataSourceConvertType extends AbstractDataImportHandlerTestCase {
- public void testConvertType() throws Throwable {
- final Locale loc = Locale.getDefault();
- assumeFalse("Derby is not happy with locale sr-Latn-*",
- Objects.equals(new Locale("sr").getLanguage(), loc.getLanguage()) &&
- Objects.equals("Latn", loc.getScript()));
-
- // ironically convertType=false causes BigDecimal to String conversion
- convertTypeTest("false", String.class);
-
- // convertType=true uses the "long" conversion (see mapping of some_i to "long")
- convertTypeTest("true", Long.class);
- }
-
- private void convertTypeTest(String convertType, Class resultClass) throws Throwable {
- JdbcDataSource dataSource = new JdbcDataSource();
- Properties p = new Properties();
- p.put("driver", "org.apache.derby.jdbc.EmbeddedDriver");
- p.put("url", "jdbc:derby:memory:tempDB;create=true;territory=en_US");
- p.put("convertType", convertType);
-
- List<Map<String, String>> flds = new ArrayList<>();
- Map<String, String> f = new HashMap<>();
- f.put("column", "some_i");
- f.put("type", "long");
- flds.add(f);
-
- Context c = getContext(null, null,
- dataSource, Context.FULL_DUMP, flds, null);
- dataSource.init(c, p);
- Iterator<Map<String, Object>> i = dataSource
- .getData("select 1 as id, CAST(9999 AS DECIMAL) as \"some_i\" from sysibm.sysdummy1");
- assertTrue(i.hasNext());
- Map<String, Object> map = i.next();
- Object val = map.get("some_i");
- assertEquals(resultClass, val.getClass());
-
- dataSource.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestLineEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestLineEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestLineEntityProcessor.java
deleted file mode 100644
index eb9989c..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestLineEntityProcessor.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
-import java.util.*;
-
-
-/**
- * <p> Test for TestLineEntityProcessor </p>
- *
- *
- * @since solr 1.4
- */
-public class TestLineEntityProcessor extends AbstractDataImportHandlerTestCase {
-
- @Test
- /************************************************************************/
- public void testSimple() throws IOException {
-
- /* we want to create the equiv of :-
- * <entity name="list_all_files"
- * processor="LineEntityProcessor"
- * fileName="dummy.lis"
- * />
- */
-
- Map attrs = createMap(
- LineEntityProcessor.URL, "dummy.lis",
- LineEntityProcessor.ACCEPT_LINE_REGEX, null,
- LineEntityProcessor.SKIP_LINE_REGEX, null
- );
-
- Context c = getContext(
- null, //parentEntity
- new VariableResolver(), //resolver
- getDataSource(filecontents), //parentDataSource
- Context.FULL_DUMP, //currProcess
- Collections.EMPTY_LIST, //entityFields
- attrs //entityAttrs
- );
- LineEntityProcessor ep = new LineEntityProcessor();
- ep.init(c);
-
- /// call the entity processor to the list of lines
- if (VERBOSE) System.out.print("\n");
- List<String> fList = new ArrayList<>();
- while (true) {
- Map<String, Object> f = ep.nextRow();
- if (f == null) break;
- fList.add((String) f.get("rawLine"));
- if (VERBOSE) System.out.print(" rawLine='" + f.get("rawLine") + "'\n");
- }
- assertEquals(24, fList.size());
- }
-
- @Test
- /************************************************************************/
- public void testOnly_xml_files() throws IOException {
-
- /* we want to create the equiv of :-
- * <entity name="list_all_files"
- * processor="LineEntityProcessor"
- * fileName="dummy.lis"
- * acceptLineRegex="xml"
- * />
- */
- Map attrs = createMap(
- LineEntityProcessor.URL, "dummy.lis",
- LineEntityProcessor.ACCEPT_LINE_REGEX, "xml",
- LineEntityProcessor.SKIP_LINE_REGEX, null
- );
-
- Context c = getContext(
- null, //parentEntity
- new VariableResolver(), //resolver
- getDataSource(filecontents), //parentDataSource
- Context.FULL_DUMP, //currProcess
- Collections.EMPTY_LIST, //entityFields
- attrs //entityAttrs
- );
- LineEntityProcessor ep = new LineEntityProcessor();
- ep.init(c);
-
- /// call the entity processor to the list of lines
- List<String> fList = new ArrayList<>();
- while (true) {
- Map<String, Object> f = ep.nextRow();
- if (f == null) break;
- fList.add((String) f.get("rawLine"));
- }
- assertEquals(5, fList.size());
- }
-
- @Test
- /************************************************************************/
- public void testOnly_xml_files_no_xsd() throws IOException {
- /* we want to create the equiv of :-
- * <entity name="list_all_files"
- * processor="LineEntityProcessor"
- * fileName="dummy.lis"
- * acceptLineRegex="\\.xml"
- * omitLineRegex="\\.xsd"
- * />
- */
- Map attrs = createMap(
- LineEntityProcessor.URL, "dummy.lis",
- LineEntityProcessor.ACCEPT_LINE_REGEX, "\\.xml",
- LineEntityProcessor.SKIP_LINE_REGEX, "\\.xsd"
- );
-
- Context c = getContext(
- null, //parentEntity
- new VariableResolver(), //resolver
- getDataSource(filecontents), //parentDataSource
- Context.FULL_DUMP, //currProcess
- Collections.EMPTY_LIST, //entityFields
- attrs //entityAttrs
- );
- LineEntityProcessor ep = new LineEntityProcessor();
- ep.init(c);
-
- /// call the entity processor to walk the directory
- List<String> fList = new ArrayList<>();
- while (true) {
- Map<String, Object> f = ep.nextRow();
- if (f == null) break;
- fList.add((String) f.get("rawLine"));
- }
- assertEquals(4, fList.size());
- }
-
- @Test
- /************************************************************************/
- public void testNo_xsd_files() throws IOException {
- /* we want to create the equiv of :-
- * <entity name="list_all_files"
- * processor="LineEntityProcessor"
- * fileName="dummy.lis"
- * omitLineRegex="\\.xsd"
- * />
- */
- Map attrs = createMap(
- LineEntityProcessor.URL, "dummy.lis",
- LineEntityProcessor.SKIP_LINE_REGEX, "\\.xsd"
- );
-
- Context c = getContext(
- null, //parentEntity
- new VariableResolver(), //resolver
- getDataSource(filecontents), //parentDataSource
- Context.FULL_DUMP, //currProcess
- Collections.EMPTY_LIST, //entityFields
- attrs //entityAttrs
- );
- LineEntityProcessor ep = new LineEntityProcessor();
- ep.init(c);
-
- /// call the entity processor to walk the directory
- List<String> fList = new ArrayList<>();
- while (true) {
- Map<String, Object> f = ep.nextRow();
- if (f == null) break;
- fList.add((String) f.get("rawLine"));
- }
- assertEquals(18, fList.size());
- }
-
- /**
- * ********************************************************************
- */
- public static Map<String, String> createField(
- String col, // DIH column name
- String type, // field type from schema.xml
- String srcCol, // DIH transformer attribute 'sourceColName'
- String re, // DIH regex attribute 'regex'
- String rw, // DIH regex attribute 'replaceWith'
- String gn // DIH regex attribute 'groupNames'
- ) {
- HashMap<String, String> vals = new HashMap<>();
- vals.put("column", col);
- vals.put("type", type);
- vals.put("sourceColName", srcCol);
- vals.put("regex", re);
- vals.put("replaceWith", rw);
- vals.put("groupNames", gn);
- return vals;
- }
-
- private DataSource<Reader> getDataSource(final String xml) {
- return new DataSource<Reader>() {
- @Override
- public void init(Context context, Properties initProps) {
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public Reader getData(String query) {
- return new StringReader(xml);
- }
- };
- }
-
- private static final String filecontents =
- "\n" +
- "# this is what the output from 'find . -ls; looks like, athough the format\n" +
- "# of the time stamp varies depending on the age of the file and your LANG \n" +
- "# env setting\n" +
- "412577 0 drwxr-xr-x 6 user group 204 1 Apr 10:53 /Volumes/spare/ts\n" +
- "412582 0 drwxr-xr-x 13 user group 442 1 Apr 10:18 /Volumes/spare/ts/config\n" +
- "412583 24 -rwxr-xr-x 1 user group 8318 1 Apr 11:10 /Volumes/spare/ts/config/dc.xsd\n" +
- "412584 32 -rwxr-xr-x 1 user group 12847 1 Apr 11:10 /Volumes/spare/ts/config/dcterms.xsd\n" +
- "412585 8 -rwxr-xr-x 1 user group 3156 1 Apr 11:10 /Volumes/spare/ts/config/s-deliver.css\n" +
- "412586 192 -rwxr-xr-x 1 user group 97764 1 Apr 11:10 /Volumes/spare/ts/config/s-deliver.xsl\n" +
- "412587 224 -rwxr-xr-x 1 user group 112700 1 Apr 11:10 /Volumes/spare/ts/config/sml-delivery-2.1.xsd\n" +
- "412588 208 -rwxr-xr-x 1 user group 103419 1 Apr 11:10 /Volumes/spare/ts/config/sml-delivery-norm-2.0.dtd\n" +
- "412589 248 -rwxr-xr-x 1 user group 125296 1 Apr 11:10 /Volumes/spare/ts/config/sml-delivery-norm-2.1.dtd\n" +
- "412590 72 -rwxr-xr-x 1 user group 36256 1 Apr 11:10 /Volumes/spare/ts/config/jm.xsd\n" +
- "412591 8 -rwxr-xr-x 1 user group 990 1 Apr 11:10 /Volumes/spare/ts/config/video.gif\n" +
- "412592 8 -rwxr-xr-x 1 user group 1498 1 Apr 11:10 /Volumes/spare/ts/config/xlink.xsd\n" +
- "412593 8 -rwxr-xr-x 1 user group 1155 1 Apr 11:10 /Volumes/spare/ts/config/xml.xsd\n" +
- "412594 0 drwxr-xr-x 4 user group 136 1 Apr 10:18 /Volumes/spare/ts/acm19\n" +
- "412621 0 drwxr-xr-x 57 user group 1938 1 Apr 10:18 /Volumes/spare/ts/acm19/data\n" +
- "412622 24 -rwxr-xr-x 1 user group 8894 1 Apr 11:09 /Volumes/spare/ts/acm19/data/00000510.xml\n" +
- "412623 32 -rwxr-xr-x 1 user group 14124 1 Apr 11:09 /Volumes/spare/ts/acm19/data/00000603.xml\n" +
- "412624 24 -rwxr-xr-x 1 user group 11976 1 Apr 11:09 /Volumes/spare/ts/acm19/data/00001292.xml\n" +
- "# tacked on an extra line to cause a file to be deleted.\n" +
- "DELETE /Volumes/spare/ts/acm19/data/00001292old.xml\n" +
- "";
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNestedChildren.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNestedChildren.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNestedChildren.java
deleted file mode 100644
index ca1bfda..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNestedChildren.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.lang.invoke.MethodHandles;
-
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestNestedChildren extends AbstractDIHJdbcTestCase {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- @Test
- public void test() throws Exception {
- h.query("/dataimport", generateRequest());
- assertQ(req("*:*"), "//*[@numFound='1']");
- assertQ(req("third_s:CHICKEN"), "//*[@numFound='1']");
- }
-
- @Override
- protected String generateConfig() {
- StringBuilder sb = new StringBuilder();
- sb.append("<dataConfig> \n");
- sb.append("<dataSource name=\"derby\" driver=\"org.apache.derby.jdbc.EmbeddedDriver\" url=\"jdbc:derby:memory:derbyDB;territory=en_US\" /> \n");
- sb.append("<document name=\"TestSimplePropertiesWriter\"> \n");
- sb.append("<entity name=\"FIRST\" processor=\"SqlEntityProcessor\" dataSource=\"derby\" ");
- sb.append(" query=\"select 1 as id, 'PORK' as FIRST_S from sysibm.sysdummy1 \" >\n");
- sb.append(" <field column=\"FIRST_S\" name=\"first_s\" /> \n");
- sb.append(" <entity name=\"SECOND\" processor=\"SqlEntityProcessor\" dataSource=\"derby\" ");
- sb.append(" query=\"select 1 as id, 2 as SECOND_ID, 'BEEF' as SECOND_S from sysibm.sysdummy1 WHERE 1=${FIRST.ID}\" >\n");
- sb.append(" <field column=\"SECOND_S\" name=\"second_s\" /> \n");
- sb.append(" <entity name=\"THIRD\" processor=\"SqlEntityProcessor\" dataSource=\"derby\" ");
- sb.append(" query=\"select 1 as id, 'CHICKEN' as THIRD_S from sysibm.sysdummy1 WHERE 2=${SECOND.SECOND_ID}\" >\n");
- sb.append(" <field column=\"THIRD_S\" name=\"third_s\" /> \n");
- sb.append(" </entity>\n");
- sb.append(" </entity>\n");
- sb.append("</entity>\n");
- sb.append("</document> \n");
- sb.append("</dataConfig> \n");
- String config = sb.toString();
- log.debug(config);
- return config;
- }
-
- @Override
- protected Database setAllowedDatabases() {
- return Database.DERBY;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java
deleted file mode 100644
index 1307927..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class TestNonWritablePersistFile extends AbstractDataImportHandlerTestCase {
- private static final String FULLIMPORT_QUERY = "select * from x";
-
- private static final String DELTA_QUERY = "select id from x where last_modified > NOW";
-
- private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
-
- private static final String dataConfig_delta =
- "<dataConfig>" +
- " <dataSource type=\"MockDataSource\"/>\n" +
- " <document>\n" +
- " <entity name=\"x\" transformer=\"TemplateTransformer\"" +
- " query=\"" + FULLIMPORT_QUERY + "\"" +
- " deletedPkQuery=\"" + DELETED_PK_QUERY + "\"" +
- " deltaImportQuery=\"select * from x where id='${dih.delta.id}'\"" +
- " deltaQuery=\"" + DELTA_QUERY + "\">\n" +
- " <field column=\"id\" name=\"id\"/>\n" +
- " <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n" +
- " <field column=\"desc\" />\n" +
- " </entity>\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>\n";
- private static String tmpSolrHome;
-
- private static File f;
-
- @BeforeClass
- public static void createTempSolrHomeAndCore() throws Exception {
- tmpSolrHome = createTempDir().toFile().getAbsolutePath();
- FileUtils.copyDirectory(getFile("dih/solr"), new File(tmpSolrHome).getAbsoluteFile());
- initCore("dataimport-solrconfig.xml", "dataimport-schema.xml",
- new File(tmpSolrHome).getAbsolutePath());
-
- // See SOLR-2551
- String configDir = h.getCore().getResourceLoader().getConfigDir();
- String filePath = configDir;
- if (configDir != null && !configDir.endsWith(File.separator))
- filePath += File.separator;
- filePath += "dataimport.properties";
- f = new File(filePath);
- // execute the test only if we are able to set file to read only mode
- assumeTrue("No dataimport.properties file", f.exists() || f.createNewFile());
- assumeTrue("dataimport.properties can't be set read only", f.setReadOnly());
- assumeFalse("dataimport.properties is still writable even though " +
- "marked readonly - test running as superuser?", f.canWrite());
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- if (f != null) {
- f.setWritable(true);
- }
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testNonWritablePersistFile() throws Exception {
- ignoreException("Properties is not writable");
-
- @SuppressWarnings("rawtypes")
- List parentRow = new ArrayList();
- parentRow.add(createMap("id", "1"));
- MockDataSource.setIterator(FULLIMPORT_QUERY, parentRow.iterator());
-
- @SuppressWarnings("rawtypes")
- List childRow = new ArrayList();
- childRow.add(createMap("desc", "hello"));
- MockDataSource.setIterator("select * from y where y.A='1'",
- childRow.iterator());
-
- runFullImport(dataConfig_delta);
- assertQ(req("id:1"), "//*[@numFound='0']");
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java
deleted file mode 100644
index 91bdd00..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.junit.Test;
-
-import java.text.DecimalFormatSymbols;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-/**
- * <p>
- * Test for NumberFormatTransformer
- * </p>
- *
- *
- * @since solr 1.3
- */
-public class TestNumberFormatTransformer extends AbstractDataImportHandlerTestCase {
- private char GROUPING_SEP = new DecimalFormatSymbols(Locale.ROOT).getGroupingSeparator();
- private char DECIMAL_SEP = new DecimalFormatSymbols(Locale.ROOT).getDecimalSeparator();
-
- @SuppressWarnings("unchecked")
- @Test
- public void testTransformRow_SingleNumber() {
- char GERMAN_GROUPING_SEP = new DecimalFormatSymbols(Locale.GERMANY).getGroupingSeparator();
- List<Map<String, String>> l = new ArrayList<>();
- l.add(createMap("column", "num",
- NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.NUMBER));
- l.add(createMap("column", "localizedNum",
- NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.NUMBER, NumberFormatTransformer.LOCALE, "de-DE"));
- Context c = getContext(null, null, null, Context.FULL_DUMP, l, null);
- Map<String,Object> m = createMap("num", "123" + GROUPING_SEP + "567", "localizedNum", "123" + GERMAN_GROUPING_SEP + "567");
- new NumberFormatTransformer().transformRow(m, c);
- assertEquals(123567L, m.get("num"));
- assertEquals(123567L, m.get("localizedNum"));
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testTransformRow_MultipleNumbers() throws Exception {
- List<Map<String, String>> fields = new ArrayList<>();
- fields.add(createMap(DataImporter.COLUMN, "inputs"));
- fields.add(createMap(DataImporter.COLUMN,
- "outputs", RegexTransformer.SRC_COL_NAME, "inputs",
- NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.NUMBER));
-
- List<String> inputs = new ArrayList<>();
- inputs.add("123" + GROUPING_SEP + "567");
- inputs.add("245" + GROUPING_SEP + "678");
- Map<String, Object> row = createMap("inputs", inputs);
-
- VariableResolver resolver = new VariableResolver();
- resolver.addNamespace("e", row);
-
- Context context = getContext(null, resolver, null, Context.FULL_DUMP, fields, null);
- new NumberFormatTransformer().transformRow(row, context);
-
- List<Long> output = new ArrayList<>();
- output.add(123567L);
- output.add(245678L);
- Map<String, Object> outputRow = createMap("inputs", inputs, "outputs", output);
-
- assertEquals(outputRow, row);
- }
-
- @Test(expected = DataImportHandlerException.class)
- @SuppressWarnings("unchecked")
- public void testTransformRow_InvalidInput1_Number() {
- List<Map<String, String>> l = new ArrayList<>();
- l.add(createMap("column", "num",
- NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.NUMBER));
- Context c = getContext(null, null, null, Context.FULL_DUMP, l, null);
- Map<String, Object> m = createMap("num", "123" + GROUPING_SEP + "5a67");
- new NumberFormatTransformer().transformRow(m, c);
- }
-
- @Test(expected = DataImportHandlerException.class)
- @SuppressWarnings("unchecked")
- public void testTransformRow_InvalidInput2_Number() {
- List<Map<String, String>> l = new ArrayList<>();
- l.add(createMap("column", "num",
- NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.NUMBER));
- Context c = getContext(null, null, null, Context.FULL_DUMP, l, null);
- Map<String, Object> m = createMap("num", "123" + GROUPING_SEP + "567b");
- new NumberFormatTransformer().transformRow(m, c);
- }
-
- @Test(expected = DataImportHandlerException.class)
- @SuppressWarnings("unchecked")
- public void testTransformRow_InvalidInput2_Currency() {
- List<Map<String, String>> l = new ArrayList<>();
- l.add(createMap("column", "num",
- NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.CURRENCY));
- Context c = getContext(null, null, null, Context.FULL_DUMP, l, null);
- Map<String, Object> m = createMap("num", "123" + GROUPING_SEP + "567b");
- new NumberFormatTransformer().transformRow(m, c);
- }
-
- @Test(expected = DataImportHandlerException.class)
- @SuppressWarnings("unchecked")
- public void testTransformRow_InvalidInput1_Percent() {
- List<Map<String, String>> l = new ArrayList<>();
- l.add(createMap("column", "num",
- NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.PERCENT));
- Context c = getContext(null, null, null, Context.FULL_DUMP, l, null);
- Map<String, Object> m = createMap("num", "123" + GROUPING_SEP + "5a67");
- new NumberFormatTransformer().transformRow(m, c);
- }
-
- @Test(expected = DataImportHandlerException.class)
- @SuppressWarnings("unchecked")
- public void testTransformRow_InvalidInput3_Currency() {
- List<Map<String, String>> l = new ArrayList<>();
- l.add(createMap("column", "num",
- NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.CURRENCY));
- Context c = getContext(null, null, null, Context.FULL_DUMP, l, null);
- Map<String, Object> m = createMap("num", "123" + DECIMAL_SEP + "456" + DECIMAL_SEP + "789");
- new NumberFormatTransformer().transformRow(m, c);
- }
-
- @Test(expected = DataImportHandlerException.class)
- @SuppressWarnings("unchecked")
- public void testTransformRow_InvalidInput3_Number() {
- List<Map<String, String>> l = new ArrayList<>();
- l.add(createMap("column", "num",
- NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.NUMBER));
- Context c = getContext(null, null, null, Context.FULL_DUMP, l, null);
- Map<String, Object> m = createMap("num", "123" + DECIMAL_SEP + "456" + DECIMAL_SEP + "789");
- new NumberFormatTransformer().transformRow(m, c);
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testTransformRow_MalformedInput_Number() {
- List<Map<String, String>> l = new ArrayList<>();
- l.add(createMap("column", "num",
- NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.NUMBER));
- Context c = getContext(null, null, null, Context.FULL_DUMP, l, null);
- Map<String, Object> m = createMap("num", "123" + GROUPING_SEP + GROUPING_SEP + "789");
- new NumberFormatTransformer().transformRow(m, c);
- assertEquals(123789L, m.get("num"));
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java
deleted file mode 100644
index 2385438..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.StringReader;
-import java.sql.Blob;
-import java.sql.SQLException;
-import java.util.Collections;
-import java.util.Properties;
-
-import org.apache.solr.common.util.Utils;
-import org.junit.Test;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-/**
- * Test for PlainTextEntityProcessor
- *
- *
- * @see org.apache.solr.handler.dataimport.PlainTextEntityProcessor
- * @since solr 1.4
- */
-public class TestPlainTextEntityProcessor extends AbstractDataImportHandlerTestCase {
- @Test
- public void testSimple() throws IOException {
- DataImporter di = new DataImporter();
- di.loadAndInit(DATA_CONFIG);
- redirectTempProperties(di);
-
- TestDocBuilder.SolrWriterImpl sw = new TestDocBuilder.SolrWriterImpl();
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- di.runCmd(rp, sw);
- assertEquals(DS.s, sw.docs.get(0).getFieldValue("x"));
- }
-
- static class BlobImpl implements Blob{
- private final byte[] bytes;
-
- BlobImpl(byte[] bytes) {
- this.bytes = bytes;
- }
-
- @Override
- public long length() throws SQLException {
- return 0;
- }
-
- @Override
- public byte[] getBytes(long pos, int length) throws SQLException {
- return bytes;
- }
-
- @Override
- public InputStream getBinaryStream() throws SQLException {
- return new ByteArrayInputStream(bytes);
- }
-
- @Override
- public long position(byte[] pattern, long start) throws SQLException {
- return 0;
- }
-
- @Override
- public long position(Blob pattern, long start) throws SQLException {
- return 0;
- }
-
- @Override
- public int setBytes(long pos, byte[] bytes) throws SQLException {
- return 0;
- }
-
- @Override
- public int setBytes(long pos, byte[] bytes, int offset, int len) throws SQLException {
- return 0;
- }
-
- @Override
- public OutputStream setBinaryStream(long pos) throws SQLException {
- return null;
- }
-
- @Override
- public void truncate(long len) throws SQLException {
-
- }
-
- @Override
- public void free() throws SQLException {
-
- }
-
- @Override
- public InputStream getBinaryStream(long pos, long length) throws SQLException {
- return new ByteArrayInputStream(bytes);
- }
- }
-
- @Test
- public void testSimple2() throws IOException {
- DataImporter di = new DataImporter();
- MockDataSource.setIterator("select id, name, blob_field from lw_table4", Collections.singletonList(Utils.makeMap("blob_field",new BlobImpl(DS.s.getBytes(UTF_8)) ) ).iterator());
-
- String dc =
-
- " <dataConfig>" +
- "<dataSource name=\"ds1\" type=\"MockDataSource\"/>\n" +
- " <!-- dataSource for FieldReaderDataSource -->\n" +
- " <dataSource dataField=\"root.blob_field\" name=\"fr\" type=\"FieldReaderDataSource\"/>\n" +
- "\n" +
- " <document name=\"items\">\n" +
- " <entity dataSource=\"ds1\" name=\"root\" pk=\"id\" query=\"select id, name, blob_field from lw_table4\" transformer=\"TemplateTransformer\">\n" +
- " <field column=\"id\" name=\"id\"/>\n" +
- "\n" +
- " <entity dataField=\"root.blob_field\" dataSource=\"fr\" format=\"text\" name=\"n1\" processor=\"PlainTextEntityProcessor\" url=\"blob_field\">\n" +
- " <field column=\"plainText\" name=\"plainText\"/>\n" +
- " </entity>\n" +
- "\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
- System.out.println(dc);
- di.loadAndInit(dc);
- redirectTempProperties(di);
-
- TestDocBuilder.SolrWriterImpl sw = new TestDocBuilder.SolrWriterImpl();
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- di.runCmd(rp, sw);
- assertEquals(DS.s, sw.docs.get(0).getFieldValue("plainText"));
- }
-
-
- public static class DS extends DataSource {
- static String s = "hello world";
-
- @Override
- public void init(Context context, Properties initProps) {
-
- }
-
- @Override
- public Object getData(String query) {
-
- return new StringReader(s);
- }
-
- @Override
- public void close() {
-
- }
- }
-
- static String DATA_CONFIG = "<dataConfig>\n" +
- "\t<dataSource type=\"TestPlainTextEntityProcessor$DS\" />\n" +
- "\t<document>\n" +
- "\t\t<entity processor=\"PlainTextEntityProcessor\" name=\"x\" query=\"x\">\n" +
- "\t\t\t<field column=\"plainText\" name=\"x\" />\n" +
- "\t\t</entity>\n" +
- "\t</document>\n" +
- "</dataConfig>";
-}
[12/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/Entity.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/Entity.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/Entity.java
new file mode 100644
index 0000000..0d0ba4f
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/Entity.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport.config;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.solr.handler.dataimport.DataImportHandlerException;
+import org.apache.solr.handler.dataimport.DataImporter;
+import org.apache.solr.schema.SchemaField;
+import org.w3c.dom.Element;
+
+public class Entity {
+ private final String name;
+ private final String pk;
+ private final String pkMappingFromSchema;
+ private final String dataSourceName;
+ private final String processorName;
+ private final Entity parentEntity;
+ private final boolean docRoot;
+ private final boolean child;
+ private final List<Entity> children;
+ private final List<EntityField> fields;
+ private final Map<String,Set<EntityField>> colNameVsField;
+ private final Map<String,String> allAttributes;
+ private final List<Map<String,String>> allFieldAttributes;
+ private final DIHConfiguration config;
+
+ public Entity(boolean docRootFound, Element element, DataImporter di, DIHConfiguration config, Entity parent) {
+ this.parentEntity = parent;
+ this.config = config;
+
+ String modName = ConfigParseUtil.getStringAttribute(element, ConfigNameConstants.NAME, null);
+ if (modName == null) {
+ throw new DataImportHandlerException(SEVERE, "Entity must have a name.");
+ }
+ if (modName.indexOf(".") != -1) {
+ throw new DataImportHandlerException(SEVERE,
+ "Entity name must not have period (.): '" + modName);
+ }
+ if (ConfigNameConstants.RESERVED_WORDS.contains(modName)) {
+ throw new DataImportHandlerException(SEVERE, "Entity name : '" + modName
+ + "' is a reserved keyword. Reserved words are: " + ConfigNameConstants.RESERVED_WORDS);
+ }
+ this.name = modName;
+ this.pk = ConfigParseUtil.getStringAttribute(element, "pk", null);
+ this.processorName = ConfigParseUtil.getStringAttribute(element, ConfigNameConstants.PROCESSOR,null);
+ this.dataSourceName = ConfigParseUtil.getStringAttribute(element, DataImporter.DATA_SRC, null);
+
+ String rawDocRootValue = ConfigParseUtil.getStringAttribute(element, ConfigNameConstants.ROOT_ENTITY, null);
+ if (!docRootFound && !"false".equals(rawDocRootValue)) {
+ // if in this chain no document root is found()
+ docRoot = true;
+ } else {
+ docRoot = false;
+ }
+
+ String childValue = ConfigParseUtil.getStringAttribute(element, ConfigNameConstants.CHILD, null);
+ child = "true".equals(childValue);
+
+ Map<String,String> modAttributes = ConfigParseUtil
+ .getAllAttributes(element);
+ modAttributes.put(ConfigNameConstants.DATA_SRC, this.dataSourceName);
+ this.allAttributes = Collections.unmodifiableMap(modAttributes);
+
+ List<Element> n = ConfigParseUtil.getChildNodes(element, "field");
+ List<EntityField> modFields = new ArrayList<>(n.size());
+ Map<String,Set<EntityField>> modColNameVsField = new HashMap<>();
+ List<Map<String,String>> modAllFieldAttributes = new ArrayList<>();
+ for (Element elem : n) {
+ EntityField.Builder fieldBuilder = new EntityField.Builder(elem);
+ if (config.getSchema() != null) {
+ if (fieldBuilder.getNameOrColumn() != null
+ && fieldBuilder.getNameOrColumn().contains("${")) {
+ fieldBuilder.dynamicName = true;
+ } else {
+ SchemaField schemaField = config.getSchemaField
+ (fieldBuilder.getNameOrColumn());
+ if (schemaField != null) {
+ fieldBuilder.name = schemaField.getName();
+ fieldBuilder.multiValued = schemaField.multiValued();
+ fieldBuilder.allAttributes.put(DataImporter.MULTI_VALUED, Boolean
+ .toString(schemaField.multiValued()));
+ fieldBuilder.allAttributes.put(DataImporter.TYPE, schemaField
+ .getType().getTypeName());
+ fieldBuilder.allAttributes.put("indexed", Boolean
+ .toString(schemaField.indexed()));
+ fieldBuilder.allAttributes.put("stored", Boolean
+ .toString(schemaField.stored()));
+ fieldBuilder.allAttributes.put("defaultValue", schemaField
+ .getDefaultValue());
+ } else {
+ fieldBuilder.toWrite = false;
+ }
+ }
+ }
+ Set<EntityField> fieldSet = modColNameVsField.get(fieldBuilder.column);
+ if (fieldSet == null) {
+ fieldSet = new HashSet<>();
+ modColNameVsField.put(fieldBuilder.column, fieldSet);
+ }
+ fieldBuilder.allAttributes.put("boost", Float
+ .toString(fieldBuilder.boost));
+ fieldBuilder.allAttributes.put("toWrite", Boolean
+ .toString(fieldBuilder.toWrite));
+ modAllFieldAttributes.add(fieldBuilder.allAttributes);
+ fieldBuilder.entity = this;
+ EntityField field = new EntityField(fieldBuilder);
+ fieldSet.add(field);
+ modFields.add(field);
+ }
+ Map<String,Set<EntityField>> modColNameVsField1 = new HashMap<>();
+ for (Map.Entry<String,Set<EntityField>> entry : modColNameVsField
+ .entrySet()) {
+ if (entry.getValue().size() > 0) {
+ modColNameVsField1.put(entry.getKey(), Collections
+ .unmodifiableSet(entry.getValue()));
+ }
+ }
+ this.colNameVsField = Collections.unmodifiableMap(modColNameVsField1);
+ this.fields = Collections.unmodifiableList(modFields);
+ this.allFieldAttributes = Collections
+ .unmodifiableList(modAllFieldAttributes);
+
+ String modPkMappingFromSchema = null;
+ if (config.getSchema() != null) {
+ SchemaField uniqueKey = config.getSchema().getUniqueKeyField();
+ if (uniqueKey != null) {
+ modPkMappingFromSchema = uniqueKey.getName();
+ // if no fields are mentioned . solr uniqueKey is same as dih 'pk'
+ for (EntityField field : fields) {
+ if (field.getName().equals(modPkMappingFromSchema)) {
+ modPkMappingFromSchema = field.getColumn();
+ // get the corresponding column mapping for the solr uniqueKey
+ // But if there are multiple columns mapping to the solr uniqueKey,
+ // it will fail
+ // so , in one off cases we may need pk
+ break;
+ }
+ }
+ }
+ }
+ pkMappingFromSchema = modPkMappingFromSchema;
+ n = ConfigParseUtil.getChildNodes(element, "entity");
+ List<Entity> modEntities = new ArrayList<>();
+ for (Element elem : n) {
+ modEntities.add(new Entity((docRootFound || this.docRoot), elem, di, config, this));
+ }
+ this.children = Collections.unmodifiableList(modEntities);
+ }
+
+ public String getPk() {
+ return pk == null ? pkMappingFromSchema : pk;
+ }
+
+ public String getSchemaPk() {
+ return pkMappingFromSchema != null ? pkMappingFromSchema : pk;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getPkMappingFromSchema() {
+ return pkMappingFromSchema;
+ }
+
+ public String getDataSourceName() {
+ return dataSourceName;
+ }
+
+ public String getProcessorName() {
+ return processorName;
+ }
+
+ public Entity getParentEntity() {
+ return parentEntity;
+ }
+
+ public boolean isDocRoot() {
+ return docRoot;
+ }
+
+ public List<Entity> getChildren() {
+ return children;
+ }
+
+ public List<EntityField> getFields() {
+ return fields;
+ }
+
+ public Map<String,Set<EntityField>> getColNameVsField() {
+ return colNameVsField;
+ }
+
+ public Map<String,String> getAllAttributes() {
+ return allAttributes;
+ }
+
+ public List<Map<String,String>> getAllFieldsList() {
+ return allFieldAttributes;
+ }
+
+ public boolean isChild() {
+ return child;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/EntityField.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/EntityField.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/EntityField.java
new file mode 100644
index 0000000..2b28cb7
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/EntityField.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport.config;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.solr.handler.dataimport.ConfigParseUtil;
+import org.apache.solr.handler.dataimport.DataImportHandlerException;
+import org.apache.solr.handler.dataimport.DataImporter;
+import org.w3c.dom.Element;
+
+public class EntityField {
+ private final String column;
+ private final String name;
+ private final boolean toWrite;
+ private final boolean multiValued;
+ private final boolean dynamicName;
+ private final Entity entity;
+ private final Map<String, String> allAttributes;
+
+ public EntityField(Builder b) {
+ this.column = b.column;
+ this.name = b.name;
+ this.toWrite = b.toWrite;
+ this.multiValued = b.multiValued;
+ this.dynamicName = b.dynamicName;
+ this.entity = b.entity;
+ this.allAttributes = Collections.unmodifiableMap(new HashMap<>(b.allAttributes));
+ }
+
+ public String getName() {
+ return name == null ? column : name;
+ }
+
+ public Entity getEntity() {
+ return entity;
+ }
+
+ public String getColumn() {
+ return column;
+ }
+
+ public boolean isToWrite() {
+ return toWrite;
+ }
+
+ public boolean isMultiValued() {
+ return multiValued;
+ }
+
+ public boolean isDynamicName() {
+ return dynamicName;
+ }
+
+ public Map<String,String> getAllAttributes() {
+ return allAttributes;
+ }
+
+ public static class Builder {
+ public String column;
+ public String name;
+ public float boost;
+ public boolean toWrite = true;
+ public boolean multiValued = false;
+ public boolean dynamicName = false;
+ public Entity entity;
+ public Map<String, String> allAttributes = new HashMap<>();
+
+ public Builder(Element e) {
+ this.name = ConfigParseUtil.getStringAttribute(e, DataImporter.NAME, null);
+ this.column = ConfigParseUtil.getStringAttribute(e, DataImporter.COLUMN, null);
+ if (column == null) {
+ throw new DataImportHandlerException(SEVERE, "Field must have a column attribute");
+ }
+ this.boost = Float.parseFloat(ConfigParseUtil.getStringAttribute(e, "boost", "1.0f"));
+ this.allAttributes = new HashMap<>(ConfigParseUtil.getAllAttributes(e));
+ }
+
+ public String getNameOrColumn() {
+ return name==null ? column : name;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/Field.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/Field.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/Field.java
new file mode 100644
index 0000000..7ff4832
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/Field.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport.config;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.solr.handler.dataimport.ConfigParseUtil;
+import org.apache.solr.handler.dataimport.DataImportHandlerException;
+import org.apache.solr.handler.dataimport.DataImporter;
+import org.w3c.dom.Element;
+
+public class Field {
+ private final String column;
+ private final String name;
+ private final float boost;
+ private final boolean toWrite;
+ private final boolean multiValued;
+ private final boolean dynamicName;
+ private final Entity entity;
+ private final Map<String, String> allAttributes;
+
+ public Field(Builder b) {
+ this.column = b.column;
+ this.name = b.name;
+ this.boost = b.boost;
+ this.toWrite = b.toWrite;
+ this.multiValued = b.multiValued;
+ this.dynamicName = b.dynamicName;
+ this.entity = b.entity;
+ this.allAttributes = Collections.unmodifiableMap(new HashMap<>(b.allAttributes));
+ }
+
+ public String getName() {
+ return name == null ? column : name;
+ }
+
+ public Entity getEntity() {
+ return entity;
+ }
+
+ public String getColumn() {
+ return column;
+ }
+
+ public float getBoost() {
+ return boost;
+ }
+
+ public boolean isToWrite() {
+ return toWrite;
+ }
+
+ public boolean isMultiValued() {
+ return multiValued;
+ }
+
+ public boolean isDynamicName() {
+ return dynamicName;
+ }
+
+ public Map<String,String> getAllAttributes() {
+ return allAttributes;
+ }
+
+ public static class Builder {
+ public String column;
+ public String name;
+ public float boost;
+ public boolean toWrite = true;
+ public boolean multiValued = false;
+ public boolean dynamicName;
+ public Entity entity;
+ public Map<String, String> allAttributes = new HashMap<>();
+
+ public Builder(Element e) {
+ this.name = ConfigParseUtil.getStringAttribute(e, DataImporter.NAME, null);
+ this.column = ConfigParseUtil.getStringAttribute(e, DataImporter.COLUMN, null);
+ if (column == null) {
+ throw new DataImportHandlerException(SEVERE, "Field must have a column attribute");
+ }
+ this.boost = Float.parseFloat(ConfigParseUtil.getStringAttribute(e, "boost", "1.0f"));
+ this.allAttributes = new HashMap<>(ConfigParseUtil.getAllAttributes(e));
+ }
+
+ public String getNameOrColumn() {
+ return name==null ? column : name;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/PropertyWriter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/PropertyWriter.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/PropertyWriter.java
new file mode 100644
index 0000000..6e91a19
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/PropertyWriter.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport.config;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class PropertyWriter {
+ private final String type;
+ private Map<String,String> parameters;
+
+ public PropertyWriter(String type, Map<String,String> parameters) {
+ this.type = type;
+ this.parameters = new HashMap<String,String>(parameters);
+ }
+
+ public Map<String,String> getParameters() {
+ return parameters;
+ }
+
+ public String getType() {
+ return type;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/Script.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/Script.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/Script.java
new file mode 100644
index 0000000..9a4bc59
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/Script.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport.config;
+
+import org.w3c.dom.Element;
+
+public class Script {
+ private final String language;
+ private final String text;
+
+ public Script(Element e) {
+ this.language = ConfigParseUtil.getStringAttribute(e, "language", "JavaScript");
+ StringBuilder buffer = new StringBuilder();
+ String script = ConfigParseUtil.getText(e, buffer);
+ if (script != null) {
+ this.text = script.trim();
+ } else {
+ this.text = null;
+ }
+ }
+ public String getLanguage() {
+ return language;
+ }
+ public String getText() {
+ return text;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/package-info.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/package-info.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/package-info.java
new file mode 100644
index 0000000..50c6d4e
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/config/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Utility classes for parsing & modeling DIH configuration.
+ */
+package org.apache.solr.handler.dataimport.config;
+
+
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/package-info.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/package-info.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/package-info.java
new file mode 100644
index 0000000..4a69d23
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * {@link org.apache.solr.handler.dataimport.DataImportHandler} and related code.
+ */
+package org.apache.solr.handler.dataimport;
+
+
+
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/overview.html
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/overview.html b/solr/contrib/dataimporthandler/src/main/java/overview.html
new file mode 100644
index 0000000..4bb2c8b
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/overview.html
@@ -0,0 +1,21 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<body>
+Apache Solr Search Server: DataImportHandler contrib
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/contentstream-solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/contentstream-solrconfig.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/contentstream-solrconfig.xml
deleted file mode 100644
index a07ab78..0000000
--- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/contentstream-solrconfig.xml
+++ /dev/null
@@ -1,293 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
- <indexConfig>
- <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
- </indexConfig>
-
- <!-- Used to specify an alternate directory to hold all index data
- other than the default ./data under the Solr home.
- If replication is in use, this should match the replication configuration. -->
- <dataDir>${solr.data.dir:}</dataDir>
-
- <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
- <schemaFactory class="ClassicIndexSchemaFactory"/>
-
- <!-- the default high-performance update handler -->
- <updateHandler class="solr.DirectUpdateHandler2">
-
- <!-- A prefix of "solr." for class names is an alias that
- causes solr to search appropriate packages, including
- org.apache.solr.(search|update|request|core|analysis)
- -->
-
- <!-- Limit the number of deletions Solr will buffer during doc updating.
-
- Setting this lower can help bound memory use during indexing.
- -->
- <maxPendingDeletes>100000</maxPendingDeletes>
-
- </updateHandler>
-
-
- <query>
- <!-- Maximum number of clauses in a boolean query... can affect
- range or prefix queries that expand to big boolean
- queries. An exception is thrown if exceeded. -->
- <maxBooleanClauses>1024</maxBooleanClauses>
-
-
- <!-- Cache used by SolrIndexSearcher for filters (DocSets),
- unordered sets of *all* documents that match a query.
- When a new searcher is opened, its caches may be prepopulated
- or "autowarmed" using data from caches in the old searcher.
- autowarmCount is the number of items to prepopulate. For LRUCache,
- the autowarmed items will be the most recently accessed items.
- Parameters:
- class - the SolrCache implementation (currently only LRUCache)
- size - the maximum number of entries in the cache
- initialSize - the initial capacity (number of entries) of
- the cache. (seel java.util.HashMap)
- autowarmCount - the number of entries to prepopulate from
- and old cache.
- -->
- <filterCache
- class="solr.LRUCache"
- size="512"
- initialSize="512"
- autowarmCount="256"/>
-
- <!-- queryResultCache caches results of searches - ordered lists of
- document ids (DocList) based on a query, a sort, and the range
- of documents requested. -->
- <queryResultCache
- class="solr.LRUCache"
- size="512"
- initialSize="512"
- autowarmCount="256"/>
-
- <!-- documentCache caches Lucene Document objects (the stored fields for each document).
- Since Lucene internal document ids are transient, this cache will not be autowarmed. -->
- <documentCache
- class="solr.LRUCache"
- size="512"
- initialSize="512"
- autowarmCount="0"/>
-
- <!-- If true, stored fields that are not requested will be loaded lazily.
-
- This can result in a significant speed improvement if the usual case is to
- not load all stored fields, especially if the skipped fields are large compressed
- text fields.
- -->
- <enableLazyFieldLoading>true</enableLazyFieldLoading>
-
- <!-- Example of a generic cache. These caches may be accessed by name
- through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert().
- The purpose is to enable easy caching of user/application level data.
- The regenerator argument should be specified as an implementation
- of solr.search.CacheRegenerator if autowarming is desired. -->
- <!--
- <cache name="myUserCache"
- class="solr.LRUCache"
- size="4096"
- initialSize="1024"
- autowarmCount="1024"
- regenerator="org.mycompany.mypackage.MyRegenerator"
- />
- -->
-
- <!-- An optimization that attempts to use a filter to satisfy a search.
- If the requested sort does not include score, then the filterCache
- will be checked for a filter matching the query. If found, the filter
- will be used as the source of document ids, and then the sort will be
- applied to that.
- <useFilterForSortedQuery>true</useFilterForSortedQuery>
- -->
-
- <!-- An optimization for use with the queryResultCache. When a search
- is requested, a superset of the requested number of document ids
- are collected. For example, if a search for a particular query
- requests matching documents 10 through 19, and queryWindowSize is 50,
- then documents 0 through 49 will be collected and cached. Any further
- requests in that range can be satisfied via the cache. -->
- <queryResultWindowSize>50</queryResultWindowSize>
-
- <!-- Maximum number of documents to cache for any entry in the
- queryResultCache. -->
- <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
-
- <!-- This entry enables an int hash representation for filters (DocSets)
- when the number of items in the set is less than maxSize. For smaller
- sets, this representation is more memory efficient, more efficient to
- iterate over, and faster to take intersections. -->
- <HashDocSet maxSize="3000" loadFactor="0.75"/>
-
- <!-- a newSearcher event is fired whenever a new searcher is being prepared
- and there is a current searcher handling requests (aka registered). -->
- <!-- QuerySenderListener takes an array of NamedList and executes a
- local query request for each NamedList in sequence. -->
- <!--<listener event="newSearcher" class="solr.QuerySenderListener">-->
- <!--<arr name="queries">-->
- <!--<lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
- <!--<lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
- <!--<lst><str name="q">static newSearcher warming query from solrconfig.xml</str></lst>-->
- <!--</arr>-->
- <!--</listener>-->
-
- <!-- a firstSearcher event is fired whenever a new searcher is being
- prepared but there is no current registered searcher to handle
- requests or to gain autowarming data from. -->
- <!--<listener event="firstSearcher" class="solr.QuerySenderListener">-->
- <!--<arr name="queries">-->
- <!--</arr>-->
- <!--</listener>-->
-
- <!-- If a search request comes in and there is no current registered searcher,
- then immediately register the still warming searcher and use it. If
- "false" then all requests will block until the first searcher is done
- warming. -->
- <useColdSearcher>false</useColdSearcher>
-
- <!-- Maximum number of searchers that may be warming in the background
- concurrently. An error is returned if this limit is exceeded. Recommend
- 1-2 for read-only slaves, higher for masters w/o cache warming. -->
- <maxWarmingSearchers>4</maxWarmingSearchers>
-
- </query>
-
- <requestDispatcher>
- <!--Make sure your system has some authentication before enabling remote streaming!
- <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="-1" />
- -->
-
- <!-- Set HTTP caching related parameters (for proxy caches and clients).
-
- To get the behaviour of Solr 1.2 (ie: no caching related headers)
- use the never304="true" option and do not specify a value for
- <cacheControl>
- -->
- <httpCaching never304="true">
- <!--httpCaching lastModifiedFrom="openTime"
- etagSeed="Solr"-->
- <!-- lastModFrom="openTime" is the default, the Last-Modified value
- (and validation against If-Modified-Since requests) will all be
- relative to when the current Searcher was opened.
- You can change it to lastModFrom="dirLastMod" if you want the
- value to exactly corrispond to when the physical index was last
- modified.
-
- etagSeed="..." is an option you can change to force the ETag
- header (and validation against If-None-Match requests) to be
- differnet even if the index has not changed (ie: when making
- significant changes to your config file)
-
- lastModifiedFrom and etagSeed are both ignored if you use the
- never304="true" option.
- -->
- <!-- If you include a <cacheControl> directive, it will be used to
- generate a Cache-Control header, as well as an Expires header
- if the value contains "max-age="
-
- By default, no Cache-Control header is generated.
-
- You can use the <cacheControl> option even if you have set
- never304="true"
- -->
- <!-- <cacheControl>max-age=30, public</cacheControl> -->
- </httpCaching>
- </requestDispatcher>
-
- <requestHandler name="/select" class="solr.SearchHandler">
- <!-- default values for query parameters -->
- <lst name="defaults">
- <str name="echoParams">explicit</str>
- <!--
- <int name="rows">10</int>
- <str name="fl">*</str>
- <str name="version">2.1</str>
- -->
- </lst>
- </requestHandler>
-
- <requestHandler name="/dataimport" class="org.apache.solr.handler.dataimport.DataImportHandler">
- <lst name="defaults">
- <str name="config">data-config.xml</str>
-
- </lst>
- </requestHandler>
-
- <!--
-
- Search components are registered to SolrCore and used by Search Handlers
-
- By default, the following components are avaliable:
-
- <searchComponent name="query" class="org.apache.solr.handler.component.QueryComponent" />
- <searchComponent name="facet" class="org.apache.solr.handler.component.FacetComponent" />
- <searchComponent name="mlt" class="org.apache.solr.handler.component.MoreLikeThisComponent" />
- <searchComponent name="highlight" class="org.apache.solr.handler.component.HighlightComponent" />
- <searchComponent name="debug" class="org.apache.solr.handler.component.DebugComponent" />
-
- If you register a searchComponent to one of the standard names, that will be used instead.
-
- -->
-
- <requestHandler name="/search" class="org.apache.solr.handler.component.SearchHandler">
- <lst name="defaults">
- <str name="echoParams">explicit</str>
- </lst>
- <!--
- By default, this will register the following components:
-
- <arr name="components">
- <str>query</str>
- <str>facet</str>
- <str>mlt</str>
- <str>highlight</str>
- <str>debug</str>
- </arr>
-
- To insert handlers before or after the 'standard' components, use:
-
- <arr name="first-components">
- <str>first</str>
- </arr>
-
- <arr name="last-components">
- <str>last</str>
- </arr>
-
- -->
- </requestHandler>
-
- <!-- config for the admin interface -->
- <admin>
- <defaultQuery>*:*</defaultQuery>
- </admin>
-
- <updateRequestProcessorChain key="contentstream" default="true">
- <processor class="org.apache.solr.handler.dataimport.AbstractDataImportHandlerTestCase$TestUpdateRequestProcessorFactory"/>
- <processor class="solr.RunUpdateProcessorFactory"/>
- <processor class="solr.LogUpdateProcessorFactory"/>
- </updateRequestProcessorChain>
-
-</config>
-
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-end-to-end.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-end-to-end.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-end-to-end.xml
deleted file mode 100644
index a582112..0000000
--- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-end-to-end.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<dataConfig>
- <dataSource name="hsqldb" driver="org.hsqldb.jdbcDriver" url="jdbc:hsqldb:mem:." />
- <document name="dih_end_to_end">
- <entity
- name="People"
- processor="SqlEntityProcessor"
- dataSource="hsqldb"
- query="SELECT ID, NAME, COUNTRY_CODES FROM PEOPLE"
- transformer="RegexTransformer"
- >
- <field column="ID" name="id" />
- <field column="COUNTRY_CODE" sourceColName="COUNTRY_CODES" splitBy="," />
-
-<!--
- Instead of using 'cachePk'/'cacheLookup' as done below, we could have done:
- where="CODE=People.COUNTRY_CODE"
--->
- <entity
- name="Countries"
- processor="SqlEntityProcessor"
- dataSource="hsqldb"
- cacheImpl="SortedMapBackedCache"
- cacheKey="CODE"
- cacheLookup="People.COUNTRY_CODE"
-
- query="SELECT CODE, COUNTRY_NAME FROM COUNTRIES"
- >
- <field column="CODE" name="DO_NOT_INDEX" />
- </entity>
-
- <entity
- name="Sports"
- processor="SqlEntityProcessor"
- dataSource="hsqldb"
- query="SELECT PERSON_ID, SPORT_NAME FROM PEOPLE_SPORTS WHERE PERSON_ID=${People.ID}"
- />
-
- </entity>
- </document>
-</dataConfig>
-
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-with-datasource.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-with-datasource.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-with-datasource.xml
deleted file mode 100644
index 46a6603..0000000
--- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-with-datasource.xml
+++ /dev/null
@@ -1,9 +0,0 @@
-<dataConfig>
- <dataSource type="MockDataSource" />
- <document>
- <entity name="x" query="select * from x">
- <field column="id" />
- <field column="desc" />
- </entity>
- </document>
-</dataConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-with-transformer.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-with-transformer.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-with-transformer.xml
deleted file mode 100644
index 925e6c2..0000000
--- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-with-transformer.xml
+++ /dev/null
@@ -1,10 +0,0 @@
-<dataConfig>
- <dataSource type="MockDataSource" />
- <dataSource name="mockDs" type="TestDocBuilder2$MockDataSource2" />
- <document>
- <entity name="x" query="select * from x" transformer="TestDocBuilder2$MockTransformer">
- <field column="id" />
- <field column="desc" />
- </entity>
- </document>
-</dataConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataconfig-contentstream.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataconfig-contentstream.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataconfig-contentstream.xml
deleted file mode 100644
index 7520e74..0000000
--- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataconfig-contentstream.xml
+++ /dev/null
@@ -1,10 +0,0 @@
-<dataConfig>
- <dataSource type="ContentStreamDataSource" name="c"/>
- <document>
- <entity name="b" dataSource="c" processor="XPathEntityProcessor"
- forEach="/root/b">
- <field column="desc" xpath="/root/b/c"/>
- <field column="id" xpath="/root/b/id"/>
- </entity>
- </document>
-</dataConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml
deleted file mode 100644
index 6754f9e..0000000
--- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml
+++ /dev/null
@@ -1,285 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
-
- <!-- Used to specify an alternate directory to hold all index data
- other than the default ./data under the Solr home.
- If replication is in use, this should match the replication configuration. -->
- <dataDir>${solr.data.dir:}</dataDir>
-
- <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
- <schemaFactory class="ClassicIndexSchemaFactory"/>
-
- <indexConfig>
- <lockType>single</lockType>
- <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
- </indexConfig>
-
- <!-- the default high-performance update handler -->
- <updateHandler class="solr.DirectUpdateHandler2">
-
- <!-- A prefix of "solr." for class names is an alias that
- causes solr to search appropriate packages, including
- org.apache.solr.(search|update|request|core|analysis)
- -->
-
- <!-- Limit the number of deletions Solr will buffer during doc updating.
-
- Setting this lower can help bound memory use during indexing.
- -->
- <maxPendingDeletes>100000</maxPendingDeletes>
-
- </updateHandler>
-
-
- <query>
- <!-- Maximum number of clauses in a boolean query... can affect
- range or prefix queries that expand to big boolean
- queries. An exception is thrown if exceeded. -->
- <maxBooleanClauses>1024</maxBooleanClauses>
-
-
- <!-- Cache used by SolrIndexSearcher for filters (DocSets),
- unordered sets of *all* documents that match a query.
- When a new searcher is opened, its caches may be prepopulated
- or "autowarmed" using data from caches in the old searcher.
- autowarmCount is the number of items to prepopulate. For LRUCache,
- the autowarmed items will be the most recently accessed items.
- Parameters:
- class - the SolrCache implementation (currently only LRUCache)
- size - the maximum number of entries in the cache
- initialSize - the initial capacity (number of entries) of
- the cache. (seel java.util.HashMap)
- autowarmCount - the number of entries to prepopulate from
- and old cache.
- -->
- <filterCache
- class="solr.LRUCache"
- size="512"
- initialSize="512"
- autowarmCount="256"/>
-
- <!-- queryResultCache caches results of searches - ordered lists of
- document ids (DocList) based on a query, a sort, and the range
- of documents requested. -->
- <queryResultCache
- class="solr.LRUCache"
- size="512"
- initialSize="512"
- autowarmCount="256"/>
-
- <!-- documentCache caches Lucene Document objects (the stored fields for each document).
- Since Lucene internal document ids are transient, this cache will not be autowarmed. -->
- <documentCache
- class="solr.LRUCache"
- size="512"
- initialSize="512"
- autowarmCount="0"/>
-
- <!-- If true, stored fields that are not requested will be loaded lazily.
-
- This can result in a significant speed improvement if the usual case is to
- not load all stored fields, especially if the skipped fields are large compressed
- text fields.
- -->
- <enableLazyFieldLoading>true</enableLazyFieldLoading>
-
- <!-- Example of a generic cache. These caches may be accessed by name
- through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert().
- The purpose is to enable easy caching of user/application level data.
- The regenerator argument should be specified as an implementation
- of solr.search.CacheRegenerator if autowarming is desired. -->
- <!--
- <cache name="myUserCache"
- class="solr.LRUCache"
- size="4096"
- initialSize="1024"
- autowarmCount="1024"
- regenerator="org.mycompany.mypackage.MyRegenerator"
- />
- -->
-
- <!-- An optimization that attempts to use a filter to satisfy a search.
- If the requested sort does not include score, then the filterCache
- will be checked for a filter matching the query. If found, the filter
- will be used as the source of document ids, and then the sort will be
- applied to that.
- <useFilterForSortedQuery>true</useFilterForSortedQuery>
- -->
-
- <!-- An optimization for use with the queryResultCache. When a search
- is requested, a superset of the requested number of document ids
- are collected. For example, if a search for a particular query
- requests matching documents 10 through 19, and queryWindowSize is 50,
- then documents 0 through 49 will be collected and cached. Any further
- requests in that range can be satisfied via the cache. -->
- <queryResultWindowSize>50</queryResultWindowSize>
-
- <!-- Maximum number of documents to cache for any entry in the
- queryResultCache. -->
- <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
-
- <!-- This entry enables an int hash representation for filters (DocSets)
- when the number of items in the set is less than maxSize. For smaller
- sets, this representation is more memory efficient, more efficient to
- iterate over, and faster to take intersections. -->
- <HashDocSet maxSize="3000" loadFactor="0.75"/>
-
- <!-- a newSearcher event is fired whenever a new searcher is being prepared
- and there is a current searcher handling requests (aka registered). -->
- <!-- QuerySenderListener takes an array of NamedList and executes a
- local query request for each NamedList in sequence. -->
- <!--<listener event="newSearcher" class="solr.QuerySenderListener">-->
- <!--<arr name="queries">-->
- <!--<lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
- <!--<lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
- <!--<lst><str name="q">static newSearcher warming query from solrconfig.xml</str></lst>-->
- <!--</arr>-->
- <!--</listener>-->
-
- <!-- a firstSearcher event is fired whenever a new searcher is being
- prepared but there is no current registered searcher to handle
- requests or to gain autowarming data from. -->
- <!--<listener event="firstSearcher" class="solr.QuerySenderListener">-->
- <!--<arr name="queries">-->
- <!--</arr>-->
- <!--</listener>-->
-
- <!-- If a search request comes in and there is no current registered searcher,
- then immediately register the still warming searcher and use it. If
- "false" then all requests will block until the first searcher is done
- warming. -->
- <useColdSearcher>false</useColdSearcher>
-
- <!-- Maximum number of searchers that may be warming in the background
- concurrently. An error is returned if this limit is exceeded. Recommend
- 1-2 for read-only slaves, higher for masters w/o cache warming. -->
- <maxWarmingSearchers>4</maxWarmingSearchers>
-
- </query>
-
- <requestDispatcher>
- <!--Make sure your system has some authentication before enabling remote streaming!
- <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="-1" />
- -->
-
- <!-- Set HTTP caching related parameters (for proxy caches and clients).
-
- To get the behaviour of Solr 1.2 (ie: no caching related headers)
- use the never304="true" option and do not specify a value for
- <cacheControl>
- -->
- <httpCaching never304="true">
- <!--httpCaching lastModifiedFrom="openTime"
- etagSeed="Solr"-->
- <!-- lastModFrom="openTime" is the default, the Last-Modified value
- (and validation against If-Modified-Since requests) will all be
- relative to when the current Searcher was opened.
- You can change it to lastModFrom="dirLastMod" if you want the
- value to exactly corrispond to when the physical index was last
- modified.
-
- etagSeed="..." is an option you can change to force the ETag
- header (and validation against If-None-Match requests) to be
- differnet even if the index has not changed (ie: when making
- significant changes to your config file)
-
- lastModifiedFrom and etagSeed are both ignored if you use the
- never304="true" option.
- -->
- <!-- If you include a <cacheControl> directive, it will be used to
- generate a Cache-Control header, as well as an Expires header
- if the value contains "max-age="
-
- By default, no Cache-Control header is generated.
-
- You can use the <cacheControl> option even if you have set
- never304="true"
- -->
- <!-- <cacheControl>max-age=30, public</cacheControl> -->
- </httpCaching>
- </requestDispatcher>
-
- <requestHandler name="/select" class="solr.SearchHandler">
- <!-- default values for query parameters -->
- <lst name="defaults">
- <str name="echoParams">explicit</str>
- <!--
- <int name="rows">10</int>
- <str name="fl">*</str>
- <str name="version">2.1</str>
- -->
- </lst>
- </requestHandler>
-
- <requestHandler name="/dataimport" class="org.apache.solr.handler.dataimport.DataImportHandler">
- </requestHandler>
-
- <!--
-
- Search components are registered to SolrCore and used by Search Handlers
-
- By default, the following components are avaliable:
-
- <searchComponent name="query" class="org.apache.solr.handler.component.QueryComponent" />
- <searchComponent name="facet" class="org.apache.solr.handler.component.FacetComponent" />
- <searchComponent name="mlt" class="org.apache.solr.handler.component.MoreLikeThisComponent" />
- <searchComponent name="highlight" class="org.apache.solr.handler.component.HighlightComponent" />
- <searchComponent name="debug" class="org.apache.solr.handler.component.DebugComponent" />
-
- If you register a searchComponent to one of the standard names, that will be used instead.
-
- -->
-
- <requestHandler name="/search" class="org.apache.solr.handler.component.SearchHandler">
- <lst name="defaults">
- <str name="echoParams">explicit</str>
- </lst>
- <!--
- By default, this will register the following components:
-
- <arr name="components">
- <str>query</str>
- <str>facet</str>
- <str>mlt</str>
- <str>highlight</str>
- <str>debug</str>
- </arr>
-
- To insert handlers before or after the 'standard' components, use:
-
- <arr name="first-components">
- <str>first</str>
- </arr>
-
- <arr name="last-components">
- <str>last</str>
- </arr>
-
- -->
- </requestHandler>
-
- <!-- config for the admin interface -->
- <admin>
- <defaultQuery>*:*</defaultQuery>
- </admin>
-
-</config>
-
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-schema.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-schema.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-schema.xml
deleted file mode 100644
index 7187138..0000000
--- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-schema.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-<schema name="dih_test" version="4.0">
-
- <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
- <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
- <fieldType name="tint" class="${solr.tests.IntegerFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tfloat" class="${solr.tests.FloatFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tlong" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tdouble" class="${solr.tests.DoubleFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="date" class="${solr.tests.DateFieldType}" docValues="${solr.tests.numeric.dv}" sortMissingLast="true" omitNorms="true"/>
- <fieldType name="text" class="solr.TextField" positionIncrementGap="100">
- <analyzer type="index">
- <tokenizer class="solr.MockTokenizerFactory"/>
- <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1"
- catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
- <filter class="solr.LowerCaseFilterFactory"/>
- <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
- <filter class="solr.FlattenGraphFilterFactory" />
- </analyzer>
- <analyzer type="query">
- <tokenizer class="solr.MockTokenizerFactory"/>
- <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0"
- catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
- <filter class="solr.LowerCaseFilterFactory"/>
- <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
- </analyzer>
- </fieldType>
- <fieldType name="textTight" class="solr.TextField" positionIncrementGap="100">
- <analyzer type="index">
- <tokenizer class="solr.MockTokenizerFactory"/>
- <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1"
- catenateNumbers="1" catenateAll="0"/>
- <filter class="solr.LowerCaseFilterFactory"/>
- <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
- <filter class="solr.FlattenGraphFilterFactory" />
- </analyzer>
- <analyzer type="query">
- <tokenizer class="solr.MockTokenizerFactory"/>
- <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1"
- catenateNumbers="1" catenateAll="0"/>
- <filter class="solr.LowerCaseFilterFactory"/>
- <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
- </analyzer>
- </fieldType>
- <fieldType name="ignored" stored="false" indexed="false" class="solr.StrField"/>
-
- <field name="id" type="string" indexed="true" stored="true" required="true"/>
- <field name="desc" type="string" indexed="true" stored="true" multiValued="true"/>
- <field name="date" type="date" indexed="true" stored="true"/>
- <field name="timestamp" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
-
- <field name="NAME" type="text" indexed="true" stored="true" multiValued="false"/>
- <field name="COUNTRY_NAME" type="text" indexed="true" stored="true" multiValued="true"/>
- <field name="SPORT_NAME" type="text" indexed="true" stored="true" multiValued="true"/>
- <field name="DO_NOT_INDEX" type="ignored"/>
-
- <field name="_version_" type="tlong" indexed="true" stored="true" multiValued="false"/>
- <field name="_root_" type="string" indexed="true" stored="true" multiValued="false"/>
-
- <dynamicField name="*_i" type="tint" indexed="true" stored="true"/>
- <dynamicField name="*_s" type="string" indexed="true" stored="true"/>
- <dynamicField name="*_mult_s" type="string" indexed="true" stored="true" multiValued="true"/>
- <dynamicField name="*_l" type="tlong" indexed="true" stored="true"/>
- <dynamicField name="*_t" type="text" indexed="true" stored="true"/>
- <dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
- <dynamicField name="*_f" type="tfloat" indexed="true" stored="true"/>
- <dynamicField name="*_d" type="tdouble" indexed="true" stored="true"/>
- <dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
-
- <uniqueKey>id</uniqueKey>
-</schema>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml
deleted file mode 100644
index e8deee4..0000000
--- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml
+++ /dev/null
@@ -1,313 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!--
- This is the Solr schema file. This file should be named "schema.xml" and
- should be in the conf directory under the solr home
- (i.e. ./solr/conf/schema.xml by default)
- or located where the classloader for the Solr webapp can find it.
-
- This example schema is the recommended starting point for users.
- It should be kept correct and concise, usable out-of-the-box.
-
- For more information, on how to customize this file, please see
- http://wiki.apache.org/solr/SchemaXml
--->
-
-<schema name="test" version="1.1">
- <!-- attribute "name" is the name of this schema and is only used for display purposes.
- Applications should change this to reflect the nature of the search collection.
- version="1.1" is Solr's version number for the schema syntax and semantics. It should
- not normally be changed by applications.
- 1.0: multiValued attribute did not exist, all fields are multiValued by nature
- 1.1: multiValued attribute introduced, false by default -->
-
-
- <!-- field type definitions. The "name" attribute is
- just a label to be used by field definitions. The "class"
- attribute and any other attributes determine the real
- behavior of the fieldType.
- Class names starting with "solr" refer to java classes in the
- org.apache.solr.analysis package.
- -->
-
- <!-- The StrField type is not analyzed, but indexed/stored verbatim.
- - StrField and TextField support an optional compressThreshold which
- limits compression (if enabled in the derived fields) to values which
- exceed a certain size (in characters).
- -->
- <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
-
- <!-- boolean type: "true" or "false" -->
- <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
-
- <!-- The optional sortMissingLast and sortMissingFirst attributes are
- currently supported on types that are sorted internally as strings.
- - If sortMissingLast="true", then a sort on this field will cause documents
- without the field to come after documents with the field,
- regardless of the requested sort order (asc or desc).
- - If sortMissingFirst="true", then a sort on this field will cause documents
- without the field to come before documents with the field,
- regardless of the requested sort order.
- - If sortMissingLast="false" and sortMissingFirst="false" (the default),
- then default lucene sorting will be used which places docs without the
- field first in an ascending sort and last in a descending sort.
- -->
-
-
- <!--
- Default numeric field types. For faster range queries, consider the tint/tfloat/tlong/tdouble types.
- -->
- <fieldType name="int" class="${solr.tests.IntegerFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="float" class="${solr.tests.FloatFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="long" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="double" class="${solr.tests.DoubleFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" positionIncrementGap="0"/>
-
- <!--
- Numeric field types that index each value at various levels of precision
- to accelerate range queries when the number of values between the range
- endpoints is large. See the javadoc for LegacyNumericRangeQuery for internal
- implementation details.
-
- Smaller precisionStep values (specified in bits) will lead to more tokens
- indexed per value, slightly larger index size, and faster range queries.
- A precisionStep of 0 disables indexing at different precision levels.
- -->
- <fieldType name="tint" class="${solr.tests.IntegerFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tfloat" class="${solr.tests.FloatFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tlong" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tdouble" class="${solr.tests.DoubleFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
-
-
- <!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
- is a more restricted form of the canonical representation of dateTime
- http://www.w3.org/TR/xmlschema-2/#dateTime
- The trailing "Z" designates UTC time and is mandatory.
- Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
- All other components are mandatory.
-
- Expressions can also be used to denote calculations that should be
- performed relative to "NOW" to determine the value, ie...
-
- NOW/HOUR
- ... Round to the start of the current hour
- NOW-1DAY
- ... Exactly 1 day prior to now
- NOW/DAY+6MONTHS+3DAYS
- ... 6 months and 3 days in the future from the start of
- the current day
-
- Consult the TrieDateField javadocs for more information.
- -->
- <fieldType name="date" class="${solr.tests.DateFieldType}" docValues="${solr.tests.numeric.dv}" sortMissingLast="true" omitNorms="true"/>
-
-
- <!-- The "RandomSortField" is not used to store or search any
- data. You can declare fields of this type it in your schema
- to generate psuedo-random orderings of your docs for sorting
- purposes. The ordering is generated based on the field name
- and the version of the index, As long as the index version
- remains unchanged, and the same field name is reused,
- the ordering of the docs will be consistent.
- If you want differend psuedo-random orderings of documents,
- for the same version of the index, use a dynamicField and
- change the name
- -->
- <fieldType name="random" class="solr.RandomSortField" indexed="true"/>
-
- <!-- solr.TextField allows the specification of custom text analyzers
- specified as a tokenizer and a list of token filters. Different
- analyzers may be specified for indexing and querying.
-
- The optional positionIncrementGap puts space between multiple fields of
- this type on the same document, with the purpose of preventing false phrase
- matching across fields.
-
- For more info on customizing your analyzer chain, please see
- http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters
- -->
-
- <!-- One can also specify an existing Analyzer class that has a
- default constructor via the class attribute on the analyzer element
- <fieldType name="text_greek" class="solr.TextField">
- <analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/>
- </fieldType>
- -->
-
- <!-- A text field that only splits on whitespace for exact matching of words -->
- <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
- <analyzer>
- <tokenizer class="solr.MockTokenizerFactory"/>
- </analyzer>
- </fieldType>
-
- <!-- A text field that uses WordDelimiterGraphFilter to enable splitting and matching of
- words on case-change, alpha numeric boundaries, and non-alphanumeric chars,
- so that a query of "wifi" or "wi fi" could match a document containing "Wi-Fi".
- Synonyms and stopwords are customized by external files, and stemming is enabled.
- Duplicate tokens at the same position (which may result from Stemmed Synonyms or
- WordDelim parts) are removed.
- -->
- <fieldType name="text" class="solr.TextField" positionIncrementGap="100">
- <analyzer type="index">
- <tokenizer class="solr.MockTokenizerFactory"/>
- <!-- in this example, we will only use synonyms at query time
- <filter class="solr.SynonymGraphFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
- -->
- <!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
- <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1"
- catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
- <filter class="solr.LowerCaseFilterFactory"/>
- <!--<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
- <filter class="solr.PorterStemFilterFactory"/>-->
- <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
- <filter class="solr.FlattenGraphFilterFactory"/>
- </analyzer>
- <analyzer type="query">
- <tokenizer class="solr.MockTokenizerFactory"/>
- <!--<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>-->
- <!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
- <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0"
- catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
- <filter class="solr.LowerCaseFilterFactory"/>
- <!--<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
- <filter class="solr.PorterStemFilterFactory"/>-->
- <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
- </analyzer>
- </fieldType>
-
-
- <!-- Less flexible matching, but less false matches. Probably not ideal for product names,
- but may be good for SKUs. Can insert dashes in the wrong place and still match. -->
- <fieldType name="textTight" class="solr.TextField" positionIncrementGap="100">
- <analyzer type="index">
- <tokenizer class="solr.MockTokenizerFactory"/>
- <!--<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="false"/>-->
- <!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
- <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1"
- catenateNumbers="1" catenateAll="0"/>
- <filter class="solr.LowerCaseFilterFactory"/>
- <!--<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
- <filter class="solr.EnglishMinimalStemFilterFactory"/>-->
- <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
- <filter class="solr.FlattenGraphFilterFactory"/>
- </analyzer>
- <analyzer type="query">
- <tokenizer class="solr.MockTokenizerFactory"/>
- <!--<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="false"/>-->
- <!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
- <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1"
- catenateNumbers="1" catenateAll="0"/>
- <filter class="solr.LowerCaseFilterFactory"/>
- <!--<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
- <filter class="solr.EnglishMinimalStemFilterFactory"/>-->
- <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
- </analyzer>
- </fieldType>
-
- <!-- This is an example of using the KeywordTokenizer along
- With various TokenFilterFactories to produce a sortable field
- that does not include some properties of the source text
- -->
- <fieldType name="alphaOnlySort" class="solr.TextField" sortMissingLast="true" omitNorms="true">
- <analyzer>
- <!-- KeywordTokenizer does no actual tokenizing, so the entire
- input string is preserved as a single token
- -->
- <tokenizer class="solr.MockTokenizerFactory" pattern="keyword"/>
- <!-- The LowerCase TokenFilter does what you expect, which can be
- when you want your sorting to be case insensitive
- -->
- <filter class="solr.LowerCaseFilterFactory"/>
- <!-- The TrimFilter removes any leading or trailing whitespace -->
- <filter class="solr.TrimFilterFactory"/>
- <!-- The PatternReplaceFilter gives you the flexibility to use
- Java Regular expression to replace any sequence of characters
- matching a pattern with an arbitrary replacement string,
- which may include back refrences to portions of the orriginal
- string matched by the pattern.
-
- See the Java Regular Expression documentation for more
- infomation on pattern and replacement string syntax.
-
- http://docs.oracle.com/javase/8/docs/api/java/util/regex/package-summary.html
- -->
- <filter class="solr.PatternReplaceFilterFactory"
- pattern="([^a-z])" replacement="" replace="all"
- />
- </analyzer>
- </fieldType>
-
- <!-- since fields of this type are by default not stored or indexed, any data added to
- them will be ignored outright
- -->
- <fieldType name="ignored" stored="false" indexed="false" class="solr.StrField"/>
-
- <!-- Valid attributes for fields:
- name: mandatory - the name for the field
- type: mandatory - the name of a previously defined type from the <fieldType>s
- indexed: true if this field should be indexed (searchable or sortable)
- stored: true if this field should be retrievable
- multiValued: true if this field may contain multiple values per document
- omitNorms: (expert) set to true to omit the norms associated with
- this field (this disables length normalization and index-time
- boosting for the field, and saves some memory). Only full-text
- fields or fields that need an index-time boost need norms.
- termVectors: [false] set to true to store the term vector for a given field.
- When using MoreLikeThis, fields used for similarity should be stored for
- best performance.
- -->
-
- <field name="solr_id" type="string" indexed="true" stored="true" required="true"/>
- <field name="desc" type="string" indexed="true" stored="true" multiValued="true"/>
-
- <field name="date" type="date" indexed="true" stored="true"/>
-
- <field name="timestamp" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
-
-
- <!-- Dynamic field definitions. If a field name is not found, dynamicFields
- will be used if the name matches any of the patterns.
- RESTRICTION: the glob-like pattern in the name attribute must have
- a "*" only at the start or the end.
- EXAMPLE: name="*_i" will match any field ending in _i (like myid_i, z_i)
- Longer patterns will be matched first. if equal size patterns
- both match, the first appearing in the schema will be used. -->
- <dynamicField name="*_i" type="int" indexed="true" stored="true"/>
- <dynamicField name="*_s" type="string" indexed="true" stored="true"/>
- <dynamicField name="*_l" type="long" indexed="true" stored="true"/>
- <dynamicField name="*_t" type="text" indexed="true" stored="true"/>
- <dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
- <dynamicField name="*_f" type="float" indexed="true" stored="true"/>
- <dynamicField name="*_d" type="double" indexed="true" stored="true"/>
- <dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
-
- <dynamicField name="random*" type="random"/>
-
- <!-- uncomment the following to ignore any fields that don't already match an existing
- field name or dynamic field, rather than reporting them as an error.
- alternately, change the type="ignored" to some other type e.g. "text" if you want
- unknown fields indexed and/or stored by default -->
- <!--dynamicField name="*" type="ignored" /-->
-
-
- <!-- Field to use to determine and enforce document uniqueness.
- Unless this field is marked with required="false", it will be a required field
- -->
- <uniqueKey>solr_id</uniqueKey>
-</schema>
[03/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestRegexTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestRegexTransformer.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestRegexTransformer.java
deleted file mode 100644
index e792804..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestRegexTransformer.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.RegexTransformer.REGEX;
-import static org.apache.solr.handler.dataimport.RegexTransformer.GROUP_NAMES;
-import static org.apache.solr.handler.dataimport.RegexTransformer.REPLACE_WITH;
-import static org.apache.solr.handler.dataimport.DataImporter.COLUMN;
-
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * <p> Test for RegexTransformer </p>
- *
- *
- * @since solr 1.3
- */
-public class TestRegexTransformer extends AbstractDataImportHandlerTestCase {
-
- @Test
- public void testCommaSeparated() {
- List<Map<String, String>> fields = new ArrayList<>();
- // <field column="col1" sourceColName="a" splitBy="," />
- fields.add(getField("col1", "string", null, "a", ","));
- Context context = getContext(null, null, null, Context.FULL_DUMP, fields, null);
-
- Map<String, Object> src = new HashMap<>();
- src.put("a", "a,bb,cc,d");
-
- Map<String, Object> result = new RegexTransformer().transformRow(src, context);
- assertEquals(2, result.size());
- assertEquals(4, ((List) result.get("col1")).size());
- }
-
-
- @Test
- public void testGroupNames() {
- List<Map<String, String>> fields = new ArrayList<>();
- // <field column="col1" regex="(\w*)(\w*) (\w*)" groupNames=",firstName,lastName"/>
- Map<String ,String > m = new HashMap<>();
- m.put(COLUMN,"fullName");
- m.put(GROUP_NAMES,",firstName,lastName");
- m.put(REGEX,"(\\w*) (\\w*) (\\w*)");
- fields.add(m);
- Context context = getContext(null, null, null, Context.FULL_DUMP, fields, null);
- Map<String, Object> src = new HashMap<>();
- src.put("fullName", "Mr Noble Paul");
-
- Map<String, Object> result = new RegexTransformer().transformRow(src, context);
- assertEquals("Noble", result.get("firstName"));
- assertEquals("Paul", result.get("lastName"));
- src= new HashMap<>();
- List<String> l= new ArrayList();
- l.add("Mr Noble Paul") ;
- l.add("Mr Shalin Mangar") ;
- src.put("fullName", l);
- result = new RegexTransformer().transformRow(src, context);
- List l1 = (List) result.get("firstName");
- List l2 = (List) result.get("lastName");
- assertEquals("Noble", l1.get(0));
- assertEquals("Shalin", l1.get(1));
- assertEquals("Paul", l2.get(0));
- assertEquals("Mangar", l2.get(1));
- }
-
- @Test
- public void testReplaceWith() {
- List<Map<String, String>> fields = new ArrayList<>();
- // <field column="name" regexp="'" replaceWith="''" />
- Map<String, String> fld = getField("name", "string", "'", null, null);
- fld.put(REPLACE_WITH, "''");
- fields.add(fld);
- Context context = getContext(null, null, null, Context.FULL_DUMP, fields, null);
-
- Map<String, Object> src = new HashMap<>();
- String s = "D'souza";
- src.put("name", s);
-
- Map<String, Object> result = new RegexTransformer().transformRow(src,
- context);
- assertEquals("D''souza", result.get("name"));
-
- fld = getField("title_underscore", "string", "\\s+", "title", null);
- fld.put(REPLACE_WITH, "_");
- fields.clear();
- fields.add(fld);
- context = getContext(null, null, null, Context.FULL_DUMP, fields, null);
- src.clear();
- src.put("title", "value with spaces"); // a value which will match the regex
- result = new RegexTransformer().transformRow(src, context);
- assertEquals("value_with_spaces", result.get("title_underscore"));
- src.clear();
- src.put("title", "valueWithoutSpaces"); // value which will not match regex
- result = new RegexTransformer().transformRow(src, context);
- assertEquals("valueWithoutSpaces", result.get("title_underscore")); // value should be returned as-is
- }
-
- @Test
- public void testMileage() {
- // init a whole pile of fields
- List<Map<String, String>> fields = getFields();
-
- // add another regex which reuses result from previous regex again!
- // <field column="hltCityMPG" sourceColName="rowdata" regexp="(${e.city_mileage})" />
- Map<String, String> fld = getField("hltCityMPG", "string",
- ".*(${e.city_mileage})", "rowdata", null);
- fld.put(REPLACE_WITH, "*** $1 ***");
- fields.add(fld);
-
- // **ATTEMPTS** a match WITHOUT a replaceWith
- // <field column="t1" sourceColName="rowdata" regexp="duff" />
- fld = getField("t1", "string","duff", "rowdata", null);
- fields.add(fld);
-
- // **ATTEMPTS** a match WITH a replaceWith (should return original data)
- // <field column="t2" sourceColName="rowdata" regexp="duff" replaceWith="60"/>
- fld = getField("t2", "string","duff", "rowdata", null);
- fld.put(REPLACE_WITH, "60");
- fields.add(fld);
-
- // regex WITH both replaceWith and groupName (groupName ignored!)
- // <field column="t3" sourceColName="rowdata" regexp="(Range)" />
- fld = getField("t3", "string","(Range)", "rowdata", null);
- fld.put(REPLACE_WITH, "range");
- fld.put(GROUP_NAMES,"t4,t5");
- fields.add(fld);
-
- Map<String, Object> row = new HashMap<>();
- String s = "Fuel Economy Range: 26 mpg Hwy, 19 mpg City";
- row.put("rowdata", s);
-
- VariableResolver resolver = new VariableResolver();
- resolver.addNamespace("e", row);
- Map<String, String> eAttrs = createMap("name", "e");
- Context context = getContext(null, resolver, null, Context.FULL_DUMP, fields, eAttrs);
-
- Map<String, Object> result = new RegexTransformer().transformRow(row, context);
- assertEquals(6, result.size());
- assertEquals(s, result.get("t2"));
- assertEquals(s, result.get("rowdata"));
- assertEquals("26", result.get("highway_mileage"));
- assertEquals("19", result.get("city_mileage"));
- assertEquals("*** 19 *** mpg City", result.get("hltCityMPG"));
- assertEquals("Fuel Economy range: 26 mpg Hwy, 19 mpg City", result.get("t3"));
- }
-
- @Test
- public void testMultiValuedRegex(){
- List<Map<String, String>> fields = new ArrayList<>();
-// <field column="participant" sourceColName="person" regex="(.*)" />
- Map<String, String> fld = getField("participant", null, "(.*)", "person", null);
- fields.add(fld);
- Context context = getContext(null, null,
- null, Context.FULL_DUMP, fields, null);
-
- ArrayList<String> strings = new ArrayList<>();
- strings.add("hello");
- strings.add("world");
- Map<String, Object> result = new RegexTransformer().transformRow(createMap("person", strings), context);
- assertEquals(strings,result.get("participant"));
- }
-
- public static List<Map<String, String>> getFields() {
- List<Map<String, String>> fields = new ArrayList<>();
-
- // <field column="city_mileage" sourceColName="rowdata" regexp=
- // "Fuel Economy Range:\\s*?\\d*?\\s*?mpg Hwy,\\s*?(\\d*?)\\s*?mpg City"
- fields.add(getField("city_mileage", "sint",
- "Fuel Economy Range:\\s*?\\d*?\\s*?mpg Hwy,\\s*?(\\d*?)\\s*?mpg City",
- "rowdata", null));
-
- // <field column="highway_mileage" sourceColName="rowdata" regexp=
- // "Fuel Economy Range:\\s*?(\\d*?)\\s*?mpg Hwy,\\s*?\\d*?\\s*?mpg City"
- fields.add(getField("highway_mileage", "sint",
- "Fuel Economy Range:\\s*?(\\d*?)\\s*?mpg Hwy,\\s*?\\d*?\\s*?mpg City",
- "rowdata", null));
-
- // <field column="seating_capacity" sourceColName="rowdata" regexp="Seating capacity:(.*)"
- fields.add(getField("seating_capacity", "sint", "Seating capacity:(.*)",
- "rowdata", null));
-
- // <field column="warranty" sourceColName="rowdata" regexp="Warranty:(.*)" />
- fields.add(getField("warranty", "string", "Warranty:(.*)", "rowdata", null));
-
- // <field column="rowdata" sourceColName="rowdata" />
- fields.add(getField("rowdata", "string", null, "rowdata", null));
- return fields;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java
deleted file mode 100644
index c106f8e..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.handler.dataimport.config.DIHConfiguration;
-import org.junit.Test;
-import org.w3c.dom.Document;
-import org.xml.sax.InputSource;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Test for ScriptTransformer
- *
- *
- * @since solr 1.3
- */
-public class TestScriptTransformer extends AbstractDataImportHandlerTestCase {
-
- @Test
- public void testBasic() {
- try {
- String script = "function f1(row,context){"
- + "row.put('name','Hello ' + row.get('name'));" + "return row;\n" + "}";
- Context context = getContext("f1", script);
- Map<String, Object> map = new HashMap<>();
- map.put("name", "Scott");
- EntityProcessorWrapper sep = new EntityProcessorWrapper(new SqlEntityProcessor(), null, null);
- sep.init(context);
- sep.applyTransformer(map);
- assertEquals("Hello Scott", map.get("name").toString());
- } catch (DataImportHandlerException e) {
- assumeFalse("This JVM does not have JavaScript installed. Test Skipped.", e
- .getMessage().startsWith("Cannot load Script Engine for language"));
- throw e;
- }
- }
-
- private Context getContext(String funcName, String script) {
- List<Map<String, String>> fields = new ArrayList<>();
- Map<String, String> entity = new HashMap<>();
- entity.put("name", "hello");
- entity.put("transformer", "script:" + funcName);
-
- TestContext context = getContext(null, null, null,
- Context.FULL_DUMP, fields, entity);
- context.script = script;
- context.scriptlang = "JavaScript";
- return context;
- }
-
- @Test
- public void testOneparam() {
- try {
- String script = "function f1(row){"
- + "row.put('name','Hello ' + row.get('name'));" + "return row;\n" + "}";
-
- Context context = getContext("f1", script);
- Map<String, Object> map = new HashMap<>();
- map.put("name", "Scott");
- EntityProcessorWrapper sep = new EntityProcessorWrapper(new SqlEntityProcessor(), null, null);
- sep.init(context);
- sep.applyTransformer(map);
- assertEquals("Hello Scott", map.get("name").toString());
- } catch (DataImportHandlerException e) {
- assumeFalse("This JVM does not have JavaScript installed. Test Skipped.", e
- .getMessage().startsWith("Cannot load Script Engine for language"));
- throw e;
- }
- }
-
- @Test
- public void testReadScriptTag() throws Exception {
- try {
- DocumentBuilder builder = DocumentBuilderFactory.newInstance()
- .newDocumentBuilder();
- Document document = builder.parse(new InputSource(new StringReader(xml)));
- DataImporter di = new DataImporter();
- DIHConfiguration dc = di.readFromXml(document);
- assertTrue(dc.getScript().getText().indexOf("checkNextToken") > -1);
- } catch (DataImportHandlerException e) {
- assumeFalse("This JVM does not have JavaScript installed. Test Skipped.", e
- .getMessage().startsWith("Cannot load Script Engine for language"));
- throw e;
- }
- }
-
- @Test
- public void testCheckScript() throws Exception {
- try {
- DocumentBuilder builder = DocumentBuilderFactory.newInstance()
- .newDocumentBuilder();
- Document document = builder.parse(new InputSource(new StringReader(xml)));
- DataImporter di = new DataImporter();
- DIHConfiguration dc = di.readFromXml(document);
- Context c = getContext("checkNextToken", dc.getScript().getText());
-
- Map map = new HashMap();
- map.put("nextToken", "hello");
- EntityProcessorWrapper sep = new EntityProcessorWrapper(new SqlEntityProcessor(), null, null);
- sep.init(c);
- sep.applyTransformer(map);
- assertEquals("true", map.get("$hasMore"));
- map = new HashMap();
- map.put("nextToken", "");
- sep.applyTransformer(map);
- assertNull(map.get("$hasMore"));
- } catch (DataImportHandlerException e) {
- assumeFalse("This JVM does not have JavaScript installed. Test Skipped.", e
- .getMessage().startsWith("Cannot load Script Engine for language"));
- throw e;
- }
- }
-
- static String xml = "<dataConfig>\n"
- + "<script><![CDATA[\n"
- + "function checkNextToken(row)\t{\n"
- + " var nt = row.get('nextToken');"
- + " if (nt && nt !='' ){ "
- + " row.put('$hasMore', 'true');}\n"
- + " return row;\n"
- + "}]]></script>\t<document>\n"
- + "\t\t<entity name=\"mbx\" pk=\"articleNumber\" processor=\"XPathEntityProcessor\"\n"
- + "\t\t\turl=\"?boardId=${dataimporter.defaults.boardId}&maxRecords=20&includeBody=true&startDate=${dataimporter.defaults.startDate}&guid=:autosearch001&reqId=1&transactionId=stringfortracing&listPos=${mbx.nextToken}\"\n"
- + "\t\t\tforEach=\"/mbmessage/articles/navigation | /mbmessage/articles/article\" transformer=\"script:checkNextToken\">\n"
- + "\n" + "\t\t\t<field column=\"nextToken\"\n"
- + "\t\t\t\txpath=\"/mbmessage/articles/navigation/nextToken\" />\n"
- + "\n" + "\t\t</entity>\n" + "\t</document>\n" + "</dataConfig>";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSimplePropertiesWriter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSimplePropertiesWriter.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSimplePropertiesWriter.java
deleted file mode 100644
index 93a26c3..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSimplePropertiesWriter.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.lang.invoke.MethodHandles;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-
-import org.apache.solr.common.util.SuppressForbidden;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public class TestSimplePropertiesWriter extends AbstractDIHJdbcTestCase {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private boolean useJdbcEscapeSyntax;
- private String dateFormat;
- private String fileLocation;
- private String fileName;
-
- @Before
- public void spwBefore() throws Exception {
- fileLocation = createTempDir().toFile().getAbsolutePath();
- fileName = "the.properties";
- }
-
- @SuppressForbidden(reason = "Needs currentTimeMillis to construct date stamps")
- @Test
- public void testSimplePropertiesWriter() throws Exception {
-
- SimpleDateFormat errMsgFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSS", Locale.ROOT);
-
- String[] d = {
- "{'ts' ''yyyy-MM-dd HH:mm:ss.SSSSSS''}",
- "{'ts' ''yyyy-MM-dd HH:mm:ss''}",
- "yyyy-MM-dd HH:mm:ss",
- "yyyy-MM-dd HH:mm:ss.SSSSSS"
- };
- for(int i=0 ; i<d.length ; i++) {
- delQ("*:*");
- commit();
- if(i<2) {
- useJdbcEscapeSyntax = true;
- } else {
- useJdbcEscapeSyntax = false;
- }
- dateFormat = d[i];
- SimpleDateFormat df = new SimpleDateFormat(dateFormat, Locale.ROOT);
- Date oneSecondAgo = new Date(System.currentTimeMillis() - 1000);
-
- Map<String,String> init = new HashMap<>();
- init.put("dateFormat", dateFormat);
- init.put("filename", fileName);
- init.put("directory", fileLocation);
- SimplePropertiesWriter spw = new SimplePropertiesWriter();
- spw.init(new DataImporter(), init);
- Map<String, Object> props = new HashMap<>();
- props.put("SomeDates.last_index_time", oneSecondAgo);
- props.put("last_index_time", oneSecondAgo);
- spw.persist(props);
-
- h.query("/dataimport", generateRequest());
- props = spw.readIndexerProperties();
- Date entityDate = df.parse((String) props.get("SomeDates.last_index_time"));
- Date docDate= df.parse((String) props.get("last_index_time"));
- int year = currentYearFromDatabase();
-
- Assert.assertTrue("This date: " + errMsgFormat.format(oneSecondAgo) + " should be prior to the document date: " + errMsgFormat.format(docDate), docDate.getTime() - oneSecondAgo.getTime() > 0);
- Assert.assertTrue("This date: " + errMsgFormat.format(oneSecondAgo) + " should be prior to the entity date: " + errMsgFormat.format(entityDate), entityDate.getTime() - oneSecondAgo.getTime() > 0);
- assertQ(req("*:*"), "//*[@numFound='1']", "//doc/str[@name=\"ayear_s\"]=\"" + year + "\"");
- }
- }
-
- private int currentYearFromDatabase() throws Exception {
- Connection conn = null;
- Statement s = null;
- ResultSet rs = null;
- try {
- conn = newConnection();
- s = conn.createStatement();
- rs = s.executeQuery("select year(current_timestamp) from sysibm.sysdummy1");
- if (rs.next()) {
- return rs.getInt(1);
- }
- Assert.fail("We should have gotten a row from the db.");
- } catch (SQLException e) {
- throw e;
- } finally {
- try {
- rs.close();
- } catch (Exception ex) {}
- try {
- s.close();
- } catch (Exception ex) {}
- try {
- conn.close();
- } catch (Exception ex) {}
- }
- return 0;
- }
-
- @Override
- protected Database setAllowedDatabases() {
- return Database.DERBY;
- }
- @Override
- protected String generateConfig() {
- StringBuilder sb = new StringBuilder();
- String q = useJdbcEscapeSyntax ? "" : "'";
- sb.append("<dataConfig> \n");
- sb.append("<propertyWriter dateFormat=\"" + dateFormat + "\" type=\"SimplePropertiesWriter\" directory=\"" + fileLocation + "\" filename=\"" + fileName + "\" />\n");
- sb.append("<dataSource name=\"derby\" driver=\"org.apache.derby.jdbc.EmbeddedDriver\" url=\"jdbc:derby:memory:derbyDB;territory=en_US\" /> \n");
- sb.append("<document name=\"TestSimplePropertiesWriter\"> \n");
- sb.append("<entity name=\"SomeDates\" processor=\"SqlEntityProcessor\" dataSource=\"derby\" ");
- sb.append("query=\"select 1 as id, YEAR(" + q + "${dih.last_index_time}" + q + ") as AYEAR_S from sysibm.sysdummy1 \" >\n");
- sb.append("<field column=\"AYEAR_S\" name=\"ayear_s\" /> \n");
- sb.append("</entity>\n");
- sb.append("</document> \n");
- sb.append("</dataConfig> \n");
- String config = sb.toString();
- log.debug(config);
- return config;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java
deleted file mode 100644
index 0e9cd33..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java
+++ /dev/null
@@ -1,373 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.lucene.util.IOUtils;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.common.SolrInputDocument;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Properties;
-
-/**
- * End-to-end test of SolrEntityProcessor. "Real" test using embedded Solr
- */
-public class TestSolrEntityProcessorEndToEnd extends AbstractDataImportHandlerTestCase {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final String SOLR_CONFIG = "dataimport-solrconfig.xml";
- private static final String SOLR_SCHEMA = "dataimport-schema.xml";
- private static final String SOURCE_CONF_DIR = "dih" + File.separator + "solr" + File.separator + "collection1" + File.separator + "conf" + File.separator;
- private static final String ROOT_DIR = "dih" + File.separator + "solr" + File.separator;
-
- private static final String DEAD_SOLR_SERVER = "http://[ff01::114]:33332/solr";
-
- private static final List<Map<String,Object>> DB_DOCS = new ArrayList<>();
- private static final List<Map<String,Object>> SOLR_DOCS = new ArrayList<>();
-
- static {
- // dynamic fields in the destination schema
- Map<String,Object> dbDoc = new HashMap<>();
- dbDoc.put("dbid_s", "1");
- dbDoc.put("dbdesc_s", "DbDescription");
- DB_DOCS.add(dbDoc);
-
- Map<String,Object> solrDoc = new HashMap<>();
- solrDoc.put("id", "1");
- solrDoc.put("desc", "SolrDescription");
- SOLR_DOCS.add(solrDoc);
- }
-
-
- private SolrInstance instance = null;
- private JettySolrRunner jetty;
-
- private String getDihConfigTagsInnerEntity() {
- return "<dataConfig>\r\n"
- + " <dataSource type='MockDataSource' />\r\n"
- + " <document>\r\n"
- + " <entity name='db' query='select * from x'>\r\n"
- + " <field column='dbid_s' />\r\n"
- + " <field column='dbdesc_s' />\r\n"
- + " <entity name='se' processor='SolrEntityProcessor' query='id:${db.dbid_s}'\n"
- + " url='" + getSourceUrl() + "' fields='id,desc'>\r\n"
- + " <field column='id' />\r\n"
- + " <field column='desc' />\r\n" + " </entity>\r\n"
- + " </entity>\r\n" + " </document>\r\n" + "</dataConfig>\r\n";
- }
-
- private String generateDIHConfig(String options, boolean useDeadServer) {
- return "<dataConfig>\r\n" + " <document>\r\n"
- + " <entity name='se' processor='SolrEntityProcessor'" + " url='"
- + (useDeadServer ? DEAD_SOLR_SERVER : getSourceUrl()) + "' " + options + " />\r\n" + " </document>\r\n"
- + "</dataConfig>\r\n";
- }
-
- private String getSourceUrl() {
- return buildUrl(jetty.getLocalPort(), "/solr/collection1");
- }
-
- //TODO: fix this test to close its directories
- static String savedFactory;
- @BeforeClass
- public static void beforeClass() {
- savedFactory = System.getProperty("solr.DirectoryFactory");
- System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
- }
-
- @AfterClass
- public static void afterClass() {
- if (savedFactory == null) {
- System.clearProperty("solr.directoryFactory");
- } else {
- System.setProperty("solr.directoryFactory", savedFactory);
- }
- }
-
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- // destination solr core
- initCore(SOLR_CONFIG, SOLR_SCHEMA);
- // data source solr instance
- instance = new SolrInstance();
- instance.setUp();
- jetty = createJetty(instance);
- }
-
- @Override
- @After
- public void tearDown() throws Exception {
- try {
- deleteCore();
- } catch (Exception e) {
- log.error("Error deleting core", e);
- }
- jetty.stop();
- instance.tearDown();
- super.tearDown();
- }
-
- //commented 23-AUG-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018
- public void testFullImport() {
- assertQ(req("*:*"), "//result[@numFound='0']");
-
- try {
- addDocumentsToSolr(SOLR_DOCS);
- runFullImport(generateDIHConfig("query='*:*' rows='2' fl='id,desc' onError='skip'", false));
- } catch (Exception e) {
- log.error(e.getMessage(), e);
- fail(e.getMessage());
- }
-
- assertQ(req("*:*"), "//result[@numFound='1']");
- assertQ(req("id:1"), "//result/doc/str[@name='id'][.='1']",
- "//result/doc/arr[@name='desc'][.='SolrDescription']");
- }
-
- public void testFullImportFqParam() {
- assertQ(req("*:*"), "//result[@numFound='0']");
-
- try {
- addDocumentsToSolr(generateSolrDocuments(30));
- Map<String,String> map = new HashMap<>();
- map.put("rows", "50");
- runFullImport(generateDIHConfig("query='*:*' fq='desc:Description1*,desc:Description*2' rows='2'", false), map);
- } catch (Exception e) {
- log.error(e.getMessage(), e);
- fail(e.getMessage());
- }
-
- assertQ(req("*:*"), "//result[@numFound='1']");
- assertQ(req("id:12"), "//result[@numFound='1']", "//result/doc/arr[@name='desc'][.='Description12']");
- }
-
- public void testFullImportFieldsParam() {
- assertQ(req("*:*"), "//result[@numFound='0']");
-
- try {
- addDocumentsToSolr(generateSolrDocuments(7));
- runFullImport(generateDIHConfig("query='*:*' fl='id' rows='2'"+(random().nextBoolean() ?" cursorMark='true' sort='id asc'":""), false));
- } catch (Exception e) {
- log.error(e.getMessage(), e);
- fail(e.getMessage());
- }
-
- assertQ(req("*:*"), "//result[@numFound='7']");
- assertQ(req("id:1"), "//result[@numFound='1']");
- try {
- assertQ(req("id:1"), "//result/doc/arr[@name='desc']");
- fail("The document has a field with name desc");
- } catch(Exception e) {
-
- }
-
- }
-
- /**
- * Receive a row from SQL (Mock) and fetch a row from Solr
- */
- public void testFullImportInnerEntity() {
- assertQ(req("*:*"), "//result[@numFound='0']");
-
- try {
- List<Map<String,Object>> DOCS = new ArrayList<>(DB_DOCS);
- Map<String, Object> doc = new HashMap<>();
- doc.put("dbid_s", "2");
- doc.put("dbdesc_s", "DbDescription2");
- DOCS.add(doc);
- MockDataSource.setIterator("select * from x", DOCS.iterator());
-
- DOCS = new ArrayList<>(SOLR_DOCS);
- Map<String,Object> solrDoc = new HashMap<>();
- solrDoc.put("id", "2");
- solrDoc.put("desc", "SolrDescription2");
- DOCS.add(solrDoc);
- addDocumentsToSolr(DOCS);
- runFullImport(getDihConfigTagsInnerEntity());
- } catch (Exception e) {
- log.error(e.getMessage(), e);
- fail(e.getMessage());
- } finally {
- MockDataSource.clearCache();
- }
-
- assertQ(req("*:*"), "//result[@numFound='2']");
- assertQ(req("id:1"), "//result/doc/str[@name='id'][.='1']",
- "//result/doc/str[@name='dbdesc_s'][.='DbDescription']",
- "//result/doc/str[@name='dbid_s'][.='1']",
- "//result/doc/arr[@name='desc'][.='SolrDescription']");
- assertQ(req("id:2"), "//result/doc/str[@name='id'][.='2']",
- "//result/doc/str[@name='dbdesc_s'][.='DbDescription2']",
- "//result/doc/str[@name='dbid_s'][.='2']",
- "//result/doc/arr[@name='desc'][.='SolrDescription2']");
- }
-
- public void testFullImportWrongSolrUrl() {
- assertQ(req("*:*"), "//result[@numFound='0']");
-
- try {
- runFullImport(generateDIHConfig("query='*:*' rows='2' fl='id,desc' onError='skip'", true /* use dead server */));
- } catch (Exception e) {
- log.error(e.getMessage(), e);
- fail(e.getMessage());
- }
-
- assertQ(req("*:*"), "//result[@numFound='0']");
- }
-
- public void testFullImportBadConfig() {
- assertQ(req("*:*"), "//result[@numFound='0']");
-
- try {
- runFullImport(generateDIHConfig("query='bogus:3' rows='2' fl='id,desc' onError='"+
- (random().nextBoolean() ? "abort" : "justtogetcoverage")+"'", false));
- } catch (Exception e) {
- log.error(e.getMessage(), e);
- fail(e.getMessage());
- }
-
- assertQ(req("*:*"), "//result[@numFound='0']");
- }
-
- public void testCursorMarkNoSort() throws SolrServerException, IOException {
- assertQ(req("*:*"), "//result[@numFound='0']");
- addDocumentsToSolr(generateSolrDocuments(7));
- try {
- List<String> errors = Arrays.asList("sort='id'", //wrong sort spec
- "", //no sort spec
- "sort='id asc' timeout='12345'"); // sort is fine, but set timeout
- Collections.shuffle(errors, random());
- String attrs = "query='*:*' rows='2' fl='id,desc' cursorMark='true' "
- + errors.get(0);
- runFullImport(generateDIHConfig(attrs,
- false));
- } catch (Exception e) {
- log.error(e.getMessage(), e);
- fail(e.getMessage());
- }
-
- assertQ(req("*:*"), "//result[@numFound='0']");
- }
-
- private static List<Map<String,Object>> generateSolrDocuments(int num) {
- List<Map<String,Object>> docList = new ArrayList<>();
- for (int i = 1; i <= num; i++) {
- Map<String,Object> map = new HashMap<>();
- map.put("id", i);
- map.put("desc", "Description" + i);
- docList.add(map);
- }
- return docList;
- }
-
- private void addDocumentsToSolr(List<Map<String,Object>> docs) throws SolrServerException, IOException {
- List<SolrInputDocument> sidl = new ArrayList<>();
- for (Map<String,Object> doc : docs) {
- SolrInputDocument sd = new SolrInputDocument();
- for (Entry<String,Object> entry : doc.entrySet()) {
- sd.addField(entry.getKey(), entry.getValue());
- }
- sidl.add(sd);
- }
-
- try (HttpSolrClient solrServer = getHttpSolrClient(getSourceUrl(), 15000, 30000)) {
- solrServer.add(sidl);
- solrServer.commit(true, true);
- }
- }
-
- private static class SolrInstance {
- File homeDir;
- File confDir;
-
- public String getHomeDir() {
- return homeDir.toString();
- }
-
- public String getSchemaFile() {
- return SOURCE_CONF_DIR + "dataimport-schema.xml";
- }
-
- public String getDataDir() {
- return initCoreDataDir.toString();
- }
-
- public String getSolrConfigFile() {
- return SOURCE_CONF_DIR + "dataimport-solrconfig.xml";
- }
-
- public String getSolrXmlFile() {
- return ROOT_DIR + "solr.xml";
- }
-
- public void setUp() throws Exception {
- homeDir = createTempDir().toFile();
- initCoreDataDir = new File(homeDir + "/collection1", "data");
- confDir = new File(homeDir + "/collection1", "conf");
-
- homeDir.mkdirs();
- initCoreDataDir.mkdirs();
- confDir.mkdirs();
-
- FileUtils.copyFile(getFile(getSolrXmlFile()), new File(homeDir, "solr.xml"));
- File f = new File(confDir, "solrconfig.xml");
- FileUtils.copyFile(getFile(getSolrConfigFile()), f);
- f = new File(confDir, "schema.xml");
-
- FileUtils.copyFile(getFile(getSchemaFile()), f);
- f = new File(confDir, "data-config.xml");
- FileUtils.copyFile(getFile(SOURCE_CONF_DIR + "dataconfig-contentstream.xml"), f);
-
- Files.createFile(confDir.toPath().resolve("../core.properties"));
- }
-
- public void tearDown() throws Exception {
- IOUtils.rm(homeDir.toPath());
- }
- }
-
- private JettySolrRunner createJetty(SolrInstance instance) throws Exception {
- Properties nodeProperties = new Properties();
- nodeProperties.setProperty("solr.data.dir", instance.getDataDir());
- JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), nodeProperties, buildJettyConfig("/solr"));
- jetty.start();
- return jetty;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java
deleted file mode 100644
index a2a9fff..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.*;
-
-import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.CursorMarkParams;
-import org.apache.solr.handler.dataimport.SolrEntityProcessor.SolrDocumentListIterator;
-import org.junit.Test;
-
-/**
- * Unit test of SolrEntityProcessor. A very basic test outside of the DIH.
- */
-public class TestSolrEntityProcessorUnit extends AbstractDataImportHandlerTestCase {
-
- private static final class NoNextMockProcessor extends SolrEntityProcessor {
- @Override
- protected void nextPage() {
- }
- }
-
- private static final String ID = "id";
-
- public void testQuery() {
- List<Doc> docs = generateUniqueDocs(2);
-
- MockSolrEntityProcessor processor = createAndInit(docs);
- try {
- assertExpectedDocs(docs, processor);
- assertEquals(1, processor.getQueryCount());
- } finally {
- processor.destroy();
- }
- }
-
- private MockSolrEntityProcessor createAndInit(List<Doc> docs) {
- return createAndInit(docs, SolrEntityProcessor.ROWS_DEFAULT);
- }
-
- public void testNumDocsGreaterThanRows() {
- List<Doc> docs = generateUniqueDocs(44);
-
- int rowsNum = 10;
- MockSolrEntityProcessor processor = createAndInit(docs, rowsNum);
- try {
- assertExpectedDocs(docs, processor);
- assertEquals(5, processor.getQueryCount());
- } finally {
- processor.destroy();
- }
- }
-
- private MockSolrEntityProcessor createAndInit(List<Doc> docs, int rowsNum) {
- MockSolrEntityProcessor processor = new MockSolrEntityProcessor(docs, rowsNum);
- HashMap<String,String> entityAttrs = new HashMap<String,String>(){{put(SolrEntityProcessor.SOLR_SERVER,"http://route:66/no");}};
- processor.init(getContext(null, null, null, null, Collections.emptyList(),
- entityAttrs));
- return processor;
- }
-
- public void testMultiValuedFields() {
- List<Doc> docs = new ArrayList<>();
- List<FldType> types = new ArrayList<>();
- types.add(new FldType(ID, ONE_ONE, new SVal('A', 'Z', 4, 4)));
- types.add(new FldType("description", new IRange(3, 3), new SVal('a', 'c', 1, 1)));
- Doc testDoc = createDoc(types);
- docs.add(testDoc);
-
- MockSolrEntityProcessor processor = createAndInit(docs);
- try {
- Map<String, Object> next = processor.nextRow();
- assertNotNull(next);
-
- @SuppressWarnings("unchecked")
- List<Comparable> multiField = (List<Comparable>) next.get("description");
- assertEquals(testDoc.getValues("description").size(), multiField.size());
- assertEquals(testDoc.getValues("description"), multiField);
- assertEquals(1, processor.getQueryCount());
- assertNull(processor.nextRow());
- } finally {
- processor.destroy();
- }
- }
- @Test (expected = DataImportHandlerException.class)
- public void testNoQuery() {
- SolrEntityProcessor processor = new SolrEntityProcessor();
-
- HashMap<String,String> entityAttrs = new HashMap<String,String>(){{put(SolrEntityProcessor.SOLR_SERVER,"http://route:66/no");}};
- processor.init(getContext(null, null, null, null, Collections.emptyList(),
- entityAttrs));
- try {
- processor.buildIterator();
- }finally {
- processor.destroy();
- }
- }
-
- public void testPagingQuery() {
- SolrEntityProcessor processor = new NoNextMockProcessor() ;
-
- HashMap<String,String> entityAttrs = new HashMap<String,String>(){{
- put(SolrEntityProcessor.SOLR_SERVER,"http://route:66/no");
- if (random().nextBoolean()) {
- List<String> noCursor = Arrays.asList("","false",CursorMarkParams.CURSOR_MARK_START);//only 'true' not '*'
- Collections.shuffle(noCursor, random());
- put(CursorMarkParams.CURSOR_MARK_PARAM, noCursor.get(0));
- }}};
- processor.init(getContext(null, null, null, null, Collections.emptyList(),
- entityAttrs));
- try {
- processor.buildIterator();
- SolrQuery query = new SolrQuery();
- ((SolrDocumentListIterator) processor.rowIterator).passNextPage(query);
- assertEquals("0", query.get(CommonParams.START));
- assertNull( query.get(CursorMarkParams.CURSOR_MARK_PARAM));
- assertNotNull( query.get(CommonParams.TIME_ALLOWED));
- }finally {
- processor.destroy();
- }
- }
-
- public void testCursorQuery() {
- SolrEntityProcessor processor = new NoNextMockProcessor() ;
-
- HashMap<String,String> entityAttrs = new HashMap<String,String>(){{
- put(SolrEntityProcessor.SOLR_SERVER,"http://route:66/no");
- put(CursorMarkParams.CURSOR_MARK_PARAM,"true");
- }};
- processor.init(getContext(null, null, null, null, Collections.emptyList(),
- entityAttrs));
- try {
- processor.buildIterator();
- SolrQuery query = new SolrQuery();
- ((SolrDocumentListIterator) processor.rowIterator).passNextPage(query);
- assertNull(query.get(CommonParams.START));
- assertEquals(CursorMarkParams.CURSOR_MARK_START, query.get(CursorMarkParams.CURSOR_MARK_PARAM));
- assertNull( query.get(CommonParams.TIME_ALLOWED));
- }finally {
- processor.destroy();
- }
- }
-
- private List<Doc> generateUniqueDocs(int numDocs) {
- List<FldType> types = new ArrayList<>();
- types.add(new FldType(ID, ONE_ONE, new SVal('A', 'Z', 4, 40)));
- types.add(new FldType("description", new IRange(1, 3), new SVal('a', 'c', 1, 1)));
-
- Set<Comparable> previousIds = new HashSet<>();
- List<Doc> docs = new ArrayList<>(numDocs);
- for (int i = 0; i < numDocs; i++) {
- Doc doc = createDoc(types);
- while (previousIds.contains(doc.id)) {
- doc = createDoc(types);
- }
- previousIds.add(doc.id);
- docs.add(doc);
- }
- return docs;
- }
-
- private static void assertExpectedDocs(List<Doc> expectedDocs, SolrEntityProcessor processor) {
- for (Doc expectedDoc : expectedDocs) {
- Map<String, Object> next = processor.nextRow();
- assertNotNull(next);
- assertEquals(expectedDoc.id, next.get("id"));
- assertEquals(expectedDoc.getValues("description"), next.get("description"));
- }
- assertNull(processor.nextRow());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSortedMapBackedCache.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSortedMapBackedCache.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSortedMapBackedCache.java
deleted file mode 100644
index 7b0e613..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSortedMapBackedCache.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.lang.invoke.MethodHandles;
-
-import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestSortedMapBackedCache extends AbstractDIHCacheTestCase {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- @Test
- public void testCacheWithKeyLookup() {
- DIHCache cache = null;
- try {
- cache = new SortedMapBackedCache();
- cache.open(getContext(new HashMap<String,String>()));
- loadData(cache, data, fieldNames, true);
- List<ControlData> testData = extractDataByKeyLookup(cache, fieldNames);
- compareData(data, testData);
- } catch (Exception e) {
- log.warn("Exception thrown: " + e.toString());
- Assert.fail();
- } finally {
- try {
- cache.destroy();
- } catch (Exception ex) {
- }
- }
- }
-
- @Test
- public void testCacheWithOrderedLookup() {
- DIHCache cache = null;
- try {
- cache = new SortedMapBackedCache();
- cache.open(getContext(new HashMap<String,String>()));
- loadData(cache, data, fieldNames, true);
- List<ControlData> testData = extractDataInKeyOrder(cache, fieldNames);
- compareData(data, testData);
- } catch (Exception e) {
- log.warn("Exception thrown: " + e.toString());
- Assert.fail();
- } finally {
- try {
- cache.destroy();
- } catch (Exception ex) {
- }
- }
- }
-
- @Test
- public void testNullKeys() throws Exception {
- //A null key should just be ignored, but not throw an exception
- DIHCache cache = null;
- try {
- cache = new SortedMapBackedCache();
- Map<String, String> cacheProps = new HashMap<>();
- cacheProps.put(DIHCacheSupport.CACHE_PRIMARY_KEY, "a_id");
- cache.open(getContext(cacheProps));
-
- Map<String,Object> data = new HashMap<>();
- data.put("a_id", null);
- data.put("bogus", "data");
- cache.add(data);
-
- Iterator<Map<String, Object>> cacheIter = cache.iterator();
- while (cacheIter.hasNext()) {
- Assert.fail("cache should be empty.");
- }
- Assert.assertNull(cache.iterator(null));
- cache.delete(null);
- } catch (Exception e) {
- throw e;
- } finally {
- try {
- cache.destroy();
- } catch (Exception ex) {
- }
- }
- }
-
- @Test
- public void testCacheReopensWithUpdate() {
- DIHCache cache = null;
- try {
- Map<String, String> cacheProps = new HashMap<>();
- cacheProps.put(DIHCacheSupport.CACHE_PRIMARY_KEY, "a_id");
-
- cache = new SortedMapBackedCache();
- cache.open(getContext(cacheProps));
- // We can let the data hit the cache with the fields out of order because
- // we've identified the pk up-front.
- loadData(cache, data, fieldNames, false);
-
- // Close the cache.
- cache.close();
-
- List<ControlData> newControlData = new ArrayList<>();
- Object[] newIdEqualsThree = null;
- int j = 0;
- for (int i = 0; i < data.size(); i++) {
- // We'll be deleting a_id=1 so remove it from the control data.
- if (data.get(i).data[0].equals(1)) {
- continue;
- }
-
- // We'll be changing "Cookie" to "Carrot" in a_id=3 so change it in the control data.
- if (data.get(i).data[0].equals(3)) {
- newIdEqualsThree = new Object[data.get(i).data.length];
- System.arraycopy(data.get(i).data, 0, newIdEqualsThree, 0, newIdEqualsThree.length);
- newIdEqualsThree[3] = "Carrot";
- newControlData.add(new ControlData(newIdEqualsThree));
- }
- // Everything else can just be copied over.
- else {
- newControlData.add(data.get(i));
- }
-
- j++;
- }
-
- // These new rows of data will get added to the cache, so add them to the control data too.
- Object[] newDataRow1 = new Object[] {99, new BigDecimal(Math.PI), "Z", "Zebra", 99.99f, Feb21_2011, null };
- Object[] newDataRow2 = new Object[] {2, new BigDecimal(Math.PI), "B", "Ballerina", 2.22f, Feb21_2011, null };
-
- newControlData.add(new ControlData(newDataRow1));
- newControlData.add(new ControlData(newDataRow2));
-
- // Re-open the cache
- cache.open(getContext(new HashMap<String,String>()));
-
- // Delete a_id=1 from the cache.
- cache.delete(1);
-
- // Because the cache allows duplicates, the only way to update is to
- // delete first then add.
- cache.delete(3);
- cache.add(controlDataToMap(new ControlData(newIdEqualsThree), fieldNames, false));
-
- // Add this row with a new Primary key.
- cache.add(controlDataToMap(new ControlData(newDataRow1), fieldNames, false));
-
- // Add this row, creating two records in the cache with a_id=2.
- cache.add(controlDataToMap(new ControlData(newDataRow2), fieldNames, false));
-
- // Read the cache back and compare to the newControlData
- List<ControlData> testData = extractDataInKeyOrder(cache, fieldNames);
- compareData(newControlData, testData);
-
- // Now try reading the cache read-only.
- cache.close();
- cache.open(getContext(new HashMap<String,String>()));
- testData = extractDataInKeyOrder(cache, fieldNames);
- compareData(newControlData, testData);
-
- } catch (Exception e) {
- log.warn("Exception thrown: " + e.toString());
- Assert.fail();
- } finally {
- try {
- cache.destroy();
- } catch (Exception ex) {
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java
deleted file mode 100644
index f1277c9..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Test with various combinations of parameters, child entities, caches, transformers.
- */
-public class TestSqlEntityProcessor extends AbstractSqlEntityProcessorTestCase {
-
- @Test
- public void testSingleEntity() throws Exception {
- singleEntity(1);
- }
- @Test
- public void testWithSimpleTransformer() throws Exception {
- simpleTransform(1);
- }
- @Test
- public void testWithComplexTransformer() throws Exception {
- complexTransform(1, 0);
- }
- @Test
- public void testChildEntities() throws Exception {
- withChildEntities(false, true);
- }
- @Test
- public void testCachedChildEntities() throws Exception {
- withChildEntities(true, true);
- }
-
- @Test
- public void testSportZipperChildEntities() throws Exception {
- sportsZipper = true;
- withChildEntities(true, true);
- }
-
- @Test
- public void testCountryZipperChildEntities() throws Exception {
- countryZipper = true;
- withChildEntities(true, true);
- }
-
- @Test
- public void testBothZipperChildEntities() throws Exception {
- countryZipper = true;
- sportsZipper = true;
- withChildEntities(true, true);
- }
-
- @Test(expected=RuntimeException.class /* DIH exceptions are not propagated, here we capturing assertQ exceptions */)
- public void testSportZipperChildEntitiesWrongOrder() throws Exception {
- if(random().nextBoolean()){
- wrongPeopleOrder = true;
- }else{
- wrongSportsOrder = true;
- }
- testSportZipperChildEntities();
- }
-
- @Test(expected=RuntimeException.class )
- public void testCountryZipperChildEntitiesWrongOrder() throws Exception {
- if(random().nextBoolean()){
- wrongPeopleOrder = true;
- }else{
- wrongCountryOrder = true;
- }
- testCountryZipperChildEntities();
- }
-
- @Test(expected=RuntimeException.class)
- public void testBothZipperChildEntitiesWrongOrder() throws Exception {
- if(random().nextBoolean()){
- wrongPeopleOrder = true;
- }else{
- if(random().nextBoolean()){
- wrongSportsOrder = true;
- }else{
- wrongCountryOrder = true;
- }
- }
- testBothZipperChildEntities();
- }
-
- @Test
- @Ignore("broken see SOLR-3857")
- public void testSimpleCacheChildEntities() throws Exception {
- simpleCacheChildEntities(true);
- }
-
- @Override
- protected String deltaQueriesCountryTable() {
- return "";
- }
- @Override
- protected String deltaQueriesPersonTable() {
- return "";
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
deleted file mode 100644
index 1f22a8c..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.lang.invoke.MethodHandles;
-
-import org.apache.solr.request.LocalSolrQueryRequest;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Test with various combinations of parameters, child entities, transformers.
- */
-public class TestSqlEntityProcessorDelta extends AbstractSqlEntityProcessorTestCase {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private boolean delta = false;
- private boolean useParentDeltaQueryParam = false;
- private IntChanges personChanges = null;
- private String[] countryChanges = null;
-
- @Before
- public void setupDeltaTest() {
- delta = false;
- personChanges = null;
- countryChanges = null;
- }
- @Test
- public void testSingleEntity() throws Exception {
- log.debug("testSingleEntity full-import...");
- singleEntity(1);
- logPropertiesFile();
- changeStuff();
- int c = calculateDatabaseCalls();
- log.debug("testSingleEntity delta-import (" + c + " database calls expected)...");
- singleEntity(c);
- validateChanges();
- }
-
- @Test
- public void testDeltaImportWithoutInitialFullImport() throws Exception {
- log.debug("testDeltaImportWithoutInitialFullImport delta-import...");
- countryEntity = false;
- delta = true;
- /*
- * We need to add 2 in total:
- * +1 for deltaQuery i.e identifying id of items to update,
- * +1 for deletedPkQuery i.e delete query
- */
- singleEntity(totalPeople() + 2);
- validateChanges();
- }
-
- @Test
- public void testWithSimpleTransformer() throws Exception {
- log.debug("testWithSimpleTransformer full-import...");
- simpleTransform(1);
- logPropertiesFile();
- changeStuff();
- int c = calculateDatabaseCalls();
- simpleTransform(c);
- log.debug("testWithSimpleTransformer delta-import (" + c + " database calls expected)...");
- validateChanges();
- }
- @Test
- public void testWithComplexTransformer() throws Exception {
- log.debug("testWithComplexTransformer full-import...");
- complexTransform(1, 0);
- logPropertiesFile();
- changeStuff();
- int c = calculateDatabaseCalls();
- log.debug("testWithComplexTransformer delta-import (" + c + " database calls expected)...");
- complexTransform(c, personChanges.deletedKeys.length);
- validateChanges();
- }
- @Test
- public void testChildEntities() throws Exception {
- log.debug("testChildEntities full-import...");
- useParentDeltaQueryParam = random().nextBoolean();
- log.debug("using parent delta? " + useParentDeltaQueryParam);
- withChildEntities(false, true);
- logPropertiesFile();
- changeStuff();
- log.debug("testChildEntities delta-import...");
- withChildEntities(false, false);
- validateChanges();
- }
-
-
- private int calculateDatabaseCalls() {
- //The main query generates 1
- //Deletes generate 1
- //Each add/mod generate 1
- int c = 1;
- if (countryChanges != null) {
- c += countryChanges.length + 1;
- }
- if (personChanges != null) {
- c += personChanges.addedKeys.length + personChanges.changedKeys.length + 1;
- }
- return c;
- }
- private void validateChanges() throws Exception
- {
- if(personChanges!=null) {
- for(int id : personChanges.addedKeys) {
- assertQ(req("id:" + id), "//*[@numFound='1']");
- }
- for(int id : personChanges.deletedKeys) {
- assertQ(req("id:" + id), "//*[@numFound='0']");
- }
- for(int id : personChanges.changedKeys) {
- assertQ(req("id:" + id), "//*[@numFound='1']", "substring(//doc/arr[@name='NAME_mult_s']/str[1], 1, 8)='MODIFIED'");
- }
- }
- if(countryChanges!=null) {
- for(String code : countryChanges) {
- assertQ(req("COUNTRY_CODE_s:" + code), "//*[@numFound='" + numberPeopleByCountryCode(code) + "']", "substring((//doc/str[@name='COUNTRY_NAME_s'])[1], 1, 8)='MODIFIED'");
- }
- }
- }
- private void changeStuff() throws Exception {
- if(countryEntity)
- {
- int n = random().nextInt(2);
- switch(n) {
- case 0:
- personChanges = modifySomePeople();
- break;
- case 1:
- countryChanges = modifySomeCountries();
- break;
- case 2:
- personChanges = modifySomePeople();
- countryChanges = modifySomeCountries();
- break;
- }
- } else {
- personChanges = modifySomePeople();
- }
- countryChangesLog();
- personChangesLog();
- delta = true;
- }
- private void countryChangesLog()
- {
- if(countryChanges!=null) {
- StringBuilder sb = new StringBuilder();
- sb.append("country changes { ");
- for(String s : countryChanges) {
- sb.append(s).append(" ");
- }
- sb.append(" }");
- log.debug(sb.toString());
- }
- }
- private void personChangesLog()
- {
- if(personChanges!=null) {
- log.debug("person changes { " + personChanges.toString() + " } ");
- }
- }
- @Override
- protected LocalSolrQueryRequest generateRequest() {
- return lrf.makeRequest("command", (delta ? "delta-import" : "full-import"), "dataConfig", generateConfig(),
- "clean", (delta ? "false" : "true"), "commit", "true", "synchronous", "true", "indent", "true");
- }
- @Override
- protected String deltaQueriesPersonTable() {
- return
- "deletedPkQuery=''SELECT ID FROM PEOPLE WHERE DELETED='Y' AND last_modified >='${dih.People.last_index_time}' '' " +
- "deltaImportQuery=''SELECT ID, NAME, COUNTRY_CODE FROM PEOPLE where ID=${dih.delta.ID} '' " +
- "deltaQuery=''" +
- "SELECT ID FROM PEOPLE WHERE DELETED!='Y' AND last_modified >='${dih.People.last_index_time}' " +
- (useParentDeltaQueryParam ? "" :
- "UNION DISTINCT " +
- "SELECT ID FROM PEOPLE WHERE DELETED!='Y' AND COUNTRY_CODE IN (SELECT CODE FROM COUNTRIES WHERE last_modified >='${dih.People.last_index_time}') "
- ) + "'' "
- ;
- }
- @Override
- protected String deltaQueriesCountryTable() {
- if(useParentDeltaQueryParam) {
- return
- "deltaQuery=''SELECT CODE FROM COUNTRIES WHERE DELETED != 'Y' AND last_modified >='${dih.last_index_time}' '' " +
- "parentDeltaQuery=''SELECT ID FROM PEOPLE WHERE DELETED != 'Y' AND COUNTRY_CODE='${Countries.CODE}' '' "
- ;
-
- }
- return "";
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestTemplateTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestTemplateTransformer.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestTemplateTransformer.java
deleted file mode 100644
index b5c3811..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestTemplateTransformer.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Arrays;
-
-/**
- * <p>
- * Test for TemplateTransformer
- * </p>
- *
- *
- * @since solr 1.3
- */
-public class TestTemplateTransformer extends AbstractDataImportHandlerTestCase {
-
- @Test
- @SuppressWarnings("unchecked")
- public void testTransformRow() {
- List fields = new ArrayList();
- fields.add(createMap("column", "firstName"));
- fields.add(createMap("column", "lastName"));
- fields.add(createMap("column", "middleName"));
- fields.add(createMap("column", "name",
- TemplateTransformer.TEMPLATE,
- "${e.lastName}, ${e.firstName} ${e.middleName}"));
- fields.add(createMap("column", "emails",
- TemplateTransformer.TEMPLATE,
- "${e.mail}"));
-
- // test reuse of template output in another template
- fields.add(createMap("column", "mrname",
- TemplateTransformer.TEMPLATE,"Mr ${e.name}"));
-
- List<String> mails = Arrays.asList("a@b.com", "c@d.com");
- Map row = createMap(
- "firstName", "Shalin",
- "middleName", "Shekhar",
- "lastName", "Mangar",
- "mail", mails);
-
- VariableResolver resolver = new VariableResolver();
- resolver.addNamespace("e", row);
- Map<String, String> entityAttrs = createMap("name", "e");
-
- Context context = getContext(null, resolver,
- null, Context.FULL_DUMP, fields, entityAttrs);
- new TemplateTransformer().transformRow(row, context);
- assertEquals("Mangar, Shalin Shekhar", row.get("name"));
- assertEquals("Mr Mangar, Shalin Shekhar", row.get("mrname"));
- assertEquals(mails,row.get("emails"));
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testTransformRowMultiValue() {
- List fields = new ArrayList();
- fields.add(createMap("column", "year"));
- fields.add(createMap("column", "month"));
- fields.add(createMap("column", "day"));
-
- // create three variations of date format
- fields.add(createMap( "column", "date",
- TemplateTransformer.TEMPLATE,
- "${e.day} ${e.month}, ${e.year}" ));
- fields.add(createMap( "column", "date",
- TemplateTransformer.TEMPLATE,
- "${e.month} ${e.day}, ${e.year}" ));
- fields.add(createMap("column", "date",
- TemplateTransformer.TEMPLATE,
- "${e.year}-${e.month}-${e.day}" ));
-
- Map row = createMap( "year", "2016",
- "month", "Apr",
- "day", "30" );
- VariableResolver resolver = new VariableResolver();
- resolver.addNamespace("e", row);
- Map<String, String> entityAttrs = createMap("date", "e");
-
- Context context = getContext(null, resolver,
- null, Context.FULL_DUMP, fields, entityAttrs);
- new TemplateTransformer().transformRow(row, context);
- assertTrue( row.get( "date" ) instanceof List );
-
- List<Object> dates = (List<Object>)row.get( "date" );
- assertEquals( dates.size(), 3 );
- assertEquals( dates.get(0).toString(), "30 Apr, 2016" );
- assertEquals( dates.get(1).toString(), "Apr 30, 2016" );
- assertEquals( dates.get(2).toString(), "2016-Apr-30" );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestURLDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestURLDataSource.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestURLDataSource.java
deleted file mode 100644
index c1acc54..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestURLDataSource.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.junit.Test;
-
-public class TestURLDataSource extends AbstractDataImportHandlerTestCase {
- private List<Map<String, String>> fields = new ArrayList<>();
- private URLDataSource dataSource = new URLDataSource();
- private VariableResolver variableResolver = new VariableResolver();
- private Context context = AbstractDataImportHandlerTestCase.getContext(null, variableResolver,
- dataSource, Context.FULL_DUMP, fields, null);
- private Properties initProps = new Properties();
-
- @Test
- public void substitutionsOnBaseUrl() throws Exception {
- String url = "http://example.com/";
-
- variableResolver.addNamespace("dataimporter.request", Collections.<String,Object>singletonMap("baseurl", url));
-
- initProps.setProperty(URLDataSource.BASE_URL, "${dataimporter.request.baseurl}");
- dataSource.init(context, initProps);
- assertEquals(url, dataSource.getBaseUrl());
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolver.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolver.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolver.java
deleted file mode 100644
index 0028564..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolver.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Properties;
-import java.util.TimeZone;
-
-import org.apache.solr.util.DateMathParser;
-import org.junit.Test;
-
-/**
- * <p>
- * Test for VariableResolver
- * </p>
- *
- *
- * @since solr 1.3
- */
-public class TestVariableResolver extends AbstractDataImportHandlerTestCase {
-
- @Test
- public void testSimpleNamespace() {
- VariableResolver vri = new VariableResolver();
- Map<String,Object> ns = new HashMap<>();
- ns.put("world", "WORLD");
- vri.addNamespace("hello", ns);
- assertEquals("WORLD", vri.resolve("hello.world"));
- }
-
- @Test
- public void testDefaults() {
- // System.out.println(System.setProperty(TestVariableResolver.class.getName(),"hello"));
- System.setProperty(TestVariableResolver.class.getName(), "hello");
- // System.out.println("s.gP()"+
- // System.getProperty(TestVariableResolver.class.getName()));
-
- Properties p = new Properties();
- p.put("hello", "world");
- VariableResolver vri = new VariableResolver(p);
- Object val = vri.resolve(TestVariableResolver.class.getName());
- // System.out.println("val = " + val);
- assertEquals("hello", val);
- assertEquals("world", vri.resolve("hello"));
- }
-
- @Test
- public void testNestedNamespace() {
- VariableResolver vri = new VariableResolver();
- Map<String,Object> ns = new HashMap<>();
- ns.put("world", "WORLD");
- vri.addNamespace("hello", ns);
- ns = new HashMap<>();
- ns.put("world1", "WORLD1");
- vri.addNamespace("hello.my", ns);
- assertEquals("WORLD1", vri.resolve("hello.my.world1"));
- }
-
- @Test
- public void test3LevelNestedNamespace() {
- VariableResolver vri = new VariableResolver();
- Map<String,Object> ns = new HashMap<>();
- ns.put("world", "WORLD");
- vri.addNamespace("hello", ns);
- ns = new HashMap<>();
- ns.put("world1", "WORLD1");
- vri.addNamespace("hello.my.new", ns);
- assertEquals("WORLD1", vri.resolve("hello.my.new.world1"));
- }
-
- @Test
- public void dateNamespaceWithValue() {
- VariableResolver vri = new VariableResolver();
- vri.setEvaluators(new DataImporter().getEvaluators(Collections
- .<Map<String,String>> emptyList()));
- Map<String,Object> ns = new HashMap<>();
- Date d = new Date();
- ns.put("dt", d);
- vri.addNamespace("A", ns);
- assertEquals(
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).format(d),
- vri.replaceTokens("${dataimporter.functions.formatDate(A.dt,'yyyy-MM-dd HH:mm:ss')}"));
- }
-
- @Test
- public void dateNamespaceWithExpr() throws Exception {
- VariableResolver vri = new VariableResolver();
- vri.setEvaluators(new DataImporter().getEvaluators(Collections
- .<Map<String,String>> emptyList()));
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
- format.setTimeZone(TimeZone.getTimeZone("UTC"));
- DateMathParser dmp = new DateMathParser(TimeZone.getDefault());
-
- String s = vri
- .replaceTokens("${dataimporter.functions.formatDate('NOW/DAY','yyyy-MM-dd HH:mm')}");
- assertEquals(
- new SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.ROOT).format(dmp.parseMath("/DAY")),
- s);
- }
-
- @Test
- public void testDefaultNamespace() {
- VariableResolver vri = new VariableResolver();
- Map<String,Object> ns = new HashMap<>();
- ns.put("world", "WORLD");
- vri.addNamespace(null, ns);
- assertEquals("WORLD", vri.resolve("world"));
- }
-
- @Test
- public void testDefaultNamespace1() {
- VariableResolver vri = new VariableResolver();
- Map<String,Object> ns = new HashMap<>();
- ns.put("world", "WORLD");
- vri.addNamespace(null, ns);
- assertEquals("WORLD", vri.resolve("world"));
- }
-
- @Test
- public void testFunctionNamespace1() throws Exception {
- VariableResolver resolver = new VariableResolver();
- final List<Map<String,String>> l = new ArrayList<>();
- Map<String,String> m = new HashMap<>();
- m.put("name", "test");
- m.put("class", E.class.getName());
- l.add(m);
- resolver.setEvaluators(new DataImporter().getEvaluators(l));
- ContextImpl context = new ContextImpl(null, resolver, null,
- Context.FULL_DUMP, Collections.EMPTY_MAP, null, null);
-
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
- format.setTimeZone(TimeZone.getTimeZone("UTC"));
- DateMathParser dmp = new DateMathParser(TimeZone.getDefault());
-
- String s = resolver
- .replaceTokens("${dataimporter.functions.formatDate('NOW/DAY','yyyy-MM-dd HH:mm')}");
- assertEquals(
- new SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.ROOT).format(dmp.parseMath("/DAY")),
- s);
- assertEquals("Hello World",
- resolver.replaceTokens("${dataimporter.functions.test('TEST')}"));
- }
-
- public static class E extends Evaluator {
- @Override
- public String evaluate(String expression, Context context) {
- return "Hello World";
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
deleted file mode 100644
index 8ee6878..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.lang.invoke.MethodHandles;
-import java.sql.Connection;
-import java.sql.Statement;
-import java.util.Locale;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import junit.framework.Assert;
-
-import org.apache.solr.request.SolrQueryRequest;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestVariableResolverEndToEnd extends AbstractDIHJdbcTestCase {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- @Test
- public void test() throws Exception {
- h.query("/dataimport", generateRequest());
- SolrQueryRequest req = null;
- try {
- req = req("q", "*:*", "wt", "json", "indent", "true");
- String response = h.query(req);
- log.debug(response);
- response = response.replaceAll("\\s","");
- Assert.assertTrue(response.contains("\"numFound\":1"));
- Pattern p = Pattern.compile("[\"]second1_s[\"][:][\"](.*?)[\"]");
- Matcher m = p.matcher(response);
- Assert.assertTrue(m.find());
- String yearStr = m.group(1);
- Assert.assertTrue(response.contains("\"second1_s\":\"" + yearStr + "\""));
- Assert.assertTrue(response.contains("\"second2_s\":\"" + yearStr + "\""));
- Assert.assertTrue(response.contains("\"second3_s\":\"" + yearStr + "\""));
- Assert.assertTrue(response.contains("\"PORK_s\":\"GRILL\""));
- Assert.assertTrue(response.contains("\"FISH_s\":\"FRY\""));
- Assert.assertTrue(response.contains("\"BEEF_CUTS_mult_s\":[\"ROUND\",\"SIRLOIN\"]"));
- } catch(Exception e) {
- throw e;
- } finally {
- req.close();
- }
- }
-
- @Override
- protected String generateConfig() {
- String thirdLocaleParam = random().nextBoolean() ? "" : (", '" + Locale.getDefault().toLanguageTag() + "'");
- StringBuilder sb = new StringBuilder();
- sb.append("<dataConfig> \n");
- sb.append("<dataSource name=\"hsqldb\" driver=\"${dataimporter.request.dots.in.hsqldb.driver}\" url=\"jdbc:hsqldb:mem:.\" /> \n");
- sb.append("<document name=\"TestEvaluators\"> \n");
- sb.append("<entity name=\"FIRST\" processor=\"SqlEntityProcessor\" dataSource=\"hsqldb\" ");
- sb.append(" query=\"" +
- "select " +
- " 1 as id, " +
- " 'SELECT' as SELECT_KEYWORD, " +
- " {ts '2017-02-18 12:34:56'} as FIRST_TS " +
- "from DUAL \" >\n");
- sb.append(" <field column=\"SELECT_KEYWORD\" name=\"select_keyword_s\" /> \n");
- sb.append(" <entity name=\"SECOND\" processor=\"SqlEntityProcessor\" dataSource=\"hsqldb\" transformer=\"TemplateTransformer\" ");
- sb.append(" query=\"" +
- "${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
- " 1 as SORT, " +
- " {ts '2017-02-18 12:34:56'} as SECOND_TS, " +
- " '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam + ")}' as SECOND1_S, " +
- " 'PORK' AS MEAT, " +
- " 'GRILL' AS METHOD, " +
- " 'ROUND' AS CUTS, " +
- " 'BEEF_CUTS' AS WHATKIND " +
- "from DUAL " +
- "WHERE 1=${FIRST.ID} " +
- "UNION " +
- "${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
- " 2 as SORT, " +
- " {ts '2017-02-18 12:34:56'} as SECOND_TS, " +
- " '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam + ")}' as SECOND1_S, " +
- " 'FISH' AS MEAT, " +
- " 'FRY' AS METHOD, " +
- " 'SIRLOIN' AS CUTS, " +
- " 'BEEF_CUTS' AS WHATKIND " +
- "from DUAL " +
- "WHERE 1=${FIRST.ID} " +
- "ORDER BY SORT \"" +
- ">\n");
- sb.append(" <field column=\"SECOND_S\" name=\"second_s\" /> \n");
- sb.append(" <field column=\"SECOND1_S\" name=\"second1_s\" /> \n");
- sb.append(" <field column=\"second2_s\" template=\"${dataimporter.functions.formatDate(SECOND.SECOND_TS, 'yyyy'" + thirdLocaleParam + ")}\" /> \n");
- sb.append(" <field column=\"second3_s\" template=\"${dih.functions.formatDate(SECOND.SECOND_TS, 'yyyy'" + thirdLocaleParam + ")}\" /> \n");
- sb.append(" <field column=\"METHOD\" name=\"${SECOND.MEAT}_s\"/>\n");
- sb.append(" <field column=\"CUTS\" name=\"${SECOND.WHATKIND}_mult_s\"/>\n");
- sb.append(" </entity>\n");
- sb.append("</entity>\n");
- sb.append("</document> \n");
- sb.append("</dataConfig> \n");
- String config = sb.toString();
- log.info(config);
- return config;
- }
- @Override
- protected void populateData(Connection conn) throws Exception {
- Statement s = null;
- try {
- s = conn.createStatement();
- s.executeUpdate("create table dual(dual char(1) not null)");
- s.executeUpdate("insert into dual values('Y')");
- conn.commit();
- } catch (Exception e) {
- throw e;
- } finally {
- try {
- s.close();
- } catch (Exception ex) {}
- try {
- conn.close();
- } catch (Exception ex) {}
- }
- }
- @Override
- protected Database setAllowedDatabases() {
- return Database.HSQLDB;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestWriterImpl.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestWriterImpl.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestWriterImpl.java
deleted file mode 100644
index e5c2a94..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestWriterImpl.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.util.*;
-
-/**
- * <p>
- * Test for writerImpl paramater (to provide own SolrWriter)
- * </p>
- *
- *
- * @since solr 4.0
- */
-public class TestWriterImpl extends AbstractDataImportHandlerTestCase {
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- initCore("dataimport-nodatasource-solrconfig.xml", "dataimport-schema.xml");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testDataConfigWithDataSource() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- rows.add(createMap("id", "2", "desc", "two"));
- rows.add(createMap("id", "3", "desc", "break"));
- rows.add(createMap("id", "4", "desc", "four"));
-
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- Map extraParams = createMap("writerImpl", TestSolrWriter.class.getName(),
- "commit", "true");
- runFullImport(loadDataConfig("data-config-with-datasource.xml"),
- extraParams);
-
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("id:2"), "//*[@numFound='1']");
- assertQ(req("id:3"), "//*[@numFound='0']");
- assertQ(req("id:4"), "//*[@numFound='1']");
- }
-
- public static class TestSolrWriter extends SolrWriter {
-
- public TestSolrWriter(UpdateRequestProcessor processor, SolrQueryRequest req) {
- super(processor, req);
- }
-
- @Override
- public boolean upload(SolrInputDocument doc) {
- if (doc.getField("desc").getFirstValue().equals("break")) {
- return false;
- }
- return super.upload(doc);
- }
-
- }
-
-}
[22/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java
deleted file mode 100644
index 20a2aad..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.handler.dataimport.config.Script;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * <p>
- * An implementation for the Context
- * </p>
- * <b>This API is experimental and subject to change</b>
- *
- * @since solr 1.3
- */
-public class ContextImpl extends Context {
- protected EntityProcessorWrapper epw;
-
- private ContextImpl parent;
-
- private VariableResolver resolver;
-
- private DataSource ds;
-
- private String currProcess;
-
- private Map<String, Object> requestParams;
-
- private DataImporter dataImporter;
-
- private Map<String, Object> entitySession, globalSession;
-
- private Exception lastException = null;
-
- DocBuilder.DocWrapper doc;
-
- DocBuilder docBuilder;
-
-
-
- public ContextImpl(EntityProcessorWrapper epw, VariableResolver resolver,
- DataSource ds, String currProcess,
- Map<String, Object> global, ContextImpl parentContext, DocBuilder docBuilder) {
- this.epw = epw;
- this.docBuilder = docBuilder;
- this.resolver = resolver;
- this.ds = ds;
- this.currProcess = currProcess;
- if (docBuilder != null) {
- this.requestParams = docBuilder.getReqParams().getRawParams();
- dataImporter = docBuilder.dataImporter;
- }
- globalSession = global;
- parent = parentContext;
- }
-
- @Override
- public String getEntityAttribute(String name) {
- return epw==null || epw.getEntity() == null ? null : epw.getEntity().getAllAttributes().get(name);
- }
-
- @Override
- public String getResolvedEntityAttribute(String name) {
- return epw==null || epw.getEntity() == null ? null : resolver.replaceTokens(epw.getEntity().getAllAttributes().get(name));
- }
-
- @Override
- public List<Map<String, String>> getAllEntityFields() {
- return epw==null || epw.getEntity() == null ? Collections.EMPTY_LIST : epw.getEntity().getAllFieldsList();
- }
-
- @Override
- public VariableResolver getVariableResolver() {
- return resolver;
- }
-
- @Override
- public DataSource getDataSource() {
- if (ds != null) return ds;
- if(epw==null) { return null; }
- if (epw!=null && epw.getDatasource() == null) {
- epw.setDatasource(dataImporter.getDataSourceInstance(epw.getEntity(), epw.getEntity().getDataSourceName(), this));
- }
- if (epw!=null && epw.getDatasource() != null && docBuilder != null && docBuilder.verboseDebug &&
- Context.FULL_DUMP.equals(currentProcess())) {
- //debug is not yet implemented properly for deltas
- epw.setDatasource(docBuilder.getDebugLogger().wrapDs(epw.getDatasource()));
- }
- return epw.getDatasource();
- }
-
- @Override
- public DataSource getDataSource(String name) {
- return dataImporter.getDataSourceInstance(epw==null ? null : epw.getEntity(), name, this);
- }
-
- @Override
- public boolean isRootEntity() {
- return epw==null ? false : epw.getEntity().isDocRoot();
- }
-
- @Override
- public String currentProcess() {
- return currProcess;
- }
-
- @Override
- public Map<String, Object> getRequestParameters() {
- return requestParams;
- }
-
- @Override
- public EntityProcessor getEntityProcessor() {
- return epw;
- }
-
- @Override
- public void setSessionAttribute(String name, Object val, String scope) {
- if(name == null) {
- return;
- }
- if (Context.SCOPE_ENTITY.equals(scope)) {
- if (entitySession == null) {
- entitySession = new HashMap<>();
- }
- entitySession.put(name, val);
- } else if (Context.SCOPE_GLOBAL.equals(scope)) {
- if (globalSession != null) {
- globalSession.put(name, val);
- }
- } else if (Context.SCOPE_DOC.equals(scope)) {
- DocBuilder.DocWrapper doc = getDocument();
- if (doc != null) {
- doc.setSessionAttribute(name, val);
- }
- } else if (SCOPE_SOLR_CORE.equals(scope)){
- if(dataImporter != null) {
- dataImporter.putToCoreScopeSession(name, val);
- }
- }
- }
-
- @Override
- public Object getSessionAttribute(String name, String scope) {
- if (Context.SCOPE_ENTITY.equals(scope)) {
- if (entitySession == null)
- return null;
- return entitySession.get(name);
- } else if (Context.SCOPE_GLOBAL.equals(scope)) {
- if (globalSession != null) {
- return globalSession.get(name);
- }
- } else if (Context.SCOPE_DOC.equals(scope)) {
- DocBuilder.DocWrapper doc = getDocument();
- return doc == null ? null: doc.getSessionAttribute(name);
- } else if (SCOPE_SOLR_CORE.equals(scope)){
- return dataImporter == null ? null : dataImporter.getFromCoreScopeSession(name);
- }
- return null;
- }
-
- @Override
- public Context getParentContext() {
- return parent;
- }
-
- private DocBuilder.DocWrapper getDocument() {
- ContextImpl c = this;
- while (true) {
- if (c.doc != null)
- return c.doc;
- if (c.parent != null)
- c = c.parent;
- else
- return null;
- }
- }
-
- void setDoc(DocBuilder.DocWrapper docWrapper) {
- this.doc = docWrapper;
- }
-
-
- @Override
- public SolrCore getSolrCore() {
- return dataImporter == null ? null : dataImporter.getCore();
- }
-
-
- @Override
- public Map<String, Object> getStats() {
- return docBuilder != null ? docBuilder.importStatistics.getStatsSnapshot() : Collections.<String, Object>emptyMap();
- }
-
- @Override
- public String getScript() {
- if (dataImporter != null) {
- Script script = dataImporter.getConfig().getScript();
- return script == null ? null : script.getText();
- }
- return null;
- }
-
- @Override
- public String getScriptLanguage() {
- if (dataImporter != null) {
- Script script = dataImporter.getConfig().getScript();
- return script == null ? null : script.getLanguage();
- }
- return null;
- }
-
- @Override
- public void deleteDoc(String id) {
- if(docBuilder != null){
- docBuilder.writer.deleteDoc(id);
- }
- }
-
- @Override
- public void deleteDocByQuery(String query) {
- if(docBuilder != null){
- docBuilder.writer.deleteByQuery(query);
- }
- }
-
- DocBuilder getDocBuilder(){
- return docBuilder;
- }
- @Override
- public Object resolve(String var) {
- return resolver.resolve(var);
- }
-
- @Override
- public String replaceTokens(String template) {
- return resolver.replaceTokens(template);
- }
-
- public Exception getLastException() { return lastException; }
-
- public void setLastException(Exception lastException) {this.lastException = lastException; }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCache.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCache.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCache.java
deleted file mode 100644
index a67b3e4..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCache.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.Iterator;
-import java.util.Map;
-
-/**
- * <p>
- * A cache that allows a DIH entity's data to persist locally prior being joined
- * to other data and/or indexed.
- * </p>
- *
- * @lucene.experimental
- */
-public interface DIHCache extends Iterable<Map<String,Object>> {
-
- /**
- * <p>
- * Opens the cache using the specified properties. The {@link Context}
- * includes any parameters needed by the cache impl. This must be called
- * before any read/write operations are permitted.
- */
- void open(Context context);
-
- /**
- * <p>
- * Releases resources used by this cache, if possible. The cache is flushed
- * but not destroyed.
- * </p>
- */
- void close();
-
- /**
- * <p>
- * Persists any pending data to the cache
- * </p>
- */
- void flush();
-
- /**
- * <p>
- * Closes the cache, if open. Then removes all data, possibly removing the
- * cache entirely from persistent storage.
- * </p>
- */
- public void destroy();
-
- /**
- * <p>
- * Adds a document. If a document already exists with the same key, both
- * documents will exist in the cache, as the cache allows duplicate keys. To
- * update a key's documents, first call delete(Object key).
- * </p>
- */
- void add(Map<String, Object> rec);
-
- /**
- * <p>
- * Returns an iterator, allowing callers to iterate through the entire cache
- * in key, then insertion, order.
- * </p>
- */
- @Override
- Iterator<Map<String,Object>> iterator();
-
- /**
- * <p>
- * Returns an iterator, allowing callers to iterate through all documents that
- * match the given key in insertion order.
- * </p>
- */
- Iterator<Map<String,Object>> iterator(Object key);
-
- /**
- * <p>
- * Delete all documents associated with the given key
- * </p>
- */
- void delete(Object key);
-
- /**
- * <p>
- * Delete all data from the cache,leaving the empty cache intact.
- * </p>
- */
- void deleteAll();
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java
deleted file mode 100644
index 2f3d957..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-
-import java.lang.invoke.MethodHandles;
-import java.lang.reflect.Constructor;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-import org.apache.solr.common.SolrException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class DIHCacheSupport {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private String cacheForeignKey;
- private String cacheImplName;
- private Map<String,DIHCache> queryVsCache = new HashMap<>();
- private Map<String,Iterator<Map<String,Object>>> queryVsCacheIterator;
- private Iterator<Map<String,Object>> dataSourceRowCache;
- private boolean cacheDoKeyLookup;
-
- public DIHCacheSupport(Context context, String cacheImplName) {
- this.cacheImplName = cacheImplName;
-
- Relation r = new Relation(context);
- cacheDoKeyLookup = r.doKeyLookup;
- String cacheKey = r.primaryKey;
- cacheForeignKey = r.foreignKey;
-
- context.setSessionAttribute(DIHCacheSupport.CACHE_PRIMARY_KEY, cacheKey,
- Context.SCOPE_ENTITY);
- context.setSessionAttribute(DIHCacheSupport.CACHE_FOREIGN_KEY, cacheForeignKey,
- Context.SCOPE_ENTITY);
- context.setSessionAttribute(DIHCacheSupport.CACHE_DELETE_PRIOR_DATA,
- "true", Context.SCOPE_ENTITY);
- context.setSessionAttribute(DIHCacheSupport.CACHE_READ_ONLY, "false",
- Context.SCOPE_ENTITY);
- }
-
- static class Relation{
- protected final boolean doKeyLookup;
- protected final String foreignKey;
- protected final String primaryKey;
-
- public Relation(Context context) {
- String where = context.getEntityAttribute("where");
- String cacheKey = context.getEntityAttribute(DIHCacheSupport.CACHE_PRIMARY_KEY);
- String lookupKey = context.getEntityAttribute(DIHCacheSupport.CACHE_FOREIGN_KEY);
- if (cacheKey != null && lookupKey == null) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "'cacheKey' is specified for the entity "
- + context.getEntityAttribute("name")
- + " but 'cacheLookup' is missing");
-
- }
- if (where == null && cacheKey == null) {
- doKeyLookup = false;
- primaryKey = null;
- foreignKey = null;
- } else {
- if (where != null) {
- String[] splits = where.split("=");
- primaryKey = splits[0];
- foreignKey = splits[1].trim();
- } else {
- primaryKey = cacheKey;
- foreignKey = lookupKey;
- }
- doKeyLookup = true;
- }
- }
-
- @Override
- public String toString() {
- return "Relation "
- + primaryKey + "="+foreignKey ;
- }
-
-
- }
-
- private DIHCache instantiateCache(Context context) {
- DIHCache cache = null;
- try {
- @SuppressWarnings("unchecked")
- Class<DIHCache> cacheClass = DocBuilder.loadClass(cacheImplName, context
- .getSolrCore());
- Constructor<DIHCache> constr = cacheClass.getConstructor();
- cache = constr.newInstance();
- cache.open(context);
- } catch (Exception e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Unable to load Cache implementation:" + cacheImplName, e);
- }
- return cache;
- }
-
- public void initNewParent(Context context) {
- dataSourceRowCache = null;
- queryVsCacheIterator = new HashMap<>();
- for (Map.Entry<String,DIHCache> entry : queryVsCache.entrySet()) {
- queryVsCacheIterator.put(entry.getKey(), entry.getValue().iterator());
- }
- }
-
- public void destroyAll() {
- if (queryVsCache != null) {
- for (DIHCache cache : queryVsCache.values()) {
- cache.destroy();
- }
- }
- queryVsCache = null;
- dataSourceRowCache = null;
- cacheForeignKey = null;
- }
-
- /**
- * <p>
- * Get all the rows from the datasource for the given query and cache them
- * </p>
- */
- public void populateCache(String query,
- Iterator<Map<String,Object>> rowIterator) {
- Map<String,Object> aRow = null;
- DIHCache cache = queryVsCache.get(query);
- while ((aRow = getNextFromCache(query, rowIterator)) != null) {
- cache.add(aRow);
- }
- }
-
- private Map<String,Object> getNextFromCache(String query,
- Iterator<Map<String,Object>> rowIterator) {
- try {
- if (rowIterator == null) return null;
- if (rowIterator.hasNext()) return rowIterator.next();
- return null;
- } catch (Exception e) {
- SolrException.log(log, "getNextFromCache() failed for query '" + query
- + "'", e);
- wrapAndThrow(DataImportHandlerException.WARN, e);
- return null;
- }
- }
-
- public Map<String,Object> getCacheData(Context context, String query,
- Iterator<Map<String,Object>> rowIterator) {
- if (cacheDoKeyLookup) {
- return getIdCacheData(context, query, rowIterator);
- } else {
- return getSimpleCacheData(context, query, rowIterator);
- }
- }
-
- /**
- * If the where clause is present the cache is sql Vs Map of key Vs List of
- * Rows.
- *
- * @param query
- * the query string for which cached data is to be returned
- *
- * @return the cached row corresponding to the given query after all variables
- * have been resolved
- */
- protected Map<String,Object> getIdCacheData(Context context, String query,
- Iterator<Map<String,Object>> rowIterator) {
- Object key = context.resolve(cacheForeignKey);
- if (key == null) {
- throw new DataImportHandlerException(DataImportHandlerException.WARN,
- "The cache lookup value : " + cacheForeignKey
- + " is resolved to be null in the entity :"
- + context.getEntityAttribute("name"));
-
- }
- if (dataSourceRowCache == null) {
- DIHCache cache = queryVsCache.get(query);
-
- if (cache == null) {
- cache = instantiateCache(context);
- queryVsCache.put(query, cache);
- populateCache(query, rowIterator);
- }
- dataSourceRowCache = cache.iterator(key);
- }
- return getFromRowCacheTransformed();
- }
-
- /**
- * If where clause is not present the cache is a Map of query vs List of Rows.
- *
- * @param query
- * string for which cached row is to be returned
- *
- * @return the cached row corresponding to the given query
- */
- protected Map<String,Object> getSimpleCacheData(Context context,
- String query, Iterator<Map<String,Object>> rowIterator) {
- if (dataSourceRowCache == null) {
- DIHCache cache = queryVsCache.get(query);
- if (cache == null) {
- cache = instantiateCache(context);
- queryVsCache.put(query, cache);
- populateCache(query, rowIterator);
- queryVsCacheIterator.put(query, cache.iterator());
- }
- Iterator<Map<String,Object>> cacheIter = queryVsCacheIterator.get(query);
- dataSourceRowCache = cacheIter;
- }
-
- return getFromRowCacheTransformed();
- }
-
- protected Map<String,Object> getFromRowCacheTransformed() {
- if (dataSourceRowCache == null || !dataSourceRowCache.hasNext()) {
- dataSourceRowCache = null;
- return null;
- }
- Map<String,Object> r = dataSourceRowCache.next();
- return r;
- }
-
- /**
- * <p>
- * Specify the class for the cache implementation
- * </p>
- */
- public static final String CACHE_IMPL = "cacheImpl";
-
- /**
- * <p>
- * If the cache supports persistent data, set to "true" to delete any prior
- * persisted data before running the entity.
- * </p>
- */
-
- public static final String CACHE_DELETE_PRIOR_DATA = "cacheDeletePriorData";
- /**
- * <p>
- * Specify the Foreign Key from the parent entity to join on. Use if the cache
- * is on a child entity.
- * </p>
- */
- public static final String CACHE_FOREIGN_KEY = "cacheLookup";
-
- /**
- * <p>
- * Specify the Primary Key field from this Entity to map the input records
- * with
- * </p>
- */
- public static final String CACHE_PRIMARY_KEY = "cacheKey";
- /**
- * <p>
- * If true, a pre-existing cache is re-opened for read-only access.
- * </p>
- */
- public static final String CACHE_READ_ONLY = "cacheReadOnly";
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHLogLevels.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHLogLevels.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHLogLevels.java
deleted file mode 100644
index 24732d1..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHLogLevels.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-public enum DIHLogLevels {
- START_ENTITY, END_ENTITY, TRANSFORMED_ROW, ENTITY_META, PRE_TRANSFORMER_ROW, START_DOC, END_DOC, ENTITY_OUT, ROW_END, TRANSFORMER_EXCEPTION, ENTITY_EXCEPTION, DISABLE_LOGGING, ENABLE_LOGGING, NONE
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHProperties.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHProperties.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHProperties.java
deleted file mode 100644
index f51ef07..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHProperties.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.Date;
-import java.util.Map;
-
-/**
- * Implementations write out properties about the last data import
- * for use by the next import. ex: to persist the last import timestamp
- * so that future delta imports can know what needs to be updated.
- *
- * @lucene.experimental
- */
-public abstract class DIHProperties {
-
- public abstract void init(DataImporter dataImporter, Map<String, String> initParams);
-
- public abstract boolean isWritable();
-
- public abstract void persist(Map<String, Object> props);
-
- public abstract Map<String, Object> readIndexerProperties();
-
- public abstract String convertDateToString(Date d);
-
- public Date getCurrentTimestamp() {
- return new Date();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriter.java
deleted file mode 100644
index bdb988d..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriter.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.solr.common.SolrInputDocument;
-
-/**
- * @lucene.experimental
- *
- */
-public interface DIHWriter {
-
- /**
- * <p>
- * If this writer supports transactions or commit points, then commit any changes,
- * optionally optimizing the data for read/write performance
- * </p>
- */
- public void commit(boolean optimize);
-
- /**
- * <p>
- * Release resources used by this writer. After calling close, reads & updates will throw exceptions.
- * </p>
- */
- public void close();
-
- /**
- * <p>
- * If this writer supports transactions or commit points, then roll back any uncommitted changes.
- * </p>
- */
- public void rollback();
-
- /**
- * <p>
- * Delete from the writer's underlying data store based the passed-in writer-specific query. (Optional Operation)
- * </p>
- */
- public void deleteByQuery(String q);
-
- /**
- * <p>
- * Delete everything from the writer's underlying data store
- * </p>
- */
- public void doDeleteAll();
-
- /**
- * <p>
- * Delete from the writer's underlying data store based on the passed-in Primary Key
- * </p>
- */
- public void deleteDoc(Object key);
-
-
-
- /**
- * <p>
- * Add a document to this writer's underlying data store.
- * </p>
- * @return true on success, false on failure
- */
- public boolean upload(SolrInputDocument doc);
-
-
-
- /**
- * <p>
- * Provide context information for this writer. init() should be called before using the writer.
- * </p>
- */
- public void init(Context context) ;
-
-
- /**
- * <p>
- * Specify the keys to be modified by a delta update (required by writers that can store duplicate keys)
- * </p>
- */
- public void setDeltaKeys(Set<Map<String, Object>> deltaKeys) ;
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriterBase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriterBase.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriterBase.java
deleted file mode 100644
index 43e92c3..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriterBase.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-public abstract class DIHWriterBase implements DIHWriter {
- protected String keyFieldName;
- protected Set<Object> deltaKeys = null;
-
- @Override
- public void setDeltaKeys(Set<Map<String,Object>> passedInDeltaKeys) {
- deltaKeys = new HashSet<>();
- for (Map<String,Object> aMap : passedInDeltaKeys) {
- if (aMap.size() > 0) {
- Object key = null;
- if (keyFieldName != null) {
- key = aMap.get(keyFieldName);
- } else {
- key = aMap.entrySet().iterator().next();
- }
- if (key != null) {
- deltaKeys.add(key);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java
deleted file mode 100644
index 71ee442..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.lang.invoke.MethodHandles;
-import java.lang.reflect.Constructor;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.MapSolrParams;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.ContentStream;
-import org.apache.solr.common.util.ContentStreamBase;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.handler.RequestHandlerBase;
-import org.apache.solr.metrics.MetricsMap;
-import org.apache.solr.metrics.SolrMetricManager;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.RawResponseWriter;
-import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
-import org.apache.solr.update.processor.UpdateRequestProcessorChain;
-import org.apache.solr.util.plugin.SolrCoreAware;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.solr.handler.dataimport.DataImporter.IMPORT_CMD;
-
-/**
- * <p>
- * Solr Request Handler for data import from databases and REST data sources.
- * </p>
- * <p>
- * It is configured in solrconfig.xml
- * </p>
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
- * <p>
- * <b>This API is experimental and subject to change</b>
- *
- * @since solr 1.3
- */
-public class DataImportHandler extends RequestHandlerBase implements
- SolrCoreAware {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private DataImporter importer;
-
- private boolean debugEnabled = true;
-
- private String myName = "dataimport";
-
- private MetricsMap metrics;
-
- private static final String PARAM_WRITER_IMPL = "writerImpl";
- private static final String DEFAULT_WRITER_NAME = "SolrWriter";
-
- public DataImporter getImporter() {
- return this.importer;
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void init(NamedList args) {
- super.init(args);
- Map<String,String> macro = new HashMap<>();
- macro.put("expandMacros", "false");
- defaults = SolrParams.wrapDefaults(defaults, new MapSolrParams(macro));
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void inform(SolrCore core) {
- try {
- String name = getPluginInfo().name;
- if (name.startsWith("/")) {
- myName = name.substring(1);
- }
- // some users may have '/' in the handler name. replace with '_'
- myName = myName.replaceAll("/", "_");
- debugEnabled = StrUtils.parseBool((String)initArgs.get(ENABLE_DEBUG), true);
- importer = new DataImporter(core, myName);
- } catch (Exception e) {
- log.error( DataImporter.MSG.LOAD_EXP, e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, DataImporter.MSG.LOAD_EXP, e);
- }
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp)
- throws Exception {
- rsp.setHttpCaching(false);
-
- //TODO: figure out why just the first one is OK...
- ContentStream contentStream = null;
- Iterable<ContentStream> streams = req.getContentStreams();
- if(streams != null){
- for (ContentStream stream : streams) {
- contentStream = stream;
- break;
- }
- }
- SolrParams params = req.getParams();
- NamedList defaultParams = (NamedList) initArgs.get("defaults");
- RequestInfo requestParams = new RequestInfo(req, getParamsMap(params), contentStream);
- String command = requestParams.getCommand();
-
- if (DataImporter.SHOW_CONF_CMD.equals(command)) {
- String dataConfigFile = params.get("config");
- String dataConfig = params.get("dataConfig");
- if(dataConfigFile != null) {
- dataConfig = SolrWriter.getResourceAsString(req.getCore().getResourceLoader().openResource(dataConfigFile));
- }
- if(dataConfig==null) {
- rsp.add("status", DataImporter.MSG.NO_CONFIG_FOUND);
- } else {
- // Modify incoming request params to add wt=raw
- ModifiableSolrParams rawParams = new ModifiableSolrParams(req.getParams());
- rawParams.set(CommonParams.WT, "raw");
- req.setParams(rawParams);
- ContentStreamBase content = new ContentStreamBase.StringStream(dataConfig);
- rsp.add(RawResponseWriter.CONTENT, content);
- }
- return;
- }
-
- rsp.add("initArgs", initArgs);
- String message = "";
-
- if (command != null) {
- rsp.add("command", command);
- }
- // If importer is still null
- if (importer == null) {
- rsp.add("status", DataImporter.MSG.NO_INIT);
- return;
- }
-
- if (command != null && DataImporter.ABORT_CMD.equals(command)) {
- importer.runCmd(requestParams, null);
- } else if (importer.isBusy()) {
- message = DataImporter.MSG.CMD_RUNNING;
- } else if (command != null) {
- if (DataImporter.FULL_IMPORT_CMD.equals(command)
- || DataImporter.DELTA_IMPORT_CMD.equals(command) ||
- IMPORT_CMD.equals(command)) {
- importer.maybeReloadConfiguration(requestParams, defaultParams);
- UpdateRequestProcessorChain processorChain =
- req.getCore().getUpdateProcessorChain(params);
- UpdateRequestProcessor processor = processorChain.createProcessor(req, rsp);
- SolrResourceLoader loader = req.getCore().getResourceLoader();
- DIHWriter sw = getSolrWriter(processor, loader, requestParams, req);
-
- if (requestParams.isDebug()) {
- if (debugEnabled) {
- // Synchronous request for the debug mode
- importer.runCmd(requestParams, sw);
- rsp.add("mode", "debug");
- rsp.add("documents", requestParams.getDebugInfo().debugDocuments);
- if (requestParams.getDebugInfo().debugVerboseOutput != null) {
- rsp.add("verbose-output", requestParams.getDebugInfo().debugVerboseOutput);
- }
- } else {
- message = DataImporter.MSG.DEBUG_NOT_ENABLED;
- }
- } else {
- // Asynchronous request for normal mode
- if(requestParams.getContentStream() == null && !requestParams.isSyncMode()){
- importer.runAsync(requestParams, sw);
- } else {
- importer.runCmd(requestParams, sw);
- }
- }
- } else if (DataImporter.RELOAD_CONF_CMD.equals(command)) {
- if(importer.maybeReloadConfiguration(requestParams, defaultParams)) {
- message = DataImporter.MSG.CONFIG_RELOADED;
- } else {
- message = DataImporter.MSG.CONFIG_NOT_RELOADED;
- }
- }
- }
- rsp.add("status", importer.isBusy() ? "busy" : "idle");
- rsp.add("importResponse", message);
- rsp.add("statusMessages", importer.getStatusMessages());
- }
-
- /** The value is converted to a String or {@code List<String>} if multi-valued. */
- private Map<String, Object> getParamsMap(SolrParams params) {
- Map<String, Object> result = new HashMap<>();
- for (Map.Entry<String, String[]> pair : params){
- String s = pair.getKey();
- String[] val = pair.getValue();
- if (val == null || val.length < 1)
- continue;
- if (val.length == 1)
- result.put(s, val[0]);
- else
- result.put(s, Arrays.asList(val));
- }
- return result;
- }
-
- private DIHWriter getSolrWriter(final UpdateRequestProcessor processor,
- final SolrResourceLoader loader, final RequestInfo requestParams,
- SolrQueryRequest req) {
- SolrParams reqParams = req.getParams();
- String writerClassStr = null;
- if (reqParams != null && reqParams.get(PARAM_WRITER_IMPL) != null) {
- writerClassStr = (String) reqParams.get(PARAM_WRITER_IMPL);
- }
- DIHWriter writer;
- if (writerClassStr != null
- && !writerClassStr.equals(DEFAULT_WRITER_NAME)
- && !writerClassStr.equals(DocBuilder.class.getPackage().getName() + "."
- + DEFAULT_WRITER_NAME)) {
- try {
- @SuppressWarnings("unchecked")
- Class<DIHWriter> writerClass = DocBuilder.loadClass(writerClassStr, req.getCore());
- Constructor<DIHWriter> cnstr = writerClass.getConstructor(new Class[] {
- UpdateRequestProcessor.class, SolrQueryRequest.class});
- return cnstr.newInstance((Object) processor, (Object) req);
- } catch (Exception e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Unable to load Writer implementation:" + writerClassStr, e);
- }
- } else {
- return new SolrWriter(processor, req) {
- @Override
- public boolean upload(SolrInputDocument document) {
- try {
- return super.upload(document);
- } catch (RuntimeException e) {
- log.error("Exception while adding: " + document, e);
- return false;
- }
- }
- };
- }
- }
-
- @Override
- public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) {
- super.initializeMetrics(manager, registryName, tag, scope);
- metrics = new MetricsMap((detailed, map) -> {
- if (importer != null) {
- DocBuilder.Statistics cumulative = importer.cumulativeStatistics;
-
- map.put("Status", importer.getStatus().toString());
-
- if (importer.docBuilder != null) {
- DocBuilder.Statistics running = importer.docBuilder.importStatistics;
- map.put("Documents Processed", running.docCount);
- map.put("Requests made to DataSource", running.queryCount);
- map.put("Rows Fetched", running.rowsCount);
- map.put("Documents Deleted", running.deletedDocCount);
- map.put("Documents Skipped", running.skipDocCount);
- }
-
- map.put(DataImporter.MSG.TOTAL_DOC_PROCESSED, cumulative.docCount);
- map.put(DataImporter.MSG.TOTAL_QUERIES_EXECUTED, cumulative.queryCount);
- map.put(DataImporter.MSG.TOTAL_ROWS_EXECUTED, cumulative.rowsCount);
- map.put(DataImporter.MSG.TOTAL_DOCS_DELETED, cumulative.deletedDocCount);
- map.put(DataImporter.MSG.TOTAL_DOCS_SKIPPED, cumulative.skipDocCount);
- }
- });
- manager.registerGauge(this, registryName, metrics, tag, true, "importer", getCategory().toString(), scope);
- }
-
- // //////////////////////SolrInfoMBeans methods //////////////////////
-
- @Override
- public String getDescription() {
- return DataImporter.MSG.JMX_DESC;
- }
-
- public static final String ENABLE_DEBUG = "enableDebug";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java
deleted file mode 100644
index e69b3fd..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-/**
- * <p> Exception class for all DataImportHandler exceptions </p>
- * <p>
- * <b>This API is experimental and subject to change</b>
- *
- * @since solr 1.3
- */
-public class DataImportHandlerException extends RuntimeException {
- private int errCode;
-
- public boolean debugged = false;
-
- public static final int SEVERE = 500, WARN = 400, SKIP = 300, SKIP_ROW =301;
-
- public DataImportHandlerException(int err) {
- super();
- errCode = err;
- }
-
- public DataImportHandlerException(int err, String message) {
- super(message + (SolrWriter.getDocCount() == null ? "" : MSG + SolrWriter.getDocCount()));
- errCode = err;
- }
-
- public DataImportHandlerException(int err, String message, Throwable cause) {
- super(message + (SolrWriter.getDocCount() == null ? "" : MSG + SolrWriter.getDocCount()), cause);
- errCode = err;
- }
-
- public DataImportHandlerException(int err, Throwable cause) {
- super(cause);
- errCode = err;
- }
-
- public int getErrCode() {
- return errCode;
- }
-
- public static DataImportHandlerException wrapAndThrow(int err, Exception e) {
- if (e instanceof DataImportHandlerException) {
- throw (DataImportHandlerException) e;
- } else {
- throw new DataImportHandlerException(err, e);
- }
- }
-
- public static DataImportHandlerException wrapAndThrow(int err, Exception e, String msg) {
- if (e instanceof DataImportHandlerException) {
- throw (DataImportHandlerException) e;
- } else {
- throw new DataImportHandlerException(err, msg, e);
- }
- }
-
-
- public static final String MSG = " Processing Document # ";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java
deleted file mode 100644
index d610d66..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java
+++ /dev/null
@@ -1,622 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.common.EmptyEntityResolver;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.util.SystemIdResolver;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.XMLErrorLogger;
-import org.apache.solr.handler.dataimport.config.ConfigNameConstants;
-import org.apache.solr.handler.dataimport.config.ConfigParseUtil;
-import org.apache.solr.handler.dataimport.config.DIHConfiguration;
-import org.apache.solr.handler.dataimport.config.Entity;
-import org.apache.solr.handler.dataimport.config.PropertyWriter;
-import org.apache.solr.handler.dataimport.config.Script;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-import static org.apache.solr.handler.dataimport.DocBuilder.loadClass;
-import static org.apache.solr.handler.dataimport.config.ConfigNameConstants.CLASS;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.NodeList;
-import org.xml.sax.InputSource;
-import org.apache.commons.io.IOUtils;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-
-import java.io.IOException;
-import java.io.StringReader;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantLock;
-
-/**
- * <p> Stores all configuration information for pulling and indexing data. </p>
- * <p>
- * <b>This API is experimental and subject to change</b>
- *
- * @since solr 1.3
- */
-public class DataImporter {
-
- public enum Status {
- IDLE, RUNNING_FULL_DUMP, RUNNING_DELTA_DUMP, JOB_FAILED
- }
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private static final XMLErrorLogger XMLLOG = new XMLErrorLogger(log);
-
- private Status status = Status.IDLE;
- private DIHConfiguration config;
- private Date indexStartTime;
- private Properties store = new Properties();
- private Map<String, Map<String,String>> requestLevelDataSourceProps = new HashMap<>();
- private IndexSchema schema;
- public DocBuilder docBuilder;
- public DocBuilder.Statistics cumulativeStatistics = new DocBuilder.Statistics();
- private SolrCore core;
- private Map<String, Object> coreScopeSession = new ConcurrentHashMap<>();
- private ReentrantLock importLock = new ReentrantLock();
- private boolean isDeltaImportSupported = false;
- private final String handlerName;
-
- /**
- * Only for testing purposes
- */
- DataImporter() {
- this.handlerName = "dataimport" ;
- }
-
- DataImporter(SolrCore core, String handlerName) {
- this.handlerName = handlerName;
- this.core = core;
- this.schema = core.getLatestSchema();
- }
-
-
-
-
- boolean maybeReloadConfiguration(RequestInfo params,
- NamedList<?> defaultParams) throws IOException {
- if (importLock.tryLock()) {
- boolean success = false;
- try {
- if (null != params.getRequest()) {
- if (schema != params.getRequest().getSchema()) {
- schema = params.getRequest().getSchema();
- }
- }
- String dataConfigText = params.getDataConfig();
- String dataconfigFile = params.getConfigFile();
- InputSource is = null;
- if(dataConfigText!=null && dataConfigText.length()>0) {
- is = new InputSource(new StringReader(dataConfigText));
- } else if(dataconfigFile!=null) {
- is = new InputSource(core.getResourceLoader().openResource(dataconfigFile));
- is.setSystemId(SystemIdResolver.createSystemIdFromResourceName(dataconfigFile));
- log.info("Loading DIH Configuration: " + dataconfigFile);
- }
- if(is!=null) {
- config = loadDataConfig(is);
- success = true;
- }
-
- Map<String,Map<String,String>> dsProps = new HashMap<>();
- if(defaultParams!=null) {
- int position = 0;
- while (position < defaultParams.size()) {
- if (defaultParams.getName(position) == null) {
- break;
- }
- String name = defaultParams.getName(position);
- if (name.equals("datasource")) {
- success = true;
- NamedList dsConfig = (NamedList) defaultParams.getVal(position);
- log.info("Getting configuration for Global Datasource...");
- Map<String,String> props = new HashMap<>();
- for (int i = 0; i < dsConfig.size(); i++) {
- props.put(dsConfig.getName(i), dsConfig.getVal(i).toString());
- }
- log.info("Adding properties to datasource: " + props);
- dsProps.put((String) dsConfig.get("name"), props);
- }
- position++;
- }
- }
- requestLevelDataSourceProps = Collections.unmodifiableMap(dsProps);
- } catch(IOException ioe) {
- throw ioe;
- } finally {
- importLock.unlock();
- }
- return success;
- } else {
- return false;
- }
- }
-
-
-
- public String getHandlerName() {
- return handlerName;
- }
-
- public IndexSchema getSchema() {
- return schema;
- }
-
- /**
- * Used by tests
- */
- void loadAndInit(String configStr) {
- config = loadDataConfig(new InputSource(new StringReader(configStr)));
- }
-
- void loadAndInit(InputSource configFile) {
- config = loadDataConfig(configFile);
- }
-
- public DIHConfiguration loadDataConfig(InputSource configFile) {
-
- DIHConfiguration dihcfg = null;
- try {
- DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
- dbf.setValidating(false);
-
- // only enable xinclude, if XML is coming from safe source (local file)
- // and a a SolrCore and SystemId is present (makes no sense otherwise):
- if (core != null && configFile.getSystemId() != null) {
- try {
- dbf.setXIncludeAware(true);
- dbf.setNamespaceAware(true);
- } catch( UnsupportedOperationException e ) {
- log.warn( "XML parser doesn't support XInclude option" );
- }
- }
-
- DocumentBuilder builder = dbf.newDocumentBuilder();
- // only enable xinclude / external entities, if XML is coming from
- // safe source (local file) and a a SolrCore and SystemId is present:
- if (core != null && configFile.getSystemId() != null) {
- builder.setEntityResolver(new SystemIdResolver(core.getResourceLoader()));
- } else {
- // Don't allow external entities without having a system ID:
- builder.setEntityResolver(EmptyEntityResolver.SAX_INSTANCE);
- }
- builder.setErrorHandler(XMLLOG);
- Document document;
- try {
- document = builder.parse(configFile);
- } finally {
- // some XML parsers are broken and don't close the byte stream (but they should according to spec)
- IOUtils.closeQuietly(configFile.getByteStream());
- }
-
- dihcfg = readFromXml(document);
- log.info("Data Configuration loaded successfully");
- } catch (Exception e) {
- throw new DataImportHandlerException(SEVERE,
- "Data Config problem: " + e.getMessage(), e);
- }
- for (Entity e : dihcfg.getEntities()) {
- if (e.getAllAttributes().containsKey(SqlEntityProcessor.DELTA_QUERY)) {
- isDeltaImportSupported = true;
- break;
- }
- }
- return dihcfg;
- }
-
- public DIHConfiguration readFromXml(Document xmlDocument) {
- DIHConfiguration config;
- List<Map<String, String >> functions = new ArrayList<>();
- Script script = null;
- Map<String, Map<String,String>> dataSources = new HashMap<>();
-
- NodeList dataConfigTags = xmlDocument.getElementsByTagName("dataConfig");
- if(dataConfigTags == null || dataConfigTags.getLength() == 0) {
- throw new DataImportHandlerException(SEVERE, "the root node '<dataConfig>' is missing");
- }
- Element e = (Element) dataConfigTags.item(0);
- List<Element> documentTags = ConfigParseUtil.getChildNodes(e, "document");
- if (documentTags.isEmpty()) {
- throw new DataImportHandlerException(SEVERE, "DataImportHandler " +
- "configuration file must have one <document> node.");
- }
-
- List<Element> scriptTags = ConfigParseUtil.getChildNodes(e, ConfigNameConstants.SCRIPT);
- if (!scriptTags.isEmpty()) {
- script = new Script(scriptTags.get(0));
- }
-
- // Add the provided evaluators
- List<Element> functionTags = ConfigParseUtil.getChildNodes(e, ConfigNameConstants.FUNCTION);
- if (!functionTags.isEmpty()) {
- for (Element element : functionTags) {
- String func = ConfigParseUtil.getStringAttribute(element, NAME, null);
- String clz = ConfigParseUtil.getStringAttribute(element, ConfigNameConstants.CLASS, null);
- if (func == null || clz == null){
- throw new DataImportHandlerException(
- SEVERE,
- "<function> must have a 'name' and 'class' attributes");
- } else {
- functions.add(ConfigParseUtil.getAllAttributes(element));
- }
- }
- }
- List<Element> dataSourceTags = ConfigParseUtil.getChildNodes(e, ConfigNameConstants.DATA_SRC);
- if (!dataSourceTags.isEmpty()) {
- for (Element element : dataSourceTags) {
- Map<String,String> p = new HashMap<>();
- HashMap<String, String> attrs = ConfigParseUtil.getAllAttributes(element);
- for (Map.Entry<String, String> entry : attrs.entrySet()) {
- p.put(entry.getKey(), entry.getValue());
- }
- dataSources.put(p.get("name"), p);
- }
- }
- if(dataSources.get(null) == null){
- for (Map<String,String> properties : dataSources.values()) {
- dataSources.put(null,properties);
- break;
- }
- }
- PropertyWriter pw = null;
- List<Element> propertyWriterTags = ConfigParseUtil.getChildNodes(e, ConfigNameConstants.PROPERTY_WRITER);
- if (propertyWriterTags.isEmpty()) {
- boolean zookeeper = false;
- if (this.core != null
- && this.core.getCoreContainer().isZooKeeperAware()) {
- zookeeper = true;
- }
- pw = new PropertyWriter(zookeeper ? "ZKPropertiesWriter"
- : "SimplePropertiesWriter", Collections.<String,String> emptyMap());
- } else if (propertyWriterTags.size() > 1) {
- throw new DataImportHandlerException(SEVERE, "Only one "
- + ConfigNameConstants.PROPERTY_WRITER + " can be configured.");
- } else {
- Element pwElement = propertyWriterTags.get(0);
- String type = null;
- Map<String,String> params = new HashMap<>();
- for (Map.Entry<String,String> entry : ConfigParseUtil.getAllAttributes(
- pwElement).entrySet()) {
- if (TYPE.equals(entry.getKey())) {
- type = entry.getValue();
- } else {
- params.put(entry.getKey(), entry.getValue());
- }
- }
- if (type == null) {
- throw new DataImportHandlerException(SEVERE, "The "
- + ConfigNameConstants.PROPERTY_WRITER + " element must specify "
- + TYPE);
- }
- pw = new PropertyWriter(type, params);
- }
- return new DIHConfiguration(documentTags.get(0), this, functions, script, dataSources, pw);
- }
-
- @SuppressWarnings("unchecked")
- private DIHProperties createPropertyWriter() {
- DIHProperties propWriter = null;
- PropertyWriter configPw = config.getPropertyWriter();
- try {
- Class<DIHProperties> writerClass = DocBuilder.loadClass(configPw.getType(), this.core);
- propWriter = writerClass.newInstance();
- propWriter.init(this, configPw.getParameters());
- } catch (Exception e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "Unable to PropertyWriter implementation:" + configPw.getType(), e);
- }
- return propWriter;
- }
-
- public DIHConfiguration getConfig() {
- return config;
- }
-
- Date getIndexStartTime() {
- return indexStartTime;
- }
-
- void setIndexStartTime(Date indextStartTime) {
- this.indexStartTime = indextStartTime;
- }
-
- void store(Object key, Object value) {
- store.put(key, value);
- }
-
- Object retrieve(Object key) {
- return store.get(key);
- }
-
- public DataSource getDataSourceInstance(Entity key, String name, Context ctx) {
- Map<String,String> p = requestLevelDataSourceProps.get(name);
- if (p == null)
- p = config.getDataSources().get(name);
- if (p == null)
- p = requestLevelDataSourceProps.get(null);// for default data source
- if (p == null)
- p = config.getDataSources().get(null);
- if (p == null)
- throw new DataImportHandlerException(SEVERE,
- "No dataSource :" + name + " available for entity :" + key.getName());
- String type = p.get(TYPE);
- DataSource dataSrc = null;
- if (type == null) {
- dataSrc = new JdbcDataSource();
- } else {
- try {
- dataSrc = (DataSource) DocBuilder.loadClass(type, getCore()).newInstance();
- } catch (Exception e) {
- wrapAndThrow(SEVERE, e, "Invalid type for data source: " + type);
- }
- }
- try {
- Properties copyProps = new Properties();
- copyProps.putAll(p);
- Map<String, Object> map = ctx.getRequestParameters();
- if (map.containsKey("rows")) {
- int rows = Integer.parseInt((String) map.get("rows"));
- if (map.containsKey("start")) {
- rows += Integer.parseInt((String) map.get("start"));
- }
- copyProps.setProperty("maxRows", String.valueOf(rows));
- }
- dataSrc.init(ctx, copyProps);
- } catch (Exception e) {
- wrapAndThrow(SEVERE, e, "Failed to initialize DataSource: " + key.getDataSourceName());
- }
- return dataSrc;
- }
-
- public Status getStatus() {
- return status;
- }
-
- public void setStatus(Status status) {
- this.status = status;
- }
-
- public boolean isBusy() {
- return importLock.isLocked();
- }
-
- public void doFullImport(DIHWriter writer, RequestInfo requestParams) {
- log.info("Starting Full Import");
- setStatus(Status.RUNNING_FULL_DUMP);
- try {
- DIHProperties dihPropWriter = createPropertyWriter();
- setIndexStartTime(dihPropWriter.getCurrentTimestamp());
- docBuilder = new DocBuilder(this, writer, dihPropWriter, requestParams);
- checkWritablePersistFile(writer, dihPropWriter);
- docBuilder.execute();
- if (!requestParams.isDebug())
- cumulativeStatistics.add(docBuilder.importStatistics);
- } catch (Exception e) {
- SolrException.log(log, "Full Import failed", e);
- docBuilder.handleError("Full Import failed", e);
- } finally {
- setStatus(Status.IDLE);
- DocBuilder.INSTANCE.set(null);
- }
-
- }
-
- private void checkWritablePersistFile(DIHWriter writer, DIHProperties dihPropWriter) {
- if (isDeltaImportSupported && !dihPropWriter.isWritable()) {
- throw new DataImportHandlerException(SEVERE,
- "Properties is not writable. Delta imports are supported by data config but will not work.");
- }
- }
-
- public void doDeltaImport(DIHWriter writer, RequestInfo requestParams) {
- log.info("Starting Delta Import");
- setStatus(Status.RUNNING_DELTA_DUMP);
- try {
- DIHProperties dihPropWriter = createPropertyWriter();
- setIndexStartTime(dihPropWriter.getCurrentTimestamp());
- docBuilder = new DocBuilder(this, writer, dihPropWriter, requestParams);
- checkWritablePersistFile(writer, dihPropWriter);
- docBuilder.execute();
- if (!requestParams.isDebug())
- cumulativeStatistics.add(docBuilder.importStatistics);
- } catch (Exception e) {
- log.error("Delta Import Failed", e);
- docBuilder.handleError("Delta Import Failed", e);
- } finally {
- setStatus(Status.IDLE);
- DocBuilder.INSTANCE.set(null);
- }
-
- }
-
- public void runAsync(final RequestInfo reqParams, final DIHWriter sw) {
- new Thread(() -> runCmd(reqParams, sw)).start();
- }
-
- void runCmd(RequestInfo reqParams, DIHWriter sw) {
- String command = reqParams.getCommand();
- if (command.equals(ABORT_CMD)) {
- if (docBuilder != null) {
- docBuilder.abort();
- }
- return;
- }
- if (!importLock.tryLock()){
- log.warn("Import command failed . another import is running");
- return;
- }
- try {
- if (FULL_IMPORT_CMD.equals(command) || IMPORT_CMD.equals(command)) {
- doFullImport(sw, reqParams);
- } else if (command.equals(DELTA_IMPORT_CMD)) {
- doDeltaImport(sw, reqParams);
- }
- } finally {
- importLock.unlock();
- }
- }
-
- @SuppressWarnings("unchecked")
- Map<String, String> getStatusMessages() {
- //this map object is a Collections.synchronizedMap(new LinkedHashMap()). if we
- // synchronize on the object it must be safe to iterate through the map
- Map statusMessages = (Map) retrieve(STATUS_MSGS);
- Map<String, String> result = new LinkedHashMap<>();
- if (statusMessages != null) {
- synchronized (statusMessages) {
- for (Object o : statusMessages.entrySet()) {
- Map.Entry e = (Map.Entry) o;
- //the toString is taken because some of the Objects create the data lazily when toString() is called
- result.put((String) e.getKey(), e.getValue().toString());
- }
- }
- }
- return result;
-
- }
-
- public DocBuilder getDocBuilder() {
- return docBuilder;
- }
-
- public DocBuilder getDocBuilder(DIHWriter writer, RequestInfo requestParams) {
- DIHProperties dihPropWriter = createPropertyWriter();
- return new DocBuilder(this, writer, dihPropWriter, requestParams);
- }
-
- Map<String, Evaluator> getEvaluators() {
- return getEvaluators(config.getFunctions());
- }
-
- /**
- * used by tests.
- */
- Map<String, Evaluator> getEvaluators(List<Map<String,String>> fn) {
- Map<String, Evaluator> evaluators = new HashMap<>();
- evaluators.put(Evaluator.DATE_FORMAT_EVALUATOR, new DateFormatEvaluator());
- evaluators.put(Evaluator.SQL_ESCAPE_EVALUATOR, new SqlEscapingEvaluator());
- evaluators.put(Evaluator.URL_ENCODE_EVALUATOR, new UrlEvaluator());
- evaluators.put(Evaluator.ESCAPE_SOLR_QUERY_CHARS, new SolrQueryEscapingEvaluator());
- SolrCore core = docBuilder == null ? null : docBuilder.dataImporter.getCore();
- for (Map<String, String> map : fn) {
- try {
- evaluators.put(map.get(NAME), (Evaluator) loadClass(map.get(CLASS), core).newInstance());
- } catch (Exception e) {
- wrapAndThrow(SEVERE, e, "Unable to instantiate evaluator: " + map.get(CLASS));
- }
- }
- return evaluators;
- }
-
- static final ThreadLocal<AtomicLong> QUERY_COUNT = new ThreadLocal<AtomicLong>() {
- @Override
- protected AtomicLong initialValue() {
- return new AtomicLong();
- }
- };
-
-
-
- static final class MSG {
- public static final String NO_CONFIG_FOUND = "Configuration not found";
-
- public static final String NO_INIT = "DataImportHandler started. Not Initialized. No commands can be run";
-
- public static final String INVALID_CONFIG = "FATAL: Could not create importer. DataImporter config invalid";
-
- public static final String LOAD_EXP = "Exception while loading DataImporter";
-
- public static final String JMX_DESC = "Manage data import from databases to Solr";
-
- public static final String CMD_RUNNING = "A command is still running...";
-
- public static final String DEBUG_NOT_ENABLED = "Debug not enabled. Add a tag <str name=\"enableDebug\">true</str> in solrconfig.xml";
-
- public static final String CONFIG_RELOADED = "Configuration Re-loaded sucessfully";
-
- public static final String CONFIG_NOT_RELOADED = "Configuration NOT Re-loaded...Data Importer is busy.";
-
- public static final String TOTAL_DOC_PROCESSED = "Total Documents Processed";
-
- public static final String TOTAL_FAILED_DOCS = "Total Documents Failed";
-
- public static final String TOTAL_QUERIES_EXECUTED = "Total Requests made to DataSource";
-
- public static final String TOTAL_ROWS_EXECUTED = "Total Rows Fetched";
-
- public static final String TOTAL_DOCS_DELETED = "Total Documents Deleted";
-
- public static final String TOTAL_DOCS_SKIPPED = "Total Documents Skipped";
- }
-
- public SolrCore getCore() {
- return core;
- }
-
- void putToCoreScopeSession(String key, Object val) {
- coreScopeSession.put(key, val);
- }
- Object getFromCoreScopeSession(String key) {
- return coreScopeSession.get(key);
- }
-
- public static final String COLUMN = "column";
-
- public static final String TYPE = "type";
-
- public static final String DATA_SRC = "dataSource";
-
- public static final String MULTI_VALUED = "multiValued";
-
- public static final String NAME = "name";
-
- public static final String STATUS_MSGS = "status-messages";
-
- public static final String FULL_IMPORT_CMD = "full-import";
-
- public static final String IMPORT_CMD = "import";
-
- public static final String DELTA_IMPORT_CMD = "delta-import";
-
- public static final String ABORT_CMD = "abort";
-
- public static final String DEBUG_MODE = "debug";
-
- public static final String RELOAD_CONF_CMD = "reload-config";
-
- public static final String SHOW_CONF_CMD = "show-config";
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataSource.java
deleted file mode 100644
index e217ddd..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataSource.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.Properties;
-
-/**
- * <p>
- * Provides data from a source with a given query.
- * </p>
- * <p>
- * Implementation of this abstract class must provide a default no-arg constructor
- * </p>
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 1.3
- */
-public abstract class DataSource<T> {
-
- /**
- * Initializes the DataSource with the <code>Context</code> and
- * initialization properties.
- * <p>
- * This is invoked by the <code>DataImporter</code> after creating an
- * instance of this class.
- */
- public abstract void init(Context context, Properties initProps);
-
- /**
- * Get records for the given query.The return type depends on the
- * implementation .
- *
- * @param query The query string. It can be a SQL for JdbcDataSource or a URL
- * for HttpDataSource or a file location for FileDataSource or a custom
- * format for your own custom DataSource.
- * @return Depends on the implementation. For instance JdbcDataSource returns
- * an Iterator<Map <String,Object>>
- */
- public abstract T getData(String query);
-
- /**
- * Cleans up resources of this DataSource after use.
- */
- public abstract void close();
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java
deleted file mode 100644
index f4df820..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.IllformedLocaleException;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
-
-import org.apache.solr.common.util.SuppressForbidden;
-import org.apache.solr.handler.dataimport.config.EntityField;
-import org.apache.solr.util.DateMathParser;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-
-/**
- * <p>Formats values using a given date format. </p>
- * <p>Pass three parameters:
- * <ul>
- * <li>An {@link EntityField} or a date expression to be parsed with
- * the {@link DateMathParser} class If the value is in a String,
- * then it is assumed to be a datemath expression, otherwise it
- * resolved using a {@link VariableResolver} instance</li>
- * <li>A date format see {@link SimpleDateFormat} for the syntax.</li>
- * <li>The {@link Locale} to parse.
- * (optional. Defaults to the Root Locale) </li>
- * </ul>
- */
-public class DateFormatEvaluator extends Evaluator {
-
- public static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
- protected Map<String, Locale> availableLocales = new HashMap<>();
- protected Set<String> availableTimezones = new HashSet<>();
-
- @SuppressForbidden(reason = "Usage of outdated locale parsing with Locale#toString() because of backwards compatibility")
- public DateFormatEvaluator() {
- for (Locale locale : Locale.getAvailableLocales()) {
- availableLocales.put(locale.toString(), locale);
- }
- for (String tz : TimeZone.getAvailableIDs()) {
- availableTimezones.add(tz);
- }
- }
-
- private SimpleDateFormat getDateFormat(String pattern, TimeZone timezone, Locale locale) {
- final SimpleDateFormat sdf = new SimpleDateFormat(pattern, locale);
- sdf.setTimeZone(timezone);
- return sdf;
- }
-
- @Override
- public String evaluate(String expression, Context context) {
- List<Object> l = parseParams(expression, context.getVariableResolver());
- if (l.size() < 2 || l.size() > 4) {
- throw new DataImportHandlerException(SEVERE, "'formatDate()' must have two, three or four parameters ");
- }
- Object o = l.get(0);
- Object format = l.get(1);
- if (format instanceof VariableWrapper) {
- VariableWrapper wrapper = (VariableWrapper) format;
- o = wrapper.resolve();
- format = o.toString();
- }
- Locale locale = Locale.ENGLISH; // we default to ENGLISH for dates for full Java 9 compatibility
- if(l.size()>2) {
- Object localeObj = l.get(2);
- String localeStr = null;
- if (localeObj instanceof VariableWrapper) {
- localeStr = ((VariableWrapper) localeObj).resolve().toString();
- } else {
- localeStr = localeObj.toString();
- }
- locale = availableLocales.get(localeStr);
- if (locale == null) try {
- locale = new Locale.Builder().setLanguageTag(localeStr).build();
- } catch (IllformedLocaleException ex) {
- throw new DataImportHandlerException(SEVERE, "Malformed / non-existent locale: " + localeStr, ex);
- }
- }
- TimeZone tz = TimeZone.getDefault(); // DWS TODO: is this the right default for us? Deserves explanation if so.
- if(l.size()==4) {
- Object tzObj = l.get(3);
- String tzStr = null;
- if (tzObj instanceof VariableWrapper) {
- tzStr = ((VariableWrapper) tzObj).resolve().toString();
- } else {
- tzStr = tzObj.toString();
- }
- if(availableTimezones.contains(tzStr)) {
- tz = TimeZone.getTimeZone(tzStr);
- } else {
- throw new DataImportHandlerException(SEVERE, "Unsupported Timezone: " + tzStr);
- }
- }
- String dateFmt = format.toString();
- SimpleDateFormat fmt = getDateFormat(dateFmt, tz, locale);
- Date date = null;
- if (o instanceof VariableWrapper) {
- date = evaluateWrapper((VariableWrapper) o, locale, tz);
- } else {
- date = evaluateString(o.toString(), locale, tz);
- }
- return fmt.format(date);
- }
-
- /**
- * NOTE: declared as a method to allow for extensibility
- *
- * @lucene.experimental this API is experimental and subject to change
- * @return the result of evaluating a string
- */
- protected Date evaluateWrapper(VariableWrapper variableWrapper, Locale locale, TimeZone tz) {
- Date date = null;
- Object variableval = resolveWrapper(variableWrapper,locale,tz);
- if (variableval instanceof Date) {
- date = (Date) variableval;
- } else {
- String s = variableval.toString();
- try {
- date = getDateFormat(DEFAULT_DATE_FORMAT, tz, locale).parse(s);
- } catch (ParseException exp) {
- wrapAndThrow(SEVERE, exp, "Invalid expression for date");
- }
- }
- return date;
- }
-
- /**
- * NOTE: declared as a method to allow for extensibility
- * @lucene.experimental
- * @return the result of evaluating a string
- */
- protected Date evaluateString(String datemathfmt, Locale locale, TimeZone tz) {
- // note: DMP does not use the locale but perhaps a subclass might use it, for e.g. parsing a date in a custom
- // string that doesn't necessarily have date math?
- //TODO refactor DateMathParser.parseMath a bit to have a static method for this logic.
- if (datemathfmt.startsWith("NOW")) {
- datemathfmt = datemathfmt.substring("NOW".length());
- }
- try {
- DateMathParser parser = new DateMathParser(tz);
- parser.setNow(new Date());// thus do *not* use SolrRequestInfo
- return parser.parseMath(datemathfmt);
- } catch (ParseException e) {
- throw wrapAndThrow(SEVERE, e, "Invalid expression for date");
- }
- }
-
- /**
- * NOTE: declared as a method to allow for extensibility
- * @lucene.experimental
- * @return the result of resolving the variable wrapper
- */
- protected Object resolveWrapper(VariableWrapper variableWrapper, Locale locale, TimeZone tz) {
- return variableWrapper.resolve();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java
deleted file mode 100644
index 6da9cc1..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.lang.invoke.MethodHandles;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.*;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * <p>
- * {@link Transformer} instance which creates {@link Date} instances out of {@link String}s.
- * </p>
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * <p>
- * <b>This API is experimental and subject to change</b>
- *
- * @since solr 1.3
- */
-public class DateFormatTransformer extends Transformer {
- private Map<String, SimpleDateFormat> fmtCache = new HashMap<>();
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- @Override
- @SuppressWarnings("unchecked")
- public Object transformRow(Map<String, Object> aRow, Context context) {
-
- for (Map<String, String> map : context.getAllEntityFields()) {
- Locale locale = Locale.ENGLISH; // we default to ENGLISH for dates for full Java 9 compatibility
- String customLocale = map.get(LOCALE);
- if (customLocale != null) {
- try {
- locale = new Locale.Builder().setLanguageTag(customLocale).build();
- } catch (IllformedLocaleException e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "Invalid Locale specified: " + customLocale, e);
- }
- }
-
- String fmt = map.get(DATE_TIME_FMT);
- if (fmt == null)
- continue;
- VariableResolver resolver = context.getVariableResolver();
- fmt = resolver.replaceTokens(fmt);
- String column = map.get(DataImporter.COLUMN);
- String srcCol = map.get(RegexTransformer.SRC_COL_NAME);
- if (srcCol == null)
- srcCol = column;
- try {
- Object o = aRow.get(srcCol);
- if (o instanceof List) {
- List inputs = (List) o;
- List<Date> results = new ArrayList<>();
- for (Object input : inputs) {
- results.add(process(input, fmt, locale));
- }
- aRow.put(column, results);
- } else {
- if (o != null) {
- aRow.put(column, process(o, fmt, locale));
- }
- }
- } catch (ParseException e) {
- log.warn("Could not parse a Date field ", e);
- }
- }
- return aRow;
- }
-
- private Date process(Object value, String format, Locale locale) throws ParseException {
- if (value == null) return null;
- String strVal = value.toString().trim();
- if (strVal.length() == 0)
- return null;
- SimpleDateFormat fmt = fmtCache.get(format);
- if (fmt == null) {
- fmt = new SimpleDateFormat(format, locale);
- fmtCache.put(format, fmt);
- }
- return fmt.parse(strVal);
- }
-
- public static final String DATE_TIME_FMT = "dateTimeFormat";
-
- public static final String LOCALE = "locale";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugInfo.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugInfo.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugInfo.java
deleted file mode 100644
index 623832f..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugInfo.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.AbstractList;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.StrUtils;
-
-public class DebugInfo {
-
- private static final class ChildRollupDocs extends AbstractList<SolrInputDocument> {
-
- private List<SolrInputDocument> delegate = new ArrayList<>();
-
- @Override
- public SolrInputDocument get(int index) {
- return delegate.get(index);
- }
-
- @Override
- public int size() {
- return delegate.size();
- }
-
- public boolean add(SolrInputDocument e) {
- SolrInputDocument transformed = e.deepCopy();
- if (transformed.hasChildDocuments()) {
- ChildRollupDocs childList = new ChildRollupDocs();
- childList.addAll(transformed.getChildDocuments());
- transformed.addField("_childDocuments_", childList);
- transformed.getChildDocuments().clear();
- }
- return delegate.add(transformed);
- }
- }
-
- public List<SolrInputDocument> debugDocuments = new ChildRollupDocs();
-
- public NamedList<String> debugVerboseOutput = null;
- public boolean verbose;
-
- public DebugInfo(Map<String,Object> requestParams) {
- verbose = StrUtils.parseBool((String) requestParams.get("verbose"), false);
- debugVerboseOutput = new NamedList<>();
- }
-}
-
[11/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml
deleted file mode 100644
index a662fd9..0000000
--- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml
+++ /dev/null
@@ -1,293 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
- <indexConfig>
- <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
- </indexConfig>
-
- <!-- Used to specify an alternate directory to hold all index data
- other than the default ./data under the Solr home.
- If replication is in use, this should match the replication configuration. -->
- <dataDir>${solr.data.dir:}</dataDir>
-
- <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
- <schemaFactory class="ClassicIndexSchemaFactory"/>
-
- <!-- the default high-performance update handler -->
- <updateHandler class="solr.DirectUpdateHandler2">
-
- <!-- A prefix of "solr." for class names is an alias that
- causes solr to search appropriate packages, including
- org.apache.solr.(search|update|request|core|analysis)
- -->
-
- <!-- Limit the number of deletions Solr will buffer during doc updating.
-
- Setting this lower can help bound memory use during indexing.
- -->
- <maxPendingDeletes>100000</maxPendingDeletes>
-
- </updateHandler>
-
-
- <query>
- <!-- Maximum number of clauses in a boolean query... can affect
- range or prefix queries that expand to big boolean
- queries. An exception is thrown if exceeded. -->
- <maxBooleanClauses>1024</maxBooleanClauses>
-
-
- <!-- Cache used by SolrIndexSearcher for filters (DocSets),
- unordered sets of *all* documents that match a query.
- When a new searcher is opened, its caches may be prepopulated
- or "autowarmed" using data from caches in the old searcher.
- autowarmCount is the number of items to prepopulate. For LRUCache,
- the autowarmed items will be the most recently accessed items.
- Parameters:
- class - the SolrCache implementation (currently only LRUCache)
- size - the maximum number of entries in the cache
- initialSize - the initial capacity (number of entries) of
- the cache. (seel java.util.HashMap)
- autowarmCount - the number of entries to prepopulate from
- and old cache.
- -->
- <filterCache
- class="solr.LRUCache"
- size="512"
- initialSize="512"
- autowarmCount="256"/>
-
- <!-- queryResultCache caches results of searches - ordered lists of
- document ids (DocList) based on a query, a sort, and the range
- of documents requested. -->
- <queryResultCache
- class="solr.LRUCache"
- size="512"
- initialSize="512"
- autowarmCount="256"/>
-
- <!-- documentCache caches Lucene Document objects (the stored fields for each document).
- Since Lucene internal document ids are transient, this cache will not be autowarmed. -->
- <documentCache
- class="solr.LRUCache"
- size="512"
- initialSize="512"
- autowarmCount="0"/>
-
- <!-- If true, stored fields that are not requested will be loaded lazily.
-
- This can result in a significant speed improvement if the usual case is to
- not load all stored fields, especially if the skipped fields are large compressed
- text fields.
- -->
- <enableLazyFieldLoading>true</enableLazyFieldLoading>
-
- <!-- Example of a generic cache. These caches may be accessed by name
- through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert().
- The purpose is to enable easy caching of user/application level data.
- The regenerator argument should be specified as an implementation
- of solr.search.CacheRegenerator if autowarming is desired. -->
- <!--
- <cache name="myUserCache"
- class="solr.LRUCache"
- size="4096"
- initialSize="1024"
- autowarmCount="1024"
- regenerator="org.mycompany.mypackage.MyRegenerator"
- />
- -->
-
- <!-- An optimization that attempts to use a filter to satisfy a search.
- If the requested sort does not include score, then the filterCache
- will be checked for a filter matching the query. If found, the filter
- will be used as the source of document ids, and then the sort will be
- applied to that.
- <useFilterForSortedQuery>true</useFilterForSortedQuery>
- -->
-
- <!-- An optimization for use with the queryResultCache. When a search
- is requested, a superset of the requested number of document ids
- are collected. For example, if a search for a particular query
- requests matching documents 10 through 19, and queryWindowSize is 50,
- then documents 0 through 49 will be collected and cached. Any further
- requests in that range can be satisfied via the cache. -->
- <queryResultWindowSize>50</queryResultWindowSize>
-
- <!-- Maximum number of documents to cache for any entry in the
- queryResultCache. -->
- <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
-
- <!-- This entry enables an int hash representation for filters (DocSets)
- when the number of items in the set is less than maxSize. For smaller
- sets, this representation is more memory efficient, more efficient to
- iterate over, and faster to take intersections. -->
- <HashDocSet maxSize="3000" loadFactor="0.75"/>
-
- <!-- a newSearcher event is fired whenever a new searcher is being prepared
- and there is a current searcher handling requests (aka registered). -->
- <!-- QuerySenderListener takes an array of NamedList and executes a
- local query request for each NamedList in sequence. -->
- <!--<listener event="newSearcher" class="solr.QuerySenderListener">-->
- <!--<arr name="queries">-->
- <!--<lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
- <!--<lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
- <!--<lst><str name="q">static newSearcher warming query from solrconfig.xml</str></lst>-->
- <!--</arr>-->
- <!--</listener>-->
-
- <!-- a firstSearcher event is fired whenever a new searcher is being
- prepared but there is no current registered searcher to handle
- requests or to gain autowarming data from. -->
- <!--<listener event="firstSearcher" class="solr.QuerySenderListener">-->
- <!--<arr name="queries">-->
- <!--</arr>-->
- <!--</listener>-->
-
- <!-- If a search request comes in and there is no current registered searcher,
- then immediately register the still warming searcher and use it. If
- "false" then all requests will block until the first searcher is done
- warming. -->
- <useColdSearcher>false</useColdSearcher>
-
- <!-- Maximum number of searchers that may be warming in the background
- concurrently. An error is returned if this limit is exceeded. Recommend
- 1-2 for read-only slaves, higher for masters w/o cache warming. -->
- <maxWarmingSearchers>4</maxWarmingSearchers>
-
- </query>
-
- <requestDispatcher>
- <!--Make sure your system has some authentication before enabling remote streaming!
- <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="-1" />
- -->
-
- <!-- Set HTTP caching related parameters (for proxy caches and clients).
-
- To get the behaviour of Solr 1.2 (ie: no caching related headers)
- use the never304="true" option and do not specify a value for
- <cacheControl>
- -->
- <httpCaching never304="true">
- <!--httpCaching lastModifiedFrom="openTime"
- etagSeed="Solr"-->
- <!-- lastModFrom="openTime" is the default, the Last-Modified value
- (and validation against If-Modified-Since requests) will all be
- relative to when the current Searcher was opened.
- You can change it to lastModFrom="dirLastMod" if you want the
- value to exactly corrispond to when the physical index was last
- modified.
-
- etagSeed="..." is an option you can change to force the ETag
- header (and validation against If-None-Match requests) to be
- differnet even if the index has not changed (ie: when making
- significant changes to your config file)
-
- lastModifiedFrom and etagSeed are both ignored if you use the
- never304="true" option.
- -->
- <!-- If you include a <cacheControl> directive, it will be used to
- generate a Cache-Control header, as well as an Expires header
- if the value contains "max-age="
-
- By default, no Cache-Control header is generated.
-
- You can use the <cacheControl> option even if you have set
- never304="true"
- -->
- <!-- <cacheControl>max-age=30, public</cacheControl> -->
- </httpCaching>
- </requestDispatcher>
-
- <requestHandler name="/select" class="solr.SearchHandler">
- <!-- default values for query parameters -->
- <lst name="defaults">
- <str name="echoParams">explicit</str>
- <str name="df">desc</str>
- <!--
- <int name="rows">10</int>
- <str name="fl">*</str>
- <str name="version">2.1</str>
- -->
- </lst>
- </requestHandler>
-
- <requestHandler name="/dataimport" class="org.apache.solr.handler.dataimport.DataImportHandler">
- <lst name="defaults">
- <str name="dots.in.hsqldb.driver">org.hsqldb.jdbcDriver</str>
- </lst>
- </requestHandler>
-
- <!--
-
- Search components are registered to SolrCore and used by Search Handlers
-
- By default, the following components are avaliable:
-
- <searchComponent name="query" class="org.apache.solr.handler.component.QueryComponent" />
- <searchComponent name="facet" class="org.apache.solr.handler.component.FacetComponent" />
- <searchComponent name="mlt" class="org.apache.solr.handler.component.MoreLikeThisComponent" />
- <searchComponent name="highlight" class="org.apache.solr.handler.component.HighlightComponent" />
- <searchComponent name="debug" class="org.apache.solr.handler.component.DebugComponent" />
-
- If you register a searchComponent to one of the standard names, that will be used instead.
-
- -->
-
- <requestHandler name="/search" class="org.apache.solr.handler.component.SearchHandler">
- <lst name="defaults">
- <str name="echoParams">explicit</str>
- </lst>
- <!--
- By default, this will register the following components:
-
- <arr name="components">
- <str>query</str>
- <str>facet</str>
- <str>mlt</str>
- <str>highlight</str>
- <str>debug</str>
- </arr>
-
- To insert handlers before or after the 'standard' components, use:
-
- <arr name="first-components">
- <str>first</str>
- </arr>
-
- <arr name="last-components">
- <str>last</str>
- </arr>
-
- -->
- </requestHandler>
-
- <!-- config for the admin interface -->
- <admin>
- <defaultQuery>*:*</defaultQuery>
- </admin>
-
- <updateRequestProcessorChain key="dataimport" default="true">
- <processor class="org.apache.solr.handler.dataimport.AbstractDataImportHandlerTestCase$TestUpdateRequestProcessorFactory"/>
- <processor class="solr.RunUpdateProcessorFactory"/>
- <processor class="solr.LogUpdateProcessorFactory"/>
- </updateRequestProcessorChain>
-
-</config>
-
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/protwords.txt
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/protwords.txt b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/protwords.txt
deleted file mode 100644
index 7878147..0000000
--- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/protwords.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#use a protected word file to avoid stemming two
-#unrelated words to the same base word.
-#to test, we will use words that would normally obviously be stemmed.
-cats
-ridding
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/single-entity-data-config.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/single-entity-data-config.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/single-entity-data-config.xml
deleted file mode 100644
index 7375a2c..0000000
--- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/single-entity-data-config.xml
+++ /dev/null
@@ -1,9 +0,0 @@
-<dataConfig>
- <dataSource type="MockDataSource"/>
- <document>
- <entity name="x" query="select * from x">
- <field column="id" />
- <field column="desc" />
- </entity>
- </document>
-</dataConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/stopwords.txt
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/stopwords.txt b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/stopwords.txt
deleted file mode 100644
index 688e307..0000000
--- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/stopwords.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-stopworda
-stopwordb
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/synonyms.txt
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/synonyms.txt b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/synonyms.txt
deleted file mode 100644
index a7624f0..0000000
--- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/synonyms.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-a => aa
-b => b1 b2
-c => c1,c2
-a\=>a => b\=>b
-a\,a => b\,b
-foo,bar,baz
-
-Television,TV,Televisions
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test-files/dih/solr/solr.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/solr.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/solr.xml
deleted file mode 100644
index 330eef1..0000000
--- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/solr.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!--
- solr.xml mimicking the old default solr.xml
--->
-
-<solr>
- <shardHandlerFactory name="shardHandlerFactory" class="HttpShardHandlerFactory">
- <str name="urlScheme">${urlScheme:}</str>
- </shardHandlerFactory>
-</solr>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java
new file mode 100644
index 0000000..9a0f3a7
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.Reader;
+import java.math.BigDecimal;
+import java.sql.Clob;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.sql.rowset.serial.SerialClob;
+
+import org.apache.solr.handler.dataimport.AbstractDataImportHandlerTestCase.TestContext;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+
+public class AbstractDIHCacheTestCase {
+ protected static final Date Feb21_2011 = new Date(1298268000000l);
+ protected final String[] fieldTypes = { "INTEGER", "BIGDECIMAL", "STRING", "STRING", "FLOAT", "DATE", "CLOB" };
+ protected final String[] fieldNames = { "a_id", "PI", "letter", "examples", "a_float", "a_date", "DESCRIPTION" };
+ protected List<ControlData> data = new ArrayList<>();
+ protected Clob APPLE = null;
+
+ @Before
+ public void setup() {
+ try {
+ APPLE = new SerialClob("Apples grow on trees and they are good to eat.".toCharArray());
+ } catch (SQLException sqe) {
+ Assert.fail("Could not Set up Test");
+ }
+
+ // The first row needs to have all non-null fields,
+ // otherwise we would have to always send the fieldTypes & fieldNames as CacheProperties when building.
+ data = new ArrayList<>();
+ data.add(new ControlData(new Object[] {1, new BigDecimal(Math.PI), "A", "Apple", 1.11f, Feb21_2011, APPLE }));
+ data.add(new ControlData(new Object[] {2, new BigDecimal(Math.PI), "B", "Ball", 2.22f, Feb21_2011, null }));
+ data.add(new ControlData(new Object[] {4, new BigDecimal(Math.PI), "D", "Dog", 4.44f, Feb21_2011, null }));
+ data.add(new ControlData(new Object[] {3, new BigDecimal(Math.PI), "C", "Cookie", 3.33f, Feb21_2011, null }));
+ data.add(new ControlData(new Object[] {4, new BigDecimal(Math.PI), "D", "Daisy", 4.44f, Feb21_2011, null }));
+ data.add(new ControlData(new Object[] {4, new BigDecimal(Math.PI), "D", "Drawing", 4.44f, Feb21_2011, null }));
+ data.add(new ControlData(new Object[] {5, new BigDecimal(Math.PI), "E",
+ Arrays.asList("Eggplant", "Ear", "Elephant", "Engine"), 5.55f, Feb21_2011, null }));
+ }
+
+ @After
+ public void teardown() {
+ APPLE = null;
+ data = null;
+ }
+
+ //A limitation of this test class is that the primary key needs to be the first one in the list.
+ //DIHCaches, however, can handle any field being the primary key.
+ static class ControlData implements Comparable<ControlData>, Iterable<Object> {
+ Object[] data;
+
+ ControlData(Object[] data) {
+ this.data = data;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public int compareTo(ControlData cd) {
+ Comparable c1 = (Comparable) data[0];
+ Comparable c2 = (Comparable) cd.data[0];
+ return c1.compareTo(c2);
+ }
+
+ @Override
+ public Iterator<Object> iterator() {
+ return Arrays.asList(data).iterator();
+ }
+ }
+
+ protected void loadData(DIHCache cache, List<ControlData> theData, String[] theFieldNames, boolean keepOrdered) {
+ for (ControlData cd : theData) {
+ cache.add(controlDataToMap(cd, theFieldNames, keepOrdered));
+ }
+ }
+
+ protected List<ControlData> extractDataInKeyOrder(DIHCache cache, String[] theFieldNames) {
+ List<Object[]> data = new ArrayList<>();
+ Iterator<Map<String, Object>> cacheIter = cache.iterator();
+ while (cacheIter.hasNext()) {
+ data.add(mapToObjectArray(cacheIter.next(), theFieldNames));
+ }
+ return listToControlData(data);
+ }
+
+ //This method assumes that the Primary Keys are integers and that the first id=1.
+ //It will look for id's sequentially until one is skipped, then will stop.
+ protected List<ControlData> extractDataByKeyLookup(DIHCache cache, String[] theFieldNames) {
+ int recId = 1;
+ List<Object[]> data = new ArrayList<>();
+ while (true) {
+ Iterator<Map<String, Object>> listORecs = cache.iterator(recId);
+ if (listORecs == null) {
+ break;
+ }
+
+ while(listORecs.hasNext()) {
+ data.add(mapToObjectArray(listORecs.next(), theFieldNames));
+ }
+ recId++;
+ }
+ return listToControlData(data);
+ }
+
+ protected List<ControlData> listToControlData(List<Object[]> data) {
+ List<ControlData> returnData = new ArrayList<>(data.size());
+ for (int i = 0; i < data.size(); i++) {
+ returnData.add(new ControlData(data.get(i)));
+ }
+ return returnData;
+ }
+
+ protected Object[] mapToObjectArray(Map<String, Object> rec, String[] theFieldNames) {
+ Object[] oos = new Object[theFieldNames.length];
+ for (int i = 0; i < theFieldNames.length; i++) {
+ oos[i] = rec.get(theFieldNames[i]);
+ }
+ return oos;
+ }
+
+ protected void compareData(List<ControlData> theControl, List<ControlData> test) {
+ // The test data should come back primarily in Key order and secondarily in insertion order.
+ List<ControlData> control = new ArrayList<>(theControl);
+ Collections.sort(control);
+
+ StringBuilder errors = new StringBuilder();
+ if (test.size() != control.size()) {
+ errors.append("-Returned data has " + test.size() + " records. expected: " + control.size() + "\n");
+ }
+ for (int i = 0; i < control.size() && i < test.size(); i++) {
+ Object[] controlRec = control.get(i).data;
+ Object[] testRec = test.get(i).data;
+ if (testRec.length != controlRec.length) {
+ errors.append("-Record indexAt=" + i + " has " + testRec.length + " data elements. extpected: " + controlRec.length + "\n");
+ }
+ for (int j = 0; j < controlRec.length && j < testRec.length; j++) {
+ Object controlObj = controlRec[j];
+ Object testObj = testRec[j];
+ if (controlObj == null && testObj != null) {
+ errors.append("-Record indexAt=" + i + ", Data Element indexAt=" + j + " is not NULL as expected.\n");
+ } else if (controlObj != null && testObj == null) {
+ errors.append("-Record indexAt=" + i + ", Data Element indexAt=" + j + " is NULL. Expected: " + controlObj + " (class="
+ + controlObj.getClass().getName() + ")\n");
+ } else if (controlObj != null && testObj != null && controlObj instanceof Clob) {
+ String controlString = clobToString((Clob) controlObj);
+ String testString = clobToString((Clob) testObj);
+ if (!controlString.equals(testString)) {
+ errors.append("-Record indexAt=" + i + ", Data Element indexAt=" + j + " has: " + testString + " (class=Clob) ... Expected: " + controlString
+ + " (class=Clob)\n");
+ }
+ } else if (controlObj != null && !controlObj.equals(testObj)) {
+ errors.append("-Record indexAt=" + i + ", Data Element indexAt=" + j + " has: " + testObj + " (class=" + testObj.getClass().getName()
+ + ") ... Expected: " + controlObj + " (class=" + controlObj.getClass().getName() + ")\n");
+ }
+ }
+ }
+ if (errors.length() > 0) {
+ Assert.fail(errors.toString());
+ }
+ }
+
+ protected Map<String, Object> controlDataToMap(ControlData cd, String[] theFieldNames, boolean keepOrdered) {
+ Map<String, Object> rec = null;
+ if (keepOrdered) {
+ rec = new LinkedHashMap<>();
+ } else {
+ rec = new HashMap<>();
+ }
+ for (int i = 0; i < cd.data.length; i++) {
+ String fieldName = theFieldNames[i];
+ Object data = cd.data[i];
+ rec.put(fieldName, data);
+ }
+ return rec;
+ }
+
+ protected String stringArrayToCommaDelimitedList(String[] strs) {
+ StringBuilder sb = new StringBuilder();
+ for (String a : strs) {
+ if (sb.length() > 0) {
+ sb.append(",");
+ }
+ sb.append(a);
+ }
+ return sb.toString();
+ }
+
+ protected String clobToString(Clob cl) {
+ StringBuilder sb = new StringBuilder();
+ try {
+ Reader in = cl.getCharacterStream();
+ char[] cbuf = new char[1024];
+ int numGot = -1;
+ while ((numGot = in.read(cbuf)) != -1) {
+ sb.append(String.valueOf(cbuf, 0, numGot));
+ }
+ } catch (Exception e) {
+ Assert.fail(e.toString());
+ }
+ return sb.toString();
+ }
+
+ public static Context getContext(final Map<String, String> entityAttrs) {
+ VariableResolver resolver = new VariableResolver();
+ final Context delegate = new ContextImpl(null, resolver, null, null, new HashMap<String, Object>(), null, null);
+ return new TestContext(entityAttrs, delegate, null, true);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java
new file mode 100644
index 0000000..1a0d888
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.OutputStream;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import junit.framework.Assert;
+
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+/**
+ * This sets up an in-memory Sql database with a little sample data.
+ */
+public abstract class AbstractDIHJdbcTestCase extends
+ AbstractDataImportHandlerTestCase {
+
+ protected Database dbToUse;
+
+ public enum Database {
+ RANDOM, DERBY, HSQLDB
+ }
+
+ protected boolean skipThisTest = false;
+
+ private static final Pattern totalRequestsPattern = Pattern
+ .compile(".str name..Total Requests made to DataSource..(\\d+)..str.");
+
+ @BeforeClass
+ public static void beforeClassDihJdbcTest() throws Exception {
+ try {
+ Class.forName("org.hsqldb.jdbcDriver").newInstance();
+ String oldProp = System.getProperty("derby.stream.error.field");
+ System
+ .setProperty("derby.stream.error.field",
+ "org.apache.solr.handler.dataimport.AbstractDIHJdbcTestCase$DerbyUtil.DEV_NULL");
+ Class.forName("org.apache.derby.jdbc.EmbeddedDriver").newInstance();
+ if (oldProp != null) {
+ System.setProperty("derby.stream.error.field", oldProp);
+ }
+ } catch (Exception e) {
+ throw e;
+ }
+ initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
+ }
+
+ @AfterClass
+ public static void afterClassDihJdbcTest() throws Exception {
+ try {
+ DriverManager.getConnection("jdbc:derby:;shutdown=true");
+ } catch (SQLException e) {
+ // ignore...we might not even be using derby this time...
+ }
+ }
+
+ protected Database setAllowedDatabases() {
+ return Database.RANDOM;
+ }
+
+ @Before
+ public void beforeDihJdbcTest() throws Exception {
+ skipThisTest = false;
+ dbToUse = setAllowedDatabases();
+ if (dbToUse == Database.RANDOM) {
+ if (random().nextBoolean()) {
+ dbToUse = Database.DERBY;
+ } else {
+ dbToUse = Database.HSQLDB;
+ }
+ }
+
+ clearIndex();
+ assertU(commit());
+ buildDatabase();
+ }
+
+ @After
+ public void afterDihJdbcTest() throws Exception {
+ Connection conn = null;
+ Statement s = null;
+ try {
+ if (dbToUse == Database.DERBY) {
+ try {
+ conn = DriverManager
+ .getConnection("jdbc:derby:memory:derbyDB;drop=true;territory=en_US");
+ } catch (SQLException e) {
+ if (!"08006".equals(e.getSQLState())) {
+ throw e;
+ }
+ }
+ } else if (dbToUse == Database.HSQLDB) {
+ conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
+ s = conn.createStatement();
+ s.executeUpdate("shutdown");
+ }
+ } catch (SQLException e) {
+ if(!skipThisTest) {
+ throw e;
+ }
+ } finally {
+ try {
+ s.close();
+ } catch (Exception ex) {}
+ try {
+ conn.close();
+ } catch (Exception ex) {}
+ }
+ }
+
+ protected Connection newConnection() throws Exception {
+ if (dbToUse == Database.DERBY) {
+ return DriverManager.getConnection("jdbc:derby:memory:derbyDB;territory=en_US");
+ } else if (dbToUse == Database.HSQLDB) {
+ return DriverManager.getConnection("jdbc:hsqldb:mem:.");
+ }
+ throw new AssertionError("Invalid database to use: " + dbToUse);
+ }
+
+ protected void buildDatabase() throws Exception {
+ Connection conn = null;
+ try {
+ if (dbToUse == Database.DERBY) {
+ conn = DriverManager
+ .getConnection("jdbc:derby:memory:derbyDB;create=true;territory=en_US");
+ } else if (dbToUse == Database.HSQLDB) {
+ conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
+ } else {
+ throw new AssertionError("Invalid database to use: " + dbToUse);
+ }
+ populateData(conn);
+ } catch (SQLException sqe) {
+ Throwable cause = sqe;
+ while(cause.getCause()!=null) {
+ cause = cause.getCause();
+ }
+ } finally {
+ try {
+ conn.close();
+ } catch (Exception e1) {}
+ }
+ }
+
+ protected void populateData(Connection conn) throws Exception {
+ // no-op
+ }
+
+ public int totalDatabaseRequests(String dihHandlerName) throws Exception {
+ LocalSolrQueryRequest request = lrf.makeRequest("indent", "true");
+ String response = h.query(dihHandlerName, request);
+ Matcher m = totalRequestsPattern.matcher(response);
+ Assert.assertTrue("The handler " + dihHandlerName
+ + " is not reporting any database requests. ",
+ m.find() && m.groupCount() == 1);
+ return Integer.parseInt(m.group(1));
+ }
+
+ public int totalDatabaseRequests() throws Exception {
+ return totalDatabaseRequests("/dataimport");
+ }
+
+ protected LocalSolrQueryRequest generateRequest() {
+ return lrf.makeRequest("command", "full-import", "dataConfig",
+ generateConfig(), "clean", "true", "commit", "true", "synchronous",
+ "true", "indent", "true");
+ }
+
+ protected abstract String generateConfig();
+
+ public static class DerbyUtil {
+ public static final OutputStream DEV_NULL = new OutputStream() {
+ @Override
+ public void write(int b) {}
+ };
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java
new file mode 100644
index 0000000..7b8ff88
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java
@@ -0,0 +1,379 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SuppressForbidden;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.update.CommitUpdateCommand;
+import org.apache.solr.update.DeleteUpdateCommand;
+import org.apache.solr.update.MergeIndexesCommand;
+import org.apache.solr.update.RollbackUpdateCommand;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
+import org.junit.Before;
+
+/**
+ * <p>
+ * Abstract base class for DataImportHandler tests
+ * </p>
+ * <p>
+ * <b>This API is experimental and subject to change</b>
+ *
+ *
+ * @since solr 1.3
+ */
+public abstract class AbstractDataImportHandlerTestCase extends
+ SolrTestCaseJ4 {
+
+ // note, a little twisted that we shadow this static method
+ public static void initCore(String config, String schema) throws Exception {
+ File testHome = createTempDir("core-home").toFile();
+ FileUtils.copyDirectory(getFile("dih/solr"), testHome);
+ initCore(config, schema, testHome.getAbsolutePath());
+ }
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ File home = createTempDir("dih-properties").toFile();
+ System.setProperty("solr.solr.home", home.getAbsolutePath());
+ }
+
+ protected String loadDataConfig(String dataConfigFileName) {
+ try {
+ SolrCore core = h.getCore();
+ return SolrWriter.getResourceAsString(core.getResourceLoader()
+ .openResource(dataConfigFileName));
+ } catch (IOException e) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+
+ protected String runFullImport(String dataConfig) throws Exception {
+ LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import",
+ "debug", "on", "clean", "true", "commit", "true", "dataConfig",
+ dataConfig);
+ return h.query("/dataimport", request);
+ }
+
+ protected void runDeltaImport(String dataConfig) throws Exception {
+ LocalSolrQueryRequest request = lrf.makeRequest("command", "delta-import",
+ "debug", "on", "clean", "false", "commit", "true", "dataConfig",
+ dataConfig);
+ h.query("/dataimport", request);
+ }
+
+ /**
+ * Redirect {@link SimplePropertiesWriter#filename} to a temporary location
+ * and return it.
+ */
+ protected File redirectTempProperties(DataImporter di) {
+ try {
+ File tempFile = createTempFile().toFile();
+ di.getConfig().getPropertyWriter().getParameters()
+ .put(SimplePropertiesWriter.FILENAME, tempFile.getAbsolutePath());
+ return tempFile;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Runs a full-import using the given dataConfig and the provided request parameters.
+ *
+ * By default, debug=on, clean=true and commit=true are passed which can be overridden.
+ *
+ * @param dataConfig the data-config xml as a string
+ * @param extraParams any extra request parameters needed to be passed to DataImportHandler
+ * @throws Exception in case of any error
+ */
+ protected void runFullImport(String dataConfig, Map<String, String> extraParams) throws Exception {
+ HashMap<String, String> params = new HashMap<>();
+ params.put("command", "full-import");
+ params.put("debug", "on");
+ params.put("dataConfig", dataConfig);
+ params.put("clean", "true");
+ params.put("commit", "true");
+ params.putAll(extraParams);
+ NamedList l = new NamedList();
+ for (Map.Entry<String, String> e : params.entrySet()) {
+ l.add(e.getKey(),e.getValue());
+ }
+ LocalSolrQueryRequest request = new LocalSolrQueryRequest(h.getCore(), l);
+ h.query("/dataimport", request);
+ }
+
+ /**
+ * Helper for creating a Context instance. Useful for testing Transformers
+ */
+ @SuppressWarnings("unchecked")
+ public static TestContext getContext(EntityProcessorWrapper parent,
+ VariableResolver resolver, DataSource parentDataSource,
+ String currProcess, final List<Map<String, String>> entityFields,
+ final Map<String, String> entityAttrs) {
+ if (resolver == null) resolver = new VariableResolver();
+ final Context delegate = new ContextImpl(parent, resolver,
+ parentDataSource, currProcess,
+ new HashMap<>(), null, null);
+ return new TestContext(entityAttrs, delegate, entityFields, parent == null);
+ }
+
+ /**
+ * Strings at even index are keys, odd-index strings are values in the
+ * returned map
+ */
+ @SuppressWarnings("unchecked")
+ public static Map createMap(Object... args) {
+ return Utils.makeMap(args);
+ }
+
+ @SuppressForbidden(reason = "Needs currentTimeMillis to set modified time for a file")
+ public static File createFile(File tmpdir, String name, byte[] content,
+ boolean changeModifiedTime) throws IOException {
+ File file = new File(tmpdir.getAbsolutePath() + File.separator + name);
+ file.deleteOnExit();
+ FileOutputStream f = new FileOutputStream(file);
+ f.write(content);
+ f.close();
+ if (changeModifiedTime)
+ file.setLastModified(System.currentTimeMillis() - 3600000);
+ return file;
+ }
+
+ public static Map<String, String> getField(String col, String type,
+ String re, String srcCol, String splitBy) {
+ HashMap<String, String> vals = new HashMap<>();
+ vals.put("column", col);
+ vals.put("type", type);
+ vals.put("regex", re);
+ vals.put("sourceColName", srcCol);
+ vals.put("splitBy", splitBy);
+ return vals;
+ }
+
+ static class TestContext extends Context {
+ private final Map<String, String> entityAttrs;
+ private final Context delegate;
+ private final List<Map<String, String>> entityFields;
+ private final boolean root;
+ String script,scriptlang;
+
+ public TestContext(Map<String, String> entityAttrs, Context delegate,
+ List<Map<String, String>> entityFields, boolean root) {
+ this.entityAttrs = entityAttrs;
+ this.delegate = delegate;
+ this.entityFields = entityFields;
+ this.root = root;
+ }
+
+ @Override
+ public String getEntityAttribute(String name) {
+ return entityAttrs == null ? delegate.getEntityAttribute(name) : entityAttrs.get(name);
+ }
+
+ @Override
+ public String getResolvedEntityAttribute(String name) {
+ return entityAttrs == null ? delegate.getResolvedEntityAttribute(name) :
+ delegate.getVariableResolver().replaceTokens(entityAttrs.get(name));
+ }
+
+ @Override
+ public List<Map<String, String>> getAllEntityFields() {
+ return entityFields == null ? delegate.getAllEntityFields()
+ : entityFields;
+ }
+
+ @Override
+ public VariableResolver getVariableResolver() {
+ return delegate.getVariableResolver();
+ }
+
+ @Override
+ public DataSource getDataSource() {
+ return delegate.getDataSource();
+ }
+
+ @Override
+ public boolean isRootEntity() {
+ return root;
+ }
+
+ @Override
+ public String currentProcess() {
+ return delegate.currentProcess();
+ }
+
+ @Override
+ public Map<String, Object> getRequestParameters() {
+ return delegate.getRequestParameters();
+ }
+
+ @Override
+ public EntityProcessor getEntityProcessor() {
+ return null;
+ }
+
+ @Override
+ public void setSessionAttribute(String name, Object val, String scope) {
+ delegate.setSessionAttribute(name, val, scope);
+ }
+
+ @Override
+ public Object getSessionAttribute(String name, String scope) {
+ return delegate.getSessionAttribute(name, scope);
+ }
+
+ @Override
+ public Context getParentContext() {
+ return delegate.getParentContext();
+ }
+
+ @Override
+ public DataSource getDataSource(String name) {
+ return delegate.getDataSource(name);
+ }
+
+ @Override
+ public SolrCore getSolrCore() {
+ return delegate.getSolrCore();
+ }
+
+ @Override
+ public Map<String, Object> getStats() {
+ return delegate.getStats();
+ }
+
+
+ @Override
+ public String getScript() {
+ return script == null ? delegate.getScript() : script;
+ }
+
+ @Override
+ public String getScriptLanguage() {
+ return scriptlang == null ? delegate.getScriptLanguage() : scriptlang;
+ }
+
+ @Override
+ public void deleteDoc(String id) {
+
+ }
+
+ @Override
+ public void deleteDocByQuery(String query) {
+
+ }
+
+ @Override
+ public Object resolve(String var) {
+ return delegate.resolve(var);
+ }
+
+ @Override
+ public String replaceTokens(String template) {
+ return delegate.replaceTokens(template);
+ }
+ }
+
+ public static class TestUpdateRequestProcessorFactory extends UpdateRequestProcessorFactory {
+
+ @Override
+ public UpdateRequestProcessor getInstance(SolrQueryRequest req,
+ SolrQueryResponse rsp, UpdateRequestProcessor next) {
+ return new TestUpdateRequestProcessor(next);
+ }
+
+ }
+
+ public static class TestUpdateRequestProcessor extends UpdateRequestProcessor {
+
+ public static boolean finishCalled = false;
+ public static boolean processAddCalled = false;
+ public static boolean processCommitCalled = false;
+ public static boolean processDeleteCalled = false;
+ public static boolean mergeIndexesCalled = false;
+ public static boolean rollbackCalled = false;
+
+ public static void reset() {
+ finishCalled = false;
+ processAddCalled = false;
+ processCommitCalled = false;
+ processDeleteCalled = false;
+ mergeIndexesCalled = false;
+ rollbackCalled = false;
+ }
+
+ public TestUpdateRequestProcessor(UpdateRequestProcessor next) {
+ super(next);
+ reset();
+ }
+
+ @Override
+ public void finish() throws IOException {
+ finishCalled = true;
+ super.finish();
+ }
+
+ @Override
+ public void processAdd(AddUpdateCommand cmd) throws IOException {
+ processAddCalled = true;
+ super.processAdd(cmd);
+ }
+
+ @Override
+ public void processCommit(CommitUpdateCommand cmd) throws IOException {
+ processCommitCalled = true;
+ super.processCommit(cmd);
+ }
+
+ @Override
+ public void processDelete(DeleteUpdateCommand cmd) throws IOException {
+ processDeleteCalled = true;
+ super.processDelete(cmd);
+ }
+
+ @Override
+ public void processMergeIndexes(MergeIndexesCommand cmd) throws IOException {
+ mergeIndexesCalled = true;
+ super.processMergeIndexes(cmd);
+ }
+
+ @Override
+ public void processRollback(RollbackUpdateCommand cmd) throws IOException {
+ rollbackCalled = true;
+ super.processRollback(cmd);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java
new file mode 100644
index 0000000..2774044
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java
@@ -0,0 +1,845 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import junit.framework.Assert;
+import org.apache.solr.common.util.SuppressForbidden;
+import org.junit.After;
+import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Files;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Timestamp;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+public abstract class AbstractSqlEntityProcessorTestCase extends
+ AbstractDIHJdbcTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ protected boolean underlyingDataModified;
+ protected boolean useSimpleCaches;
+ protected boolean countryEntity;
+ protected boolean countryCached;
+ protected boolean countryZipper;
+ protected boolean sportsEntity;
+ protected boolean sportsCached;
+ protected boolean sportsZipper;
+
+ protected boolean wrongPeopleOrder ;
+ protected boolean wrongSportsOrder ;
+ protected boolean wrongCountryOrder;
+
+ protected String rootTransformerName;
+ protected boolean countryTransformer;
+ protected boolean sportsTransformer;
+ protected String fileLocation;
+ protected String fileName;
+
+ @Before
+ public void beforeSqlEntitiyProcessorTestCase() throws Exception {
+ File tmpdir = createTempDir().toFile();
+ fileLocation = tmpdir.getPath();
+ fileName = "the.properties";
+ }
+
+ @After
+ public void afterSqlEntitiyProcessorTestCase() throws Exception {
+ useSimpleCaches = false;
+ countryEntity = false;
+ countryCached = false;
+ countryZipper = false;
+ sportsEntity = false;
+ sportsCached = false;
+ sportsZipper = false;
+
+ wrongPeopleOrder = false;
+ wrongSportsOrder = false;
+ wrongCountryOrder= false;
+
+ rootTransformerName = null;
+ countryTransformer = false;
+ sportsTransformer = false;
+ underlyingDataModified = false;
+
+ //If an Assume was tripped while setting up the test,
+ //the file might not ever have been created...
+ if(fileLocation!=null) {
+ Files.deleteIfExists(new File(fileLocation + File.separatorChar + fileName).toPath());
+ Files.deleteIfExists(new File(fileLocation).toPath());
+ }
+ }
+
+ protected void logPropertiesFile() {
+ Map<String,String> init = new HashMap<>();
+ init.put("filename", fileName);
+ init.put("directory", fileLocation);
+ SimplePropertiesWriter spw = new SimplePropertiesWriter();
+ spw.init(new DataImporter(), init);
+ Map<String,Object> props = spw.readIndexerProperties();
+ if(props!=null) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\ndataimporter.properties: \n");
+ for(Map.Entry<String,Object> entry : props.entrySet()) {
+ sb.append(" > key=" + entry.getKey() + " / value=" + entry.getValue() + "\n");
+ }
+ log.debug(sb.toString());
+ }
+ }
+
+ protected abstract String deltaQueriesCountryTable();
+
+ protected abstract String deltaQueriesPersonTable();
+
+ protected void singleEntity(int numToExpect) throws Exception {
+ h.query("/dataimport", generateRequest());
+ assertQ("There should be 1 document per person in the database: "
+ + totalPeople(), req("*:*"), "//*[@numFound='" + totalPeople() + "']");
+ Assert.assertTrue("Expecting " + numToExpect
+ + " database calls, but DIH reported " + totalDatabaseRequests(),
+ totalDatabaseRequests() == numToExpect);
+ }
+
+ protected void simpleTransform(int numToExpect) throws Exception {
+ rootTransformerName = "AddAColumnTransformer";
+ h.query("/dataimport", generateRequest());
+ assertQ(
+ "There should be 1 document with a transformer-added column per person is the database: "
+ + totalPeople(), req("AddAColumn_s:Added"), "//*[@numFound='"
+ + totalPeople() + "']");
+ Assert.assertTrue("Expecting " + numToExpect
+ + " database calls, but DIH reported " + totalDatabaseRequests(),
+ totalDatabaseRequests() == numToExpect);
+ }
+
+ /**
+ * A delta update will not clean up documents added by a transformer even if
+ * the parent document that the transformer used to base the new documents
+ * were deleted
+ */
+ protected void complexTransform(int numToExpect, int numDeleted)
+ throws Exception {
+ rootTransformerName = "TripleThreatTransformer";
+ h.query("/dataimport", generateRequest());
+ int totalDocs = ((totalPeople() * 3) + (numDeleted * 2));
+ int totalAddedDocs = (totalPeople() + numDeleted);
+ assertQ(
+ req("q", "*:*", "rows", "" + (totalPeople() * 3), "sort", "id asc"),
+ "//*[@numFound='" + totalDocs + "']");
+ assertQ(req("id:TripleThreat-1-*"), "//*[@numFound='" + totalAddedDocs
+ + "']");
+ assertQ(req("id:TripleThreat-2-*"), "//*[@numFound='" + totalAddedDocs
+ + "']");
+ if (personNameExists("Michael") && countryCodeExists("NR")) {
+ assertQ(
+ "Michael and NR are assured to be in the database. Therefore the transformer should have added leahciM and RN on the same document as id:TripleThreat-1-3",
+ req("+id:TripleThreat-1-3 +NAME_mult_s:Michael +NAME_mult_s:leahciM +COUNTRY_CODES_mult_s:NR +COUNTRY_CODES_mult_s:RN"),
+ "//*[@numFound='1']");
+ }
+ assertQ(req("AddAColumn_s:Added"), "//*[@numFound='" + totalAddedDocs
+ + "']");
+ Assert.assertTrue("Expecting " + numToExpect
+ + " database calls, but DIH reported " + totalDatabaseRequests(),
+ totalDatabaseRequests() == numToExpect);
+ }
+
+ protected void withChildEntities(boolean cached, boolean checkDatabaseRequests)
+ throws Exception {
+ rootTransformerName = random().nextBoolean() ? null
+ : "AddAColumnTransformer";
+ int numChildren = random().nextInt(1) + 1;
+ int numDatabaseRequests = 1;
+ if (underlyingDataModified) {
+ if (countryEntity) {
+ if (cached) {
+ numDatabaseRequests++;
+ } else {
+ numDatabaseRequests += totalPeople();
+ }
+ }
+ if (sportsEntity) {
+ if (cached) {
+ numDatabaseRequests++;
+ } else {
+ numDatabaseRequests += totalPeople();
+ }
+ }
+ } else {
+ countryEntity = true;
+ sportsEntity = true;
+ if(countryZipper||sportsZipper){// zipper tests fully cover nums of children
+ countryEntity = countryZipper;
+ sportsEntity = sportsZipper;
+ }else{// apply default randomization on cached cases
+ if (numChildren == 1) {
+ countryEntity = random().nextBoolean();
+ sportsEntity = !countryEntity;
+ }
+ }
+ if (countryEntity) {
+ countryTransformer = random().nextBoolean();
+ if (cached) {
+ numDatabaseRequests++;
+ countryCached = true;
+ } else {
+ numDatabaseRequests += totalPeople();
+ }
+ }
+ if (sportsEntity) {
+ sportsTransformer = random().nextBoolean();
+ if (cached) {
+ numDatabaseRequests++;
+ sportsCached = true;
+ } else {
+ numDatabaseRequests += totalPeople();
+ }
+ }
+ }
+ h.query("/dataimport", generateRequest());
+
+ assertQ("There should be 1 document per person in the database: "
+ + totalPeople(), req("*:*"), "//*[@numFound='" + (totalPeople()) + "']");
+ if (!underlyingDataModified
+ && "AddAColumnTransformer".equals(rootTransformerName)) {
+ assertQ(
+ "There should be 1 document with a transformer-added column per person is the database: "
+ + totalPeople(), req("AddAColumn_s:Added"), "//*[@numFound='"
+ + (totalPeople()) + "']");
+ }
+ if (countryEntity) {
+ {
+ String[] people = getStringsFromQuery("SELECT NAME FROM PEOPLE WHERE DELETED != 'Y'");
+ String man = people[random().nextInt(people.length)];
+ String[] countryNames = getStringsFromQuery("SELECT C.COUNTRY_NAME FROM PEOPLE P "
+ + "INNER JOIN COUNTRIES C ON P.COUNTRY_CODE=C.CODE "
+ + "WHERE P.DELETED!='Y' AND C.DELETED!='Y' AND P.NAME='" + man + "'");
+
+ assertQ(req("{!term f=NAME_mult_s}"+ man), "//*[@numFound='1']",
+ countryNames.length>0?
+ "//doc/str[@name='COUNTRY_NAME_s']='" + countryNames[random().nextInt(countryNames.length)] + "'"
+ :"//doc[count(*[@name='COUNTRY_NAME_s'])=0]");
+ }
+ {
+ String[] countryCodes = getStringsFromQuery("SELECT CODE FROM COUNTRIES WHERE DELETED != 'Y'");
+ String theCode = countryCodes[random().nextInt(countryCodes.length)];
+ int num = numberPeopleByCountryCode(theCode);
+ if(num>0){
+ String nrName = countryNameByCode(theCode);
+ assertQ(req("COUNTRY_CODES_mult_s:"+theCode), "//*[@numFound='" + num + "']",
+ "//doc/str[@name='COUNTRY_NAME_s']='" + nrName + "'");
+ }else{ // no one lives there anyway
+ assertQ(req("COUNTRY_CODES_mult_s:"+theCode), "//*[@numFound='" + num + "']");
+ }
+ }
+ if (countryTransformer && !underlyingDataModified) {
+ assertQ(req("countryAdded_s:country_added"), "//*[@numFound='"
+ + totalPeople() + "']");
+ }
+ }
+ if (sportsEntity) {
+ if (!underlyingDataModified) {
+ assertQ(req("SPORT_NAME_mult_s:Sailing"), "//*[@numFound='2']");
+ }
+ String [] names = getStringsFromQuery("SELECT NAME FROM PEOPLE WHERE DELETED != 'Y'");
+ String name = names[random().nextInt(names.length)];
+ int personId = getIntFromQuery("SELECT ID FROM PEOPLE WHERE DELETED != 'Y' AND NAME='"+name+"'");
+ String[] michaelsSports = sportNamesByPersonId(personId);
+
+ String[] xpath = new String[michaelsSports.length + 1];
+ xpath[0] = "//*[@numFound='1']";
+ int i = 1;
+ for (String ms : michaelsSports) {
+ xpath[i] = "//doc/arr[@name='SPORT_NAME_mult_s']/str='"//[" + i + "]='" don't care about particular order
+ + ms + "'";
+ i++;
+ }
+ assertQ(req("NAME_mult_s:" + name.replaceAll("\\W", "\\\\$0")),
+ xpath);
+ if (!underlyingDataModified && sportsTransformer) {
+ assertQ(req("sportsAdded_s:sport_added"), "//*[@numFound='"
+ + (totalSportsmen()) + "']");
+ }
+ assertQ("checking orphan sport is absent",
+ req("{!term f=SPORT_NAME_mult_s}No Fishing"), "//*[@numFound='0']");
+ }
+ if (checkDatabaseRequests) {
+ Assert.assertTrue("Expecting " + numDatabaseRequests
+ + " database calls, but DIH reported " + totalDatabaseRequests(),
+ totalDatabaseRequests() == numDatabaseRequests);
+ }
+ }
+
+ protected void simpleCacheChildEntities(boolean checkDatabaseRequests)
+ throws Exception {
+ useSimpleCaches = true;
+ countryEntity = true;
+ sportsEntity = true;
+ countryCached = true;
+ sportsCached = true;
+ int dbRequestsMoreThan = 3;
+ int dbRequestsLessThan = totalPeople() * 2 + 1;
+ h.query("/dataimport", generateRequest());
+ assertQ(req("*:*"), "//*[@numFound='" + (totalPeople()) + "']");
+ if (!underlyingDataModified
+ || (personNameExists("Samantha") && "Nauru"
+ .equals(countryNameByCode("NR")))) {
+ assertQ(req("NAME_mult_s:Samantha"), "//*[@numFound='1']",
+ "//doc/str[@name='COUNTRY_NAME_s']='Nauru'");
+ }
+ if (!underlyingDataModified) {
+ assertQ(req("COUNTRY_CODES_mult_s:NR"), "//*[@numFound='2']",
+ "//doc/str[@name='COUNTRY_NAME_s']='Nauru'");
+ assertQ(req("SPORT_NAME_mult_s:Sailing"), "//*[@numFound='2']");
+ }
+ String[] michaelsSports = sportNamesByPersonId(3);
+ if (!underlyingDataModified || michaelsSports.length > 0) {
+ String[] xpath = new String[michaelsSports.length + 1];
+ xpath[0] = "//*[@numFound='1']";
+ int i = 1;
+ for (String ms : michaelsSports) {
+ xpath[i] = "//doc/arr[@name='SPORT_NAME_mult_s']/str[" + i + "]='" + ms
+ + "'";
+ i++;
+ }
+ assertQ(req("NAME_mult_s:Michael"), xpath);
+ }
+ if (checkDatabaseRequests) {
+ Assert.assertTrue("Expecting more than " + dbRequestsMoreThan
+ + " database calls, but DIH reported " + totalDatabaseRequests(),
+ totalDatabaseRequests() > dbRequestsMoreThan);
+ Assert.assertTrue("Expecting fewer than " + dbRequestsLessThan
+ + " database calls, but DIH reported " + totalDatabaseRequests(),
+ totalDatabaseRequests() < dbRequestsLessThan);
+ }
+ }
+
+
+ private int getIntFromQuery(String query) throws Exception {
+ Connection conn = null;
+ Statement s = null;
+ ResultSet rs = null;
+ try {
+ conn = newConnection();
+ s = conn.createStatement();
+ rs = s.executeQuery(query);
+ if (rs.next()) {
+ return rs.getInt(1);
+ }
+ return 0;
+ } catch (SQLException e) {
+ throw e;
+ } finally {
+ try {
+ rs.close();
+ } catch (Exception ex) {}
+ try {
+ s.close();
+ } catch (Exception ex) {}
+ try {
+ conn.close();
+ } catch (Exception ex) {}
+ }
+ }
+
+ private String[] getStringsFromQuery(String query) throws Exception {
+ Connection conn = null;
+ Statement s = null;
+ ResultSet rs = null;
+ try {
+ conn = newConnection();
+ s = conn.createStatement();
+ rs = s.executeQuery(query);
+ List<String> results = new ArrayList<>();
+ while (rs.next()) {
+ results.add(rs.getString(1));
+ }
+ return results.toArray(new String[results.size()]);
+ } catch (SQLException e) {
+ throw e;
+ } finally {
+ try {
+ rs.close();
+ } catch (Exception ex) {}
+ try {
+ s.close();
+ } catch (Exception ex) {}
+ try {
+ conn.close();
+ } catch (Exception ex) {}
+ }
+ }
+
+ public int totalCountries() throws Exception {
+ return getIntFromQuery("SELECT COUNT(1) FROM COUNTRIES WHERE DELETED != 'Y' ");
+ }
+
+ public int totalPeople() throws Exception {
+ return getIntFromQuery("SELECT COUNT(1) FROM PEOPLE WHERE DELETED != 'Y' ");
+ }
+
+ public int totalSportsmen() throws Exception {
+ return getIntFromQuery("SELECT COUNT(*) FROM PEOPLE WHERE "
+ + "EXISTS(SELECT ID FROM PEOPLE_SPORTS WHERE PERSON_ID=PEOPLE.ID AND PEOPLE_SPORTS.DELETED != 'Y')"
+ + " AND PEOPLE.DELETED != 'Y'");
+ }
+
+ public boolean countryCodeExists(String cc) throws Exception {
+ return getIntFromQuery("SELECT COUNT(1) country_name FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='"
+ + cc + "'") > 0;
+ }
+
+ public String countryNameByCode(String cc) throws Exception {
+ String[] s = getStringsFromQuery("SELECT country_name FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='"
+ + cc + "'");
+ return s.length == 0 ? null : s[0];
+ }
+
+ public int numberPeopleByCountryCode(String cc) throws Exception {
+ return getIntFromQuery("Select count(1) " + "from people p "
+ + "inner join countries c on p.country_code=c.code "
+ + "where p.deleted!='Y' and c.deleted!='Y' and c.code='" + cc + "'");
+ }
+
+ public String[] sportNamesByPersonId(int personId) throws Exception {
+ return getStringsFromQuery("SELECT ps.SPORT_NAME "
+ + "FROM people_sports ps "
+ + "INNER JOIN PEOPLE p ON p.id = ps.person_Id "
+ + "WHERE ps.DELETED != 'Y' AND p.DELETED != 'Y' " + "AND ps.person_id="
+ + personId + " " + "ORDER BY ps.id");
+ }
+
+ public boolean personNameExists(String pn) throws Exception {
+ return getIntFromQuery("SELECT COUNT(1) FROM PEOPLE WHERE DELETED != 'Y' AND NAME='"
+ + pn + "'") > 0;
+ }
+
+ public String personNameById(int id) throws Exception {
+ String[] nameArr = getStringsFromQuery("SELECT NAME FROM PEOPLE WHERE ID="
+ + id);
+ if (nameArr.length == 0) {
+ return null;
+ }
+ return nameArr[0];
+ }
+
+ @SuppressForbidden(reason = "Needs currentTimeMillis to set change time for SQL query")
+ public IntChanges modifySomePeople() throws Exception {
+ underlyingDataModified = true;
+ int numberToChange = random().nextInt(people.length + 1);
+ Set<Integer> changeSet = new HashSet<>();
+ Set<Integer> deleteSet = new HashSet<>();
+ Set<Integer> addSet = new HashSet<>();
+ Connection conn = null;
+ PreparedStatement change = null;
+ PreparedStatement delete = null;
+ PreparedStatement add = null;
+ // One second in the future ensures a change time after the last import (DIH
+ // uses second precision only)
+ Timestamp theTime = new Timestamp(System.currentTimeMillis() + 1000);
+ log.debug("PEOPLE UPDATE USING TIMESTAMP: "
+ + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.ROOT)
+ .format(theTime));
+ try {
+ conn = newConnection();
+ change = conn
+ .prepareStatement("update people set name=?, last_modified=? where id=?");
+ delete = conn
+ .prepareStatement("update people set deleted='Y', last_modified=? where id=?");
+ add = conn
+ .prepareStatement("insert into people (id,name,country_code,last_modified) values (?,?,'ZZ',?)");
+ for (int i = 0; i < numberToChange; i++) {
+ int tryIndex = random().nextInt(people.length);
+ Integer id = (Integer) people[tryIndex][0];
+ if (!changeSet.contains(id) && !deleteSet.contains(id)) {
+ boolean changeDontDelete = random().nextBoolean();
+ if (changeDontDelete) {
+ changeSet.add(id);
+ change.setString(1, "MODIFIED " + people[tryIndex][1]);
+ change.setTimestamp(2, theTime);
+ change.setInt(3, id);
+ Assert.assertEquals(1, change.executeUpdate());
+ } else {
+ deleteSet.add(id);
+ delete.setTimestamp(1, theTime);
+ delete.setInt(2, id);
+ Assert.assertEquals(1, delete.executeUpdate());
+ }
+ }
+ }
+ int numberToAdd = random().nextInt(3);
+ for (int i = 0; i < numberToAdd; i++) {
+ int tryIndex = random().nextInt(people.length);
+ Integer id = (Integer) people[tryIndex][0];
+ Integer newId = id + 1000;
+ String newDesc = "ADDED " + people[tryIndex][1];
+ if (!addSet.contains(newId)) {
+ addSet.add(newId);
+ add.setInt(1, newId);
+ add.setString(2, newDesc);
+ add.setTimestamp(3, theTime);
+ Assert.assertEquals(1, add.executeUpdate());
+ }
+ }
+ conn.commit();
+ } catch (SQLException e) {
+ throw e;
+ } finally {
+ try {
+ change.close();
+ } catch (Exception ex) {}
+ try {
+ conn.close();
+ } catch (Exception ex) {}
+ }
+ IntChanges c = new IntChanges();
+ c.changedKeys = changeSet.toArray(new Integer[changeSet.size()]);
+ c.deletedKeys = deleteSet.toArray(new Integer[deleteSet.size()]);
+ c.addedKeys = addSet.toArray(new Integer[addSet.size()]);
+ return c;
+ }
+
+ @SuppressForbidden(reason = "Needs currentTimeMillis to set change time for SQL query")
+ public String[] modifySomeCountries() throws Exception {
+ underlyingDataModified = true;
+ int numberToChange = random().nextInt(countries.length + 1);
+ Set<String> changeSet = new HashSet<>();
+ Connection conn = null;
+ PreparedStatement change = null;
+ // One second in the future ensures a change time after the last import (DIH
+ // uses second precision only)
+ Timestamp theTime = new Timestamp(System.currentTimeMillis() + 1000);
+ log.debug("COUNTRY UPDATE USING TIMESTAMP: "
+ + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.ROOT)
+ .format(theTime));
+ try {
+ conn = newConnection();
+ change = conn
+ .prepareStatement("update countries set country_name=?, last_modified=? where code=?");
+ for (int i = 0; i < numberToChange; i++) {
+ int tryIndex = random().nextInt(countries.length);
+ String code = countries[tryIndex][0];
+ if (!changeSet.contains(code)) {
+ changeSet.add(code);
+ change.setString(1, "MODIFIED " + countries[tryIndex][1]);
+ change.setTimestamp(2, theTime);
+ change.setString(3, code);
+ Assert.assertEquals(1, change.executeUpdate());
+
+ }
+ }
+ } catch (SQLException e) {
+ throw e;
+ } finally {
+ try {
+ change.close();
+ } catch (Exception ex) {}
+ try {
+ conn.close();
+ } catch (Exception ex) {}
+ }
+ return changeSet.toArray(new String[changeSet.size()]);
+ }
+
+ static class IntChanges {
+ public Integer[] changedKeys;
+ public Integer[] deletedKeys;
+ public Integer[] addedKeys;
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ if(changedKeys!=null) {
+ sb.append("changes: ");
+ for(int i : changedKeys) {
+ sb.append(i).append(" ");
+ }
+ }
+ if(deletedKeys!=null) {
+ sb.append("deletes: ");
+ for(int i : deletedKeys) {
+ sb.append(i).append(" ");
+ }
+ }
+ if(addedKeys!=null) {
+ sb.append("adds: ");
+ for(int i : addedKeys) {
+ sb.append(i).append(" ");
+ }
+ }
+ return sb.toString();
+ }
+ }
+
+ @Override
+ protected String generateConfig() {
+ String ds = null;
+ if (dbToUse == Database.DERBY) {
+ ds = "derby";
+ } else if (dbToUse == Database.HSQLDB) {
+ ds = "hsqldb";
+ } else {
+ throw new AssertionError("Invalid database to use: " + dbToUse);
+ }
+ StringBuilder sb = new StringBuilder();
+ sb.append("\n<dataConfig> \n");
+ sb.append("<propertyWriter type=''SimplePropertiesWriter'' directory=''" + fileLocation + "'' filename=''" + fileName + "'' />\n");
+ sb.append("<dataSource name=''hsqldb'' driver=''org.hsqldb.jdbcDriver'' url=''jdbc:hsqldb:mem:.'' /> \n");
+ sb.append("<dataSource name=''derby'' driver=''org.apache.derby.jdbc.EmbeddedDriver'' url=''jdbc:derby:memory:derbyDB;territory=en_US'' /> \n");
+ sb.append("<document name=''TestSqlEntityProcessor''> \n");
+ sb.append("<entity name=''People'' ");
+ sb.append("pk=''" + (random().nextBoolean() ? "ID" : "People.ID") + "'' ");
+ sb.append("processor=''SqlEntityProcessor'' ");
+ sb.append("dataSource=''" + ds + "'' ");
+ sb.append(rootTransformerName != null ? "transformer=''"
+ + rootTransformerName + "'' " : "");
+
+ sb.append("query=''SELECT ID, NAME, COUNTRY_CODE FROM PEOPLE WHERE DELETED != 'Y' "
+ +((sportsZipper||countryZipper?"ORDER BY ID":"")
+ +(wrongPeopleOrder? " DESC":""))+"'' ");
+
+ sb.append(deltaQueriesPersonTable());
+ sb.append("> \n");
+
+ sb.append("<field column=''NAME'' name=''NAME_mult_s'' /> \n");
+ sb.append("<field column=''COUNTRY_CODE'' name=''COUNTRY_CODES_mult_s'' /> \n");
+
+ if (countryEntity) {
+ sb.append("<entity name=''Countries'' ");
+ sb.append("pk=''" + (random().nextBoolean() ? "CODE" : "Countries.CODE")
+ + "'' ");
+ sb.append("dataSource=''" + ds + "'' ");
+ sb.append(countryTransformer ? "transformer=''AddAColumnTransformer'' "
+ + "newColumnName=''countryAdded_s'' newColumnValue=''country_added'' "
+ : "");
+ if (countryCached) {
+ sb.append("processor=''SqlEntityProcessor'' cacheImpl=''SortedMapBackedCache'' ");
+ if (useSimpleCaches) {
+ sb.append("query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='${People.COUNTRY_CODE}' ''>\n");
+ } else {
+
+ if(countryZipper){// really odd join btw. it sends duped countries
+ sb.append(random().nextBoolean() ? "cacheKey=''ID'' cacheLookup=''People.ID'' "
+ : "where=''ID=People.ID'' ");
+ sb.append("join=''zipper'' query=''SELECT PEOPLE.ID, CODE, COUNTRY_NAME FROM COUNTRIES"
+ + " JOIN PEOPLE ON COUNTRIES.CODE=PEOPLE.COUNTRY_CODE "
+ + "WHERE PEOPLE.DELETED != 'Y' ORDER BY PEOPLE.ID "+
+ (wrongCountryOrder ? " DESC":"")
+ + "'' ");
+ }else{
+ sb.append(random().nextBoolean() ? "cacheKey=''CODE'' cacheLookup=''People.COUNTRY_CODE'' "
+ : "where=''CODE=People.COUNTRY_CODE'' ");
+ sb.append("query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES'' ");
+ }
+ sb.append("> \n");
+ }
+ } else {
+ sb.append("processor=''SqlEntityProcessor'' query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='${People.COUNTRY_CODE}' '' ");
+ sb.append(deltaQueriesCountryTable());
+ sb.append("> \n");
+ }
+ sb.append("<field column=''CODE'' name=''COUNTRY_CODE_s'' /> \n");
+ sb.append("<field column=''COUNTRY_NAME'' name=''COUNTRY_NAME_s'' /> \n");
+ sb.append("</entity> \n");
+ }
+ if (sportsEntity) {
+ sb.append("<entity name=''Sports'' ");
+ sb.append("dataSource=''" + ds + "'' ");
+ sb.append(sportsTransformer ? "transformer=''AddAColumnTransformer'' "
+ + "newColumnName=''sportsAdded_s'' newColumnValue=''sport_added'' "
+ : "");
+ if (sportsCached) {
+ sb.append("processor=''SqlEntityProcessor'' cacheImpl=''SortedMapBackedCache'' ");
+ if (useSimpleCaches) {
+ sb.append("query=''SELECT ID, SPORT_NAME FROM PEOPLE_SPORTS WHERE DELETED != 'Y' AND PERSON_ID=${People.ID} ORDER BY ID'' ");
+ } else {
+ sb.append(random().nextBoolean() ? "cacheKey=''PERSON_ID'' cacheLookup=''People.ID'' "
+ : "where=''PERSON_ID=People.ID'' ");
+ if(sportsZipper){
+ sb.append("join=''zipper'' query=''SELECT ID, PERSON_ID, SPORT_NAME FROM PEOPLE_SPORTS ORDER BY PERSON_ID"
+ + (wrongSportsOrder?" DESC" : "")+
+ "'' ");
+ }
+ else{
+ sb.append("query=''SELECT ID, PERSON_ID, SPORT_NAME FROM PEOPLE_SPORTS ORDER BY ID'' ");
+ }
+ }
+ } else {
+ sb.append("processor=''SqlEntityProcessor'' query=''SELECT ID, SPORT_NAME FROM PEOPLE_SPORTS WHERE DELETED != 'Y' AND PERSON_ID=${People.ID} ORDER BY ID'' ");
+ }
+ sb.append("> \n");
+ sb.append("<field column=''SPORT_NAME'' name=''SPORT_NAME_mult_s'' /> \n");
+ sb.append("<field column=''id'' name=''SPORT_ID_mult_s'' /> \n");
+ sb.append("</entity> \n");
+ }
+
+ sb.append("</entity> \n");
+ sb.append("</document> \n");
+ sb.append("</dataConfig> \n");
+ String config = sb.toString().replaceAll("[']{2}", "\"");
+ log.debug(config);
+ return config;
+ }
+
+ @SuppressForbidden(reason = "Needs currentTimeMillis to set change time for SQL query")
+ @Override
+ protected void populateData(Connection conn) throws Exception {
+ Statement s = null;
+ PreparedStatement ps = null;
+ Timestamp theTime = new Timestamp(System.currentTimeMillis() - 10000); // 10 seconds ago
+ try {
+ s = conn.createStatement();
+ s.executeUpdate("create table countries(code varchar(3) not null primary key, country_name varchar(50), deleted char(1) default 'N', last_modified timestamp not null)");
+ s.executeUpdate("create table people(id int not null primary key, name varchar(50), country_code char(2), deleted char(1) default 'N', last_modified timestamp not null)");
+ s.executeUpdate("create table people_sports(id int not null primary key, person_id int, sport_name varchar(50), deleted char(1) default 'N', last_modified timestamp not null)");
+ log.debug("INSERTING DB DATA USING TIMESTAMP: "
+ + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.ROOT)
+ .format(theTime));
+ ps = conn
+ .prepareStatement("insert into countries (code, country_name, last_modified) values (?,?,?)");
+ for (String[] country : countries) {
+ ps.setString(1, country[0]);
+ ps.setString(2, country[1]);
+ ps.setTimestamp(3, theTime);
+ Assert.assertEquals(1, ps.executeUpdate());
+ }
+ ps.close();
+
+ ps = conn
+ .prepareStatement("insert into people (id, name, country_code, last_modified) values (?,?,?,?)");
+ for (Object[] person : people) {
+ ps.setInt(1, (Integer) person[0]);
+ ps.setString(2, (String) person[1]);
+ ps.setString(3, (String) person[2]);
+ ps.setTimestamp(4, theTime);
+ Assert.assertEquals(1, ps.executeUpdate());
+ }
+ ps.close();
+
+ ps = conn
+ .prepareStatement("insert into people_sports (id, person_id, sport_name, last_modified) values (?,?,?,?)");
+ for (Object[] sport : people_sports) {
+ ps.setInt(1, (Integer) sport[0]);
+ ps.setInt(2, (Integer) sport[1]);
+ ps.setString(3, (String) sport[2]);
+ ps.setTimestamp(4, theTime);
+ Assert.assertEquals(1, ps.executeUpdate());
+ }
+ ps.close();
+ conn.commit();
+ conn.close();
+ } catch (Exception e) {
+ throw e;
+ } finally {
+ try {
+ ps.close();
+ } catch (Exception ex) {}
+ try {
+ s.close();
+ } catch (Exception ex) {}
+ try {
+ conn.close();
+ } catch (Exception ex) {}
+ }
+ }
+ public static final String[][] countries = {
+ {"NA", "Namibia"},
+ {"NC", "New Caledonia"},
+ {"NE", "Niger"},
+ {"NF", "Norfolk Island"},
+ {"NG", "Nigeria"},
+ {"NI", "Nicaragua"},
+ {"NL", "Netherlands"},
+ {"NO", "Norway"},
+ {"NP", "Nepal"},
+ {"NR", "Nauru"},
+ {"NU", "Niue"},
+ {"NZ", "New Zealand"}
+ };
+
+ public static final Object[][] people = {
+ {1,"Jacob","NZ"},
+ {2,"Ethan","NU"},
+ {3,"Michael","NR"},
+ {4,"Jayden","NP"},
+ {5,"William","NO"},
+ {6,"Alexander","NL"},
+ {7,"Noah","NI"},
+ {8,"Daniel","NG"},
+ {9,"Aiden","NF"},
+
+ {21,"Anthony","NE"}, // there is no ID=10 anymore
+
+ {11,"Emma","NL"},
+ {12,"Grace","NI"},
+ {13,"Hailey","NG"},
+ {14,"Isabella","NF"},
+ {15,"Lily","NE"},
+ {16,"Madison","NC"},
+ {17,"Mia","NA"},
+ {18,"Natalie","NZ"},
+ {19,"Olivia","NU"},
+ {20,"Samantha","NR"}
+ };
+
+ public static final Object[][] people_sports = {
+ {100, 1, "Swimming"},
+ {200, 2, "Triathlon"},
+ {300, 3, "Water polo"},
+ {310, 3, "Underwater rugby"},
+ {320, 3, "Kayaking"},
+ {400, 4, "Snorkeling"},
+ {500, 5, "Synchronized diving"},
+ {600, 6, "Underwater rugby"},
+ {700, 7, "Boating"},
+ {800, 8, "Bodyboarding"},
+ {900, 9, "Canoeing"},
+
+ {1000, 10, "No Fishing"}, // orhpaned sport
+ //
+
+ {1100, 11, "Jet Ski"},
+ {1110, 11, "Rowing"},
+ {1120, 11, "Sailing"},
+ {1200, 12, "Kayaking"},
+ {1210, 12, "Canoeing"},
+ {1300, 13, "Kite surfing"},
+ {1400, 14, "Parasailing"},
+ {1500, 15, "Rafting"},
+ //{1600, 16, "Rowing"}, Madison has no sport
+ {1700, 17, "Sailing"},
+ {1800, 18, "White Water Rafting"},
+ {1900, 19, "Water skiing"},
+ {2000, 20, "Windsurfing"},
+ {2100, 21, "Concrete diving"},
+ {2110, 21, "Bubble rugby"}
+ };
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AddAColumnTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AddAColumnTransformer.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AddAColumnTransformer.java
new file mode 100644
index 0000000..5e665d4
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AddAColumnTransformer.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.Map;
+
+public class AddAColumnTransformer extends Transformer {
+ @Override
+ public Object transformRow(Map<String,Object> aRow, Context context) {
+ String colName = context.getEntityAttribute("newColumnName");
+ colName = colName==null ? "AddAColumn_s" : colName;
+ String colValue = context.getEntityAttribute("newColumnValue");
+ colValue = colValue==null ? "Added" : colValue;
+ aRow.put(colName, colValue);
+ return aRow;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/DestroyCountCache.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/DestroyCountCache.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/DestroyCountCache.java
new file mode 100644
index 0000000..d14f43e
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/DestroyCountCache.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static org.hamcrest.CoreMatchers.nullValue;
+
+import java.util.IdentityHashMap;
+import java.util.Map;
+
+import org.junit.Assert;
+
+public class DestroyCountCache extends SortedMapBackedCache {
+ static Map<DIHCache,DIHCache> destroyed = new IdentityHashMap<>();
+
+ @Override
+ public void destroy() {
+ super.destroy();
+ Assert.assertThat(destroyed.put(this, this), nullValue());
+ }
+
+ public DestroyCountCache() {}
+
+}
\ No newline at end of file
[18/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java
deleted file mode 100644
index 13daf49..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java
+++ /dev/null
@@ -1,669 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.common.util.XMLErrorLogger;
-import org.apache.solr.common.EmptyEntityResolver;
-import javax.xml.stream.XMLInputFactory;
-import static javax.xml.stream.XMLStreamConstants.*;
-import javax.xml.stream.XMLStreamException;
-import javax.xml.stream.XMLStreamReader;
-import java.io.IOException;
-import java.io.Reader;
-import java.lang.invoke.MethodHandles;
-import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * <p>
- * A streaming xpath parser which uses StAX for XML parsing. It supports only
- * a subset of xpath syntax.
- * </p><pre>
- * /a/b/subject[@qualifier='fullTitle']
- * /a/b/subject[@qualifier=]/subtag
- * /a/b/subject/@qualifier
- * //a
- * //a/b...
- * /a//b
- * /a//b...
- * /a/b/c
- * </pre>
- * A record is a Map<String,Object> . The key is the provided name
- * and the value is a String or a List<String>
- *
- * This class is thread-safe for parsing xml. But adding fields is not
- * thread-safe. The recommended usage is to addField() in one thread and
- * then share the instance across threads.
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 1.3
- */
-public class XPathRecordReader {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private static final XMLErrorLogger XMLLOG = new XMLErrorLogger(log);
-
- private Node rootNode = new Node("/", null);
-
- /**
- * The FLATTEN flag indicates that all text and cdata under a specific
- * tag should be recursivly fetched and appended to the current Node's
- * value.
- */
- public static final int FLATTEN = 1;
-
- /**
- * A constructor called with a '|' separated list of Xpath expressions
- * which define sub sections of the XML stream that are to be emitted as
- * separate records.
- *
- * @param forEachXpath The XPATH for which a record is emitted. Once the
- * xpath tag is encountered, the Node.parse method starts collecting wanted
- * fields and at the close of the tag, a record is emitted containing all
- * fields collected since the tag start. Once
- * emitted the collected fields are cleared. Any fields collected in the
- * parent tag or above will also be included in the record, but these are
- * not cleared after emitting the record.
- *
- * It uses the ' | ' syntax of XPATH to pass in multiple xpaths.
- */
- public XPathRecordReader(String forEachXpath) {
- String[] splits = forEachXpath.split("\\|");
- for (String split : splits) {
- split = split.trim();
- if (split.startsWith("//"))
- throw new RuntimeException("forEach cannot start with '//': " + split);
- if (split.length() == 0)
- continue;
- // The created Node has a name set to the full forEach attribute xpath
- addField0(split, split, false, true, 0);
- }
- }
-
- /**
- * A wrapper around <code>addField0</code> to create a series of
- * Nodes based on the supplied Xpath and a given fieldName. The created
- * nodes are inserted into a Node tree.
- *
- * @param name The name for this field in the emitted record
- * @param xpath The xpath expression for this field
- * @param multiValued If 'true' then the emitted record will have values in
- * a List<String>
- */
- public synchronized XPathRecordReader addField(String name, String xpath, boolean multiValued) {
- addField0(xpath, name, multiValued, false, 0);
- return this;
- }
-
- /**
- * A wrapper around <code>addField0</code> to create a series of
- * Nodes based on the supplied Xpath and a given fieldName. The created
- * nodes are inserted into a Node tree.
- *
- * @param name The name for this field in the emitted record
- * @param xpath The xpath expression for this field
- * @param multiValued If 'true' then the emitted record will have values in
- * a List<String>
- * @param flags FLATTEN: Recursively combine text from all child XML elements
- */
- public synchronized XPathRecordReader addField(String name, String xpath, boolean multiValued, int flags) {
- addField0(xpath, name, multiValued, false, flags);
- return this;
- }
-
- /**
- * Splits the XPATH into a List of xpath segments and calls build() to
- * construct a tree of Nodes representing xpath segments. The resulting
- * tree structure ends up describing all the Xpaths we are interested in.
- *
- * @param xpath The xpath expression for this field
- * @param name The name for this field in the emitted record
- * @param multiValued If 'true' then the emitted record will have values in
- * a List<String>
- * @param isRecord Flags that this XPATH is from a forEach statement
- * @param flags The only supported flag is 'FLATTEN'
- */
- private void addField0(String xpath, String name, boolean multiValued,
- boolean isRecord, int flags) {
- if (!xpath.startsWith("/"))
- throw new RuntimeException("xpath must start with '/' : " + xpath);
- List<String> paths = splitEscapeQuote(xpath);
- // deal with how split behaves when separator starts a string!
- if ("".equals(paths.get(0).trim()))
- paths.remove(0);
- rootNode.build(paths, name, multiValued, isRecord, flags);
- rootNode.buildOptimise(null);
- }
-
- /**
- * Uses {@link #streamRecords streamRecords} to parse the XML source but with
- * a handler that collects all the emitted records into a single List which
- * is returned upon completion.
- *
- * @param r the stream reader
- * @return results a List of emitted records
- */
- public List<Map<String, Object>> getAllRecords(Reader r) {
- final List<Map<String, Object>> results = new ArrayList<>();
- streamRecords(r, (record, s) -> results.add(record));
- return results;
- }
-
- /**
- * Creates an XML stream reader on top of whatever reader has been
- * configured. Then calls parse() with a handler which is
- * invoked forEach record emitted.
- *
- * @param r the stream reader
- * @param handler The callback instance
- */
- public void streamRecords(Reader r, Handler handler) {
- try {
- XMLStreamReader parser = factory.createXMLStreamReader(r);
- rootNode.parse(parser, handler, new HashMap<>(),
- new Stack<>(), false);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
-
- /**
- * For each node/leaf in the Node tree there is one object of this class.
- * This tree of objects represents all the XPaths we are interested in.
- * For each Xpath segment of interest we create a node. In most cases the
- * node (branch) is rather basic , but for the final portion (leaf) of any
- * Xpath we add more information to the Node. When parsing the XML document
- * we step though this tree as we stream records from the reader. If the XML
- * document departs from this tree we skip start tags till we are back on
- * the tree.
- */
- private static class Node {
- String name; // genrally: segment of the Xpath represented by this Node
- String fieldName; // the fieldname in the emitted record (key of the map)
- String xpathName; // the segment of the Xpath represented by this Node
- String forEachPath; // the full Xpath from the forEach entity attribute
- List<Node> attributes; // List of attribute Nodes associated with this Node
- List<Node> childNodes; // List of immediate child Nodes of this node
- List<Node> wildCardNodes; // List of '//' style decendants of this Node
- List<Map.Entry<String, String>> attribAndValues;
- Node wildAncestor; // ancestor Node containing '//' style decendants
- Node parent; // parent Node in the tree
- boolean hasText=false; // flag: store/emit streamed text for this node
- boolean multiValued=false; //flag: this fields values are returned as a List
- boolean isRecord=false; //flag: this Node starts a new record
- private boolean flatten; //flag: child text is also to be emitted
-
-
- public Node(String name, Node p) {
- // Create a basic Node, suitable for the mid portions of any Xpath.
- // Node.xpathName and Node.name are set to same value.
- xpathName = this.name = name;
- parent = p;
- }
-
- public Node(String name, String fieldName, boolean multiValued) {
- // This is only called from build() when describing an attribute.
- this.name = name; // a segment from the Xpath
- this.fieldName = fieldName; // name to store collected values against
- this.multiValued = multiValued; // return collected values in a List
- }
-
- /**
- * This is the method where all the XML parsing happens. For each
- * tag/subtag read from the source, this method is called recursively.
- *
- */
- private void parse(XMLStreamReader parser,
- Handler handler,
- Map<String, Object> values,
- Stack<Set<String>> stack, // lists of values to purge
- boolean recordStarted
- ) throws IOException, XMLStreamException {
- Set<String> valuesAddedinThisFrame = null;
- if (isRecord) {
- // This Node is a match for an XPATH from a forEach attribute,
- // prepare for the clean up that will occurr when the record
- // is emitted after its END_ELEMENT is matched
- recordStarted = true;
- valuesAddedinThisFrame = new HashSet<>();
- stack.push(valuesAddedinThisFrame);
- } else if (recordStarted) {
- // This node is a child of some parent which matched against forEach
- // attribute. Continue to add values to an existing record.
- valuesAddedinThisFrame = stack.peek();
- }
-
- try {
- /* The input stream has deposited us at this Node in our tree of
- * intresting nodes. Depending on how this node is of interest,
- * process further tokens from the input stream and decide what
- * we do next
- */
- if (attributes != null) {
- // we interested in storing attributes from the input stream
- for (Node node : attributes) {
- String value = parser.getAttributeValue(null, node.name);
- if (value != null || (recordStarted && !isRecord)) {
- putText(values, value, node.fieldName, node.multiValued);
- valuesAddedinThisFrame.add(node.fieldName);
- }
- }
- }
-
- Set<Node> childrenFound = new HashSet<>();
- int event = -1;
- int flattenedStarts=0; // our tag depth when flattening elements
- StringBuilder text = new StringBuilder();
-
- while (true) {
- event = parser.next();
-
- if (event == END_ELEMENT) {
- if (flattenedStarts > 0) flattenedStarts--;
- else {
- if (hasText && valuesAddedinThisFrame != null) {
- valuesAddedinThisFrame.add(fieldName);
- putText(values, text.toString(), fieldName, multiValued);
- }
- if (isRecord) handler.handle(getDeepCopy(values), forEachPath);
- if (childNodes != null && recordStarted && !isRecord && !childrenFound.containsAll(childNodes)) {
- // nonReccord nodes where we have not collected text for ALL
- // the child nodes.
- for (Node n : childNodes) {
- // For the multivalue child nodes where we could have, but
- // didnt, collect text. Push a null string into values.
- if (!childrenFound.contains(n)) n.putNulls(values, valuesAddedinThisFrame);
- }
- }
- return;
- }
- }
- else if (hasText && (event==CDATA || event==CHARACTERS || event==SPACE)) {
- text.append(parser.getText());
- }
- else if (event == START_ELEMENT) {
- if ( flatten )
- flattenedStarts++;
- else
- handleStartElement(parser, childrenFound, handler, values, stack, recordStarted);
- }
- // END_DOCUMENT is least likely to appear and should be
- // last in if-then-else skip chain
- else if (event == END_DOCUMENT) return;
- }
- }finally {
- if ((isRecord || !recordStarted) && !stack.empty()) {
- Set<String> cleanThis = stack.pop();
- if (cleanThis != null) {
- for (String fld : cleanThis) values.remove(fld);
- }
- }
- }
- }
-
- /**
- * If a new tag is encountered, check if it is of interest or not by seeing
- * if it matches against our node tree. If we have deperted from the node
- * tree then walk back though the tree's ancestor nodes checking to see if
- * any // expressions exist for the node and compare them against the new
- * tag. If matched then "jump" to that node, otherwise ignore the tag.
- *
- * Note, the list of // expressions found while walking back up the tree
- * is chached in the HashMap decends. Then if the new tag is to be skipped,
- * any inner chil tags are compared against the cache and jumped to if
- * matched.
- */
- private void handleStartElement(XMLStreamReader parser, Set<Node> childrenFound,
- Handler handler, Map<String, Object> values,
- Stack<Set<String>> stack, boolean recordStarted)
- throws IOException, XMLStreamException {
- Node n = getMatchingNode(parser,childNodes);
- Map<String, Object> decends=new HashMap<>();
- if (n != null) {
- childrenFound.add(n);
- n.parse(parser, handler, values, stack, recordStarted);
- return;
- }
- // The stream has diverged from the tree of interesting elements, but
- // are there any wildCardNodes ... anywhere in our path from the root?
- Node dn = this; // checking our Node first!
-
- do {
- if (dn.wildCardNodes != null) {
- // Check to see if the streams tag matches one of the "//" all
- // decendents type expressions for this node.
- n = getMatchingNode(parser, dn.wildCardNodes);
- if (n != null) {
- childrenFound.add(n);
- n.parse(parser, handler, values, stack, recordStarted);
- break;
- }
- // add the list of this nodes wild decendents to the cache
- for (Node nn : dn.wildCardNodes) decends.put(nn.name, nn);
- }
- dn = dn.wildAncestor; // leap back along the tree toward root
- } while (dn != null) ;
-
- if (n == null) {
- // we have a START_ELEMENT which is not within the tree of
- // interesting nodes. Skip over the contents of this element
- // but recursivly repeat the above for any START_ELEMENTs
- // found within this element.
- int count = 1; // we have had our first START_ELEMENT
- while (count != 0) {
- int token = parser.next();
- if (token == START_ELEMENT) {
- Node nn = (Node) decends.get(parser.getLocalName());
- if (nn != null) {
- // We have a //Node which matches the stream's parser.localName
- childrenFound.add(nn);
- // Parse the contents of this stream element
- nn.parse(parser, handler, values, stack, recordStarted);
- }
- else count++;
- }
- else if (token == END_ELEMENT) count--;
- }
- }
- }
-
-
- /**
- * Check if the current tag is to be parsed or not. We step through the
- * supplied List "searchList" looking for a match. If matched, return the
- * Node object.
- */
- private Node getMatchingNode(XMLStreamReader parser,List<Node> searchL){
- if (searchL == null)
- return null;
- String localName = parser.getLocalName();
- for (Node n : searchL) {
- if (n.name.equals(localName)) {
- if (n.attribAndValues == null)
- return n;
- if (checkForAttributes(parser, n.attribAndValues))
- return n;
- }
- }
- return null;
- }
-
- private boolean checkForAttributes(XMLStreamReader parser,
- List<Map.Entry<String, String>> attrs) {
- for (Map.Entry<String, String> e : attrs) {
- String val = parser.getAttributeValue(null, e.getKey());
- if (val == null)
- return false;
- if (e.getValue() != null && !e.getValue().equals(val))
- return false;
- }
- return true;
- }
-
- /**
- * A recursive routine that walks the Node tree from a supplied start
- * pushing a null string onto every multiValued fieldName's List of values
- * where a value has not been provided from the stream.
- */
- private void putNulls(Map<String, Object> values, Set<String> valuesAddedinThisFrame) {
- if (attributes != null) {
- for (Node n : attributes) {
- if (n.multiValued) {
- putANull(n.fieldName, values, valuesAddedinThisFrame);
- }
- }
- }
- if (hasText && multiValued) {
- putANull(fieldName, values, valuesAddedinThisFrame);
- }
- if (childNodes != null) {
- for (Node childNode : childNodes) {
- childNode.putNulls(values, valuesAddedinThisFrame);
- }
- }
- }
-
- private void putANull(String thisFieldName, Map<String, Object> values, Set<String> valuesAddedinThisFrame) {
- putText(values, null, thisFieldName, true);
- if( valuesAddedinThisFrame != null) {
- valuesAddedinThisFrame.add(thisFieldName);
- }
- }
-
- /**
- * Add the field name and text into the values Map. If it is a non
- * multivalued field, then the text is simply placed in the object
- * portion of the Map. If it is a multivalued field then the text is
- * pushed onto a List which is the object portion of the Map.
- */
- @SuppressWarnings("unchecked")
- private void putText(Map<String, Object> values, String value,
- String fieldName, boolean multiValued) {
- if (multiValued) {
- List<String> v = (List<String>) values.get(fieldName);
- if (v == null) {
- v = new ArrayList<>();
- values.put(fieldName, v);
- }
- v.add(value);
- } else {
- values.put(fieldName, value);
- }
- }
-
-
- /**
- * Walk the Node tree propagating any wildDescentant information to
- * child nodes. This allows us to optimise the performance of the
- * main parse method.
- */
- private void buildOptimise(Node wa) {
- wildAncestor=wa;
- if ( wildCardNodes != null ) wa = this;
- if ( childNodes != null )
- for ( Node n : childNodes ) n.buildOptimise(wa);
- }
-
- /**
- * Build a Node tree structure representing all Xpaths of intrest to us.
- * This must be done before parsing of the XML stream starts. Each node
- * holds one portion of an Xpath. Taking each Xpath segment in turn this
- * method walks the Node tree and finds where the new segment should be
- * inserted. It creates a Node representing a field's name, XPATH and
- * some flags and inserts the Node into the Node tree.
- */
- private void build(
- List<String> paths, // a List of segments from the split xpaths
- String fieldName, // the fieldName assoc with this Xpath
- boolean multiValued, // flag if this fieldName is multiValued or not
- boolean record, // is this xpath a record or a field
- int flags // are we to flatten matching xpaths
- ) {
- // recursivly walk the paths Lists adding new Nodes as required
- String xpseg = paths.remove(0); // shift out next Xpath segment
-
- if (paths.isEmpty() && xpseg.startsWith("@")) {
- // we have reached end of element portion of Xpath and can now only
- // have an element attribute. Add it to this nodes list of attributes
- if (attributes == null) {
- attributes = new ArrayList<>();
- }
- xpseg = xpseg.substring(1); // strip the '@'
- attributes.add(new Node(xpseg, fieldName, multiValued));
- }
- else if ( xpseg.length() == 0) {
- // we have a '//' selector for all decendents of the current nodes
- xpseg = paths.remove(0); // shift out next Xpath segment
- if (wildCardNodes == null) wildCardNodes = new ArrayList<>();
- Node n = getOrAddNode(xpseg, wildCardNodes);
- if (paths.isEmpty()) {
- // We are current a leaf node.
- // xpath with content we want to store and return
- n.hasText = true; // we have to store text found here
- n.fieldName = fieldName; // name to store collected text against
- n.multiValued = multiValued; // true: text be stored in a List
- n.flatten = flags == FLATTEN; // true: store text from child tags
- }
- else {
- // recurse to handle next paths segment
- n.build(paths, fieldName, multiValued, record, flags);
- }
- }
- else {
- if (childNodes == null)
- childNodes = new ArrayList<>();
- // does this "name" already exist as a child node.
- Node n = getOrAddNode(xpseg,childNodes);
- if (paths.isEmpty()) {
- // We have emptied paths, we are for the moment a leaf of the tree.
- // When parsing the actual input we have traversed to a position
- // where we actutally have to do something. getOrAddNode() will
- // have created and returned a new minimal Node with name and
- // xpathName already populated. We need to add more information.
- if (record) {
- // forEach attribute
- n.isRecord = true; // flag: forEach attribute, prepare to emit rec
- n.forEachPath = fieldName; // the full forEach attribute xpath
- } else {
- // xpath with content we want to store and return
- n.hasText = true; // we have to store text found here
- n.fieldName = fieldName; // name to store collected text against
- n.multiValued = multiValued; // true: text be stored in a List
- n.flatten = flags == FLATTEN; // true: store text from child tags
- }
- } else {
- // recurse to handle next paths segment
- n.build(paths, fieldName, multiValued, record, flags);
- }
- }
- }
-
- private Node getOrAddNode(String xpathName, List<Node> searchList ) {
- for (Node n : searchList)
- if (n.xpathName.equals(xpathName)) return n;
- // new territory! add a new node for this Xpath bitty
- Node n = new Node(xpathName, this); // a minimal Node initialization
- Matcher m = ATTRIB_PRESENT_WITHVAL.matcher(xpathName);
- if (m.find()) {
- n.name = m.group(1);
- int start = m.start(2);
- while (true) {
- HashMap<String, String> attribs = new HashMap<>();
- if (!m.find(start))
- break;
- attribs.put(m.group(3), m.group(5));
- start = m.end(6);
- if (n.attribAndValues == null)
- n.attribAndValues = new ArrayList<>();
- n.attribAndValues.addAll(attribs.entrySet());
- }
- }
- searchList.add(n);
- return n;
- }
-
- /**
- * Copies a supplied Map to a new Map which is returned. Used to copy a
- * records values. If a fields value is a List then they have to be
- * deep-copied for thread safety
- */
- private static Map<String, Object> getDeepCopy(Map<String, Object> values) {
- Map<String, Object> result = new HashMap<>();
- for (Map.Entry<String, Object> entry : values.entrySet()) {
- if (entry.getValue() instanceof List) {
- result.put(entry.getKey(), new ArrayList((List) entry.getValue()));
- } else {
- result.put(entry.getKey(), entry.getValue());
- }
- }
- return result;
- }
- } // end of class Node
-
-
- /**
- * The Xpath is split into segments using the '/' as a separator. However
- * this method deals with special cases where there is a slash '/' character
- * inside the attribute value e.g. x/@html='text/html'. We split by '/' but
- * then reassemble things were the '/' appears within a quoted sub-string.
- *
- * We have already enforced that the string must begin with a separator. This
- * method depends heavily on how split behaves if the string starts with the
- * separator or if a sequence of multiple separator's appear.
- */
- private static List<String> splitEscapeQuote(String str) {
- List<String> result = new LinkedList<>();
- String[] ss = str.split("/");
- for (int i=0; i<ss.length; i++) { // i=1: skip separator at start of string
- StringBuilder sb = new StringBuilder();
- int quoteCount = 0;
- while (true) {
- sb.append(ss[i]);
- for (int j=0; j<ss[i].length(); j++)
- if (ss[i].charAt(j) == '\'') quoteCount++;
- // have we got a split inside quoted sub-string?
- if ((quoteCount % 2) == 0) break;
- // yes!; replace the '/' and loop to concat next token
- i++;
- sb.append("/");
- }
- result.add(sb.toString());
- }
- return result;
- }
-
- static XMLInputFactory factory = XMLInputFactory.newInstance();
- static {
- EmptyEntityResolver.configureXMLInputFactory(factory);
- factory.setXMLReporter(XMLLOG);
- try {
- // The java 1.6 bundled stax parser (sjsxp) does not currently have a thread-safe
- // XMLInputFactory, as that implementation tries to cache and reuse the
- // XMLStreamReader. Setting the parser-specific "reuse-instance" property to false
- // prevents this.
- // All other known open-source stax parsers (and the bea ref impl)
- // have thread-safe factories.
- factory.setProperty("reuse-instance", Boolean.FALSE);
- } catch (IllegalArgumentException ex) {
- // Other implementations will likely throw this exception since "reuse-instance"
- // isimplementation specific.
- log.debug("Unable to set the 'reuse-instance' property for the input chain: " + factory);
- }
- }
-
- /**Implement this interface to stream records as and when one is found.
- *
- */
- public interface Handler {
- /**
- * @param record The record map. The key is the field name as provided in
- * the addField() methods. The value can be a single String (for single
- * valued fields) or a List<String> (for multiValued).
- * @param xpath The forEach XPATH for which this record is being emitted
- * If there is any change all parsing will be aborted and the Exception
- * is propagated up
- */
- void handle(Map<String, Object> record, String xpath);
- }
-
- private static final Pattern ATTRIB_PRESENT_WITHVAL = Pattern
- .compile("(\\S*?)?(\\[@)(\\S*?)(='(.*?)')?(\\])");
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java
deleted file mode 100644
index 64a776c..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.StringReader;
-import java.io.StringWriter;
-import java.lang.invoke.MethodHandles;
-import java.nio.charset.StandardCharsets;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.zookeeper.KeeperException.NodeExistsException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * <p>
- * A SolrCloud-friendly extension of {@link SimplePropertiesWriter}.
- * This implementation ignores the "directory" parameter, saving
- * the properties file under /configs/[solrcloud collection name]/
- */
-public class ZKPropertiesWriter extends SimplePropertiesWriter {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private String path;
- private SolrZkClient zkClient;
-
- @Override
- public void init(DataImporter dataImporter, Map<String, String> params) {
- super.init(dataImporter, params);
- zkClient = dataImporter.getCore().getCoreContainer().getZkController().getZkClient();
- }
-
- @Override
- protected void findDirectory(DataImporter dataImporter, Map<String, String> params) {
- String collection = dataImporter.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName();
- path = "/configs/" + collection + "/" + filename;
- }
-
- @Override
- public boolean isWritable() {
- return true;
- }
-
- @Override
- public void persist(Map<String, Object> propObjs) {
- Properties existing = mapToProperties(readIndexerProperties());
- existing.putAll(mapToProperties(propObjs));
- StringWriter output = new StringWriter();
- try {
- existing.store(output, null);
- byte[] bytes = output.toString().getBytes(StandardCharsets.UTF_8);
- if (!zkClient.exists(path, false)) {
- try {
- zkClient.makePath(path, false);
- } catch (NodeExistsException e) {}
- }
- zkClient.setData(path, bytes, false);
- } catch (Exception e) {
- SolrZkClient.checkInterrupted(e);
- log.warn("Could not persist properties to " + path + " :" + e.getClass(), e);
- }
- }
-
- @Override
- public Map<String, Object> readIndexerProperties() {
- Properties props = new Properties();
- try {
- byte[] data = zkClient.getData(path, null, null, true);
- if (data != null) {
- props.load(new StringReader(new String(data, StandardCharsets.UTF_8)));
- }
- } catch (Exception e) {
- SolrZkClient.checkInterrupted(e);
- log.warn("Could not read DIH properties from " + path + " :" + e.getClass(), e);
- }
- return propertiesToMap(props);
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Zipper.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Zipper.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Zipper.java
deleted file mode 100644
index 096b3a8..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Zipper.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.lang.invoke.MethodHandles;
-import java.util.Iterator;
-import java.util.Map;
-
-import org.apache.solr.handler.dataimport.DIHCacheSupport.Relation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Iterators;
-import com.google.common.collect.PeekingIterator;
-
-class Zipper {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private final DIHCacheSupport.Relation relation;
-
- @SuppressWarnings("rawtypes")
- private Comparable parentId;
- @SuppressWarnings("rawtypes")
- private Comparable lastChildId;
-
- private Iterator<Map<String,Object>> rowIterator;
- private PeekingIterator<Map<String,Object>> peeker;
-
- /** @return initialized zipper or null */
- public static Zipper createOrNull(Context context){
- if("zipper".equals(context.getEntityAttribute("join"))){
- DIHCacheSupport.Relation r = new DIHCacheSupport.Relation(context);
- if(r.doKeyLookup){
- return new Zipper(r);
- }
- }
- return null;
- }
-
-
- private Zipper(Relation relation) {
- this.relation = relation;
- }
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- public Map<String,Object> supplyNextChild(
- Iterator<Map<String,Object>> rowIterator) {
- preparePeeker(rowIterator);
-
- while(peeker.hasNext()){
- Map<String,Object> current = peeker.peek();
- Comparable childId = (Comparable) current.get(relation.primaryKey);
-
- if(lastChildId!=null && lastChildId.compareTo(childId)>0){
- throw new IllegalArgumentException("expect increasing foreign keys for "+relation+
- " got: "+lastChildId+","+childId);
- }
- lastChildId = childId;
- int cmp = childId.compareTo(parentId);
- if(cmp==0){
- Map<String,Object> child = peeker.next();
- assert child==current: "peeker should be right but "+current+" != " + child;
- log.trace("yeild child {} entry {}",relation, current);
- return child;// TODO it's for one->many for many->one it should be just peek()
- }else{
- if(cmp<0){ // child belongs to 10th and parent is 20th, skip for the next one
- Map<String,Object> child = peeker.next();
- assert child==current: "peeker should be right but "+current+" != " + child;
- log.trace("skip child {}, {} > {}",relation, parentId, childId);
- }else{ // child belongs to 20th and parent is 10th, no more children, go to next parent
- log.trace("childen is over {}, {} < {}", relation, parentId, current);
- return null;
- }
- }
- }
-
- return null;
- }
-
- private void preparePeeker(Iterator<Map<String,Object>> rowIterator) {
- if(this.rowIterator==null){
- this.rowIterator = rowIterator;
- peeker = Iterators.peekingIterator(rowIterator);
- }else{
- assert this.rowIterator==rowIterator: "rowIterator should never change but "+this.rowIterator+
- " supplied before has been changed to "+rowIterator;
- }
- }
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- public void onNewParent(Context context) {
- Comparable newParent = (Comparable) context.resolve(relation.foreignKey);
- if(parentId!=null && parentId.compareTo(newParent)>=0){
- throw new IllegalArgumentException("expect strictly increasing primary keys for "+relation+
- " got: "+parentId+","+newParent);
- }
- log.trace("{}: {}->{}",relation, newParent, parentId);
- parentId = newParent;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigNameConstants.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigNameConstants.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigNameConstants.java
deleted file mode 100644
index a34e31d..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigNameConstants.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport.config;
-
-import org.apache.solr.handler.dataimport.SolrWriter;
-
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
-
-public class ConfigNameConstants {
- public static final String SCRIPT = "script";
-
- public static final String NAME = "name";
-
- public static final String PROCESSOR = "processor";
-
- public static final String PROPERTY_WRITER = "propertyWriter";
-
- public static final String IMPORTER_NS = "dataimporter";
-
- public static final String IMPORTER_NS_SHORT = "dih";
-
- public static final String ROOT_ENTITY = "rootEntity";
-
- public static final String CHILD = "child";
-
- public static final String FUNCTION = "function";
-
- public static final String CLASS = "class";
-
- public static final String DATA_SRC = "dataSource";
-
- public static final Set<String> RESERVED_WORDS;
- static{
- Set<String> rw = new HashSet<>();
- rw.add(IMPORTER_NS);
- rw.add(IMPORTER_NS_SHORT);
- rw.add("request");
- rw.add("delta");
- rw.add("functions");
- rw.add("session");
- rw.add(SolrWriter.LAST_INDEX_KEY);
- RESERVED_WORDS = Collections.unmodifiableSet(rw);
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java
deleted file mode 100644
index c369296..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport.config;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import org.w3c.dom.Element;
-import org.w3c.dom.NamedNodeMap;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-
-public class ConfigParseUtil {
-
- public static String getStringAttribute(Element e, String name, String def) {
- String r = e.getAttribute(name);
- if (r == null || "".equals(r.trim())) r = def;
- return r;
- }
-
- public static HashMap<String,String> getAllAttributes(Element e) {
- HashMap<String,String> m = new HashMap<>();
- NamedNodeMap nnm = e.getAttributes();
- for (int i = 0; i < nnm.getLength(); i++) {
- m.put(nnm.item(i).getNodeName(), nnm.item(i).getNodeValue());
- }
- return m;
- }
-
- public static String getText(Node elem, StringBuilder buffer) {
- if (elem.getNodeType() != Node.CDATA_SECTION_NODE) {
- NodeList childs = elem.getChildNodes();
- for (int i = 0; i < childs.getLength(); i++) {
- Node child = childs.item(i);
- short childType = child.getNodeType();
- if (childType != Node.COMMENT_NODE
- && childType != Node.PROCESSING_INSTRUCTION_NODE) {
- getText(child, buffer);
- }
- }
- } else {
- buffer.append(elem.getNodeValue());
- }
-
- return buffer.toString();
- }
-
- public static List<Element> getChildNodes(Element e, String byName) {
- List<Element> result = new ArrayList<>();
- NodeList l = e.getChildNodes();
- for (int i = 0; i < l.getLength(); i++) {
- if (e.equals(l.item(i).getParentNode())
- && byName.equals(l.item(i).getNodeName())) result.add((Element) l
- .item(i));
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java
deleted file mode 100644
index 0ba13ea..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport.config;
-
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-import org.apache.solr.handler.dataimport.DataImporter;
-import org.apache.solr.handler.dataimport.DocBuilder;
-import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.schema.SchemaField;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Element;
-
-/**
- * <p>
- * Mapping for data-config.xml
- * </p>
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
- * <p>
- * <b>This API is experimental and subject to change</b>
- *
- * @since solr 1.3
- */
-public class DIHConfiguration {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- // TODO - remove from here and add it to entity
- private final String deleteQuery;
-
- private final List<Entity> entities;
- private final String onImportStart;
- private final String onImportEnd;
- private final String onError;
- private final List<Map<String, String>> functions;
- private final Script script;
- private final Map<String, Map<String,String>> dataSources;
- private final PropertyWriter propertyWriter;
- private final IndexSchema schema;
- private final Map<String,SchemaField> lowerNameVsSchemaField;
-
- public DIHConfiguration(Element element, DataImporter di,
- List<Map<String,String>> functions, Script script,
- Map<String,Map<String,String>> dataSources, PropertyWriter pw) {
- schema = di.getSchema();
- lowerNameVsSchemaField = null == schema ? Collections.<String,SchemaField>emptyMap() : loadSchemaFieldMap();
- this.deleteQuery = ConfigParseUtil.getStringAttribute(element, "deleteQuery", null);
- this.onImportStart = ConfigParseUtil.getStringAttribute(element, "onImportStart", null);
- this.onImportEnd = ConfigParseUtil.getStringAttribute(element, "onImportEnd", null);
- this.onError = ConfigParseUtil.getStringAttribute(element, "onError", null);
- List<Entity> modEntities = new ArrayList<>();
- List<Element> l = ConfigParseUtil.getChildNodes(element, "entity");
- boolean docRootFound = false;
- for (Element e : l) {
- Entity entity = new Entity(docRootFound, e, di, this, null);
- Map<String, EntityField> fields = gatherAllFields(di, entity);
- verifyWithSchema(fields);
- modEntities.add(entity);
- }
- this.entities = Collections.unmodifiableList(modEntities);
- if(functions==null) {
- functions = Collections.emptyList();
- }
- List<Map<String, String>> modFunc = new ArrayList<>(functions.size());
- for(Map<String, String> f : functions) {
- modFunc.add(Collections.unmodifiableMap(f));
- }
- this.functions = Collections.unmodifiableList(modFunc);
- this.script = script;
- this.dataSources = Collections.unmodifiableMap(dataSources);
- this.propertyWriter = pw;
- }
-
- private void verifyWithSchema(Map<String,EntityField> fields) {
- Map<String,SchemaField> schemaFields = null;
- if (schema == null) {
- schemaFields = Collections.emptyMap();
- } else {
- schemaFields = schema.getFields();
- }
- for (Map.Entry<String,SchemaField> entry : schemaFields.entrySet()) {
- SchemaField sf = entry.getValue();
- if (!fields.containsKey(sf.getName())) {
- if (sf.isRequired()) {
- log.info(sf.getName() + " is a required field in SolrSchema . But not found in DataConfig");
- }
- }
- }
- for (Map.Entry<String,EntityField> entry : fields.entrySet()) {
- EntityField fld = entry.getValue();
- SchemaField field = getSchemaField(fld.getName());
- if (field == null && !isSpecialCommand(fld.getName())) {
- log.info("The field :" + fld.getName() + " present in DataConfig does not have a counterpart in Solr Schema");
- }
- }
- }
-
- private Map<String,EntityField> gatherAllFields(DataImporter di, Entity e) {
- Map<String,EntityField> fields = new HashMap<>();
- if (e.getFields() != null) {
- for (EntityField f : e.getFields()) {
- fields.put(f.getName(), f);
- }
- }
- for (Entity e1 : e.getChildren()) {
- fields.putAll(gatherAllFields(di, e1));
- }
- return fields;
- }
-
- private Map<String,SchemaField> loadSchemaFieldMap() {
- Map<String, SchemaField> modLnvsf = new HashMap<>();
- for (Map.Entry<String, SchemaField> entry : schema.getFields().entrySet()) {
- modLnvsf.put(entry.getKey().toLowerCase(Locale.ROOT), entry.getValue());
- }
- return Collections.unmodifiableMap(modLnvsf);
- }
-
- public SchemaField getSchemaField(String caseInsensitiveName) {
- SchemaField schemaField = null;
- if(schema!=null) {
- schemaField = schema.getFieldOrNull(caseInsensitiveName);
- }
- if (schemaField == null) {
- schemaField = lowerNameVsSchemaField.get(caseInsensitiveName.toLowerCase(Locale.ROOT));
- }
- return schemaField;
- }
-
-
- public String getDeleteQuery() {
- return deleteQuery;
- }
- public List<Entity> getEntities() {
- return entities;
- }
- public String getOnImportStart() {
- return onImportStart;
- }
- public String getOnImportEnd() {
- return onImportEnd;
- }
- public String getOnError() {
- return onError;
- }
- public List<Map<String,String>> getFunctions() {
- return functions;
- }
- public Map<String,Map<String,String>> getDataSources() {
- return dataSources;
- }
- public Script getScript() {
- return script;
- }
- public PropertyWriter getPropertyWriter() {
- return propertyWriter;
- }
-
- public IndexSchema getSchema() {
- return schema;
- }
-
- public static boolean isSpecialCommand(String fld) {
- return DocBuilder.DELETE_DOC_BY_ID.equals(fld) ||
- DocBuilder.DELETE_DOC_BY_QUERY.equals(fld) ||
- DocBuilder.DOC_BOOST.equals(fld) ||
- DocBuilder.SKIP_DOC.equals(fld) ||
- DocBuilder.SKIP_ROW.equals(fld);
-
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Entity.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Entity.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Entity.java
deleted file mode 100644
index 0d0ba4f..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Entity.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport.config;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.solr.handler.dataimport.DataImportHandlerException;
-import org.apache.solr.handler.dataimport.DataImporter;
-import org.apache.solr.schema.SchemaField;
-import org.w3c.dom.Element;
-
-public class Entity {
- private final String name;
- private final String pk;
- private final String pkMappingFromSchema;
- private final String dataSourceName;
- private final String processorName;
- private final Entity parentEntity;
- private final boolean docRoot;
- private final boolean child;
- private final List<Entity> children;
- private final List<EntityField> fields;
- private final Map<String,Set<EntityField>> colNameVsField;
- private final Map<String,String> allAttributes;
- private final List<Map<String,String>> allFieldAttributes;
- private final DIHConfiguration config;
-
- public Entity(boolean docRootFound, Element element, DataImporter di, DIHConfiguration config, Entity parent) {
- this.parentEntity = parent;
- this.config = config;
-
- String modName = ConfigParseUtil.getStringAttribute(element, ConfigNameConstants.NAME, null);
- if (modName == null) {
- throw new DataImportHandlerException(SEVERE, "Entity must have a name.");
- }
- if (modName.indexOf(".") != -1) {
- throw new DataImportHandlerException(SEVERE,
- "Entity name must not have period (.): '" + modName);
- }
- if (ConfigNameConstants.RESERVED_WORDS.contains(modName)) {
- throw new DataImportHandlerException(SEVERE, "Entity name : '" + modName
- + "' is a reserved keyword. Reserved words are: " + ConfigNameConstants.RESERVED_WORDS);
- }
- this.name = modName;
- this.pk = ConfigParseUtil.getStringAttribute(element, "pk", null);
- this.processorName = ConfigParseUtil.getStringAttribute(element, ConfigNameConstants.PROCESSOR,null);
- this.dataSourceName = ConfigParseUtil.getStringAttribute(element, DataImporter.DATA_SRC, null);
-
- String rawDocRootValue = ConfigParseUtil.getStringAttribute(element, ConfigNameConstants.ROOT_ENTITY, null);
- if (!docRootFound && !"false".equals(rawDocRootValue)) {
- // if in this chain no document root is found()
- docRoot = true;
- } else {
- docRoot = false;
- }
-
- String childValue = ConfigParseUtil.getStringAttribute(element, ConfigNameConstants.CHILD, null);
- child = "true".equals(childValue);
-
- Map<String,String> modAttributes = ConfigParseUtil
- .getAllAttributes(element);
- modAttributes.put(ConfigNameConstants.DATA_SRC, this.dataSourceName);
- this.allAttributes = Collections.unmodifiableMap(modAttributes);
-
- List<Element> n = ConfigParseUtil.getChildNodes(element, "field");
- List<EntityField> modFields = new ArrayList<>(n.size());
- Map<String,Set<EntityField>> modColNameVsField = new HashMap<>();
- List<Map<String,String>> modAllFieldAttributes = new ArrayList<>();
- for (Element elem : n) {
- EntityField.Builder fieldBuilder = new EntityField.Builder(elem);
- if (config.getSchema() != null) {
- if (fieldBuilder.getNameOrColumn() != null
- && fieldBuilder.getNameOrColumn().contains("${")) {
- fieldBuilder.dynamicName = true;
- } else {
- SchemaField schemaField = config.getSchemaField
- (fieldBuilder.getNameOrColumn());
- if (schemaField != null) {
- fieldBuilder.name = schemaField.getName();
- fieldBuilder.multiValued = schemaField.multiValued();
- fieldBuilder.allAttributes.put(DataImporter.MULTI_VALUED, Boolean
- .toString(schemaField.multiValued()));
- fieldBuilder.allAttributes.put(DataImporter.TYPE, schemaField
- .getType().getTypeName());
- fieldBuilder.allAttributes.put("indexed", Boolean
- .toString(schemaField.indexed()));
- fieldBuilder.allAttributes.put("stored", Boolean
- .toString(schemaField.stored()));
- fieldBuilder.allAttributes.put("defaultValue", schemaField
- .getDefaultValue());
- } else {
- fieldBuilder.toWrite = false;
- }
- }
- }
- Set<EntityField> fieldSet = modColNameVsField.get(fieldBuilder.column);
- if (fieldSet == null) {
- fieldSet = new HashSet<>();
- modColNameVsField.put(fieldBuilder.column, fieldSet);
- }
- fieldBuilder.allAttributes.put("boost", Float
- .toString(fieldBuilder.boost));
- fieldBuilder.allAttributes.put("toWrite", Boolean
- .toString(fieldBuilder.toWrite));
- modAllFieldAttributes.add(fieldBuilder.allAttributes);
- fieldBuilder.entity = this;
- EntityField field = new EntityField(fieldBuilder);
- fieldSet.add(field);
- modFields.add(field);
- }
- Map<String,Set<EntityField>> modColNameVsField1 = new HashMap<>();
- for (Map.Entry<String,Set<EntityField>> entry : modColNameVsField
- .entrySet()) {
- if (entry.getValue().size() > 0) {
- modColNameVsField1.put(entry.getKey(), Collections
- .unmodifiableSet(entry.getValue()));
- }
- }
- this.colNameVsField = Collections.unmodifiableMap(modColNameVsField1);
- this.fields = Collections.unmodifiableList(modFields);
- this.allFieldAttributes = Collections
- .unmodifiableList(modAllFieldAttributes);
-
- String modPkMappingFromSchema = null;
- if (config.getSchema() != null) {
- SchemaField uniqueKey = config.getSchema().getUniqueKeyField();
- if (uniqueKey != null) {
- modPkMappingFromSchema = uniqueKey.getName();
- // if no fields are mentioned . solr uniqueKey is same as dih 'pk'
- for (EntityField field : fields) {
- if (field.getName().equals(modPkMappingFromSchema)) {
- modPkMappingFromSchema = field.getColumn();
- // get the corresponding column mapping for the solr uniqueKey
- // But if there are multiple columns mapping to the solr uniqueKey,
- // it will fail
- // so , in one off cases we may need pk
- break;
- }
- }
- }
- }
- pkMappingFromSchema = modPkMappingFromSchema;
- n = ConfigParseUtil.getChildNodes(element, "entity");
- List<Entity> modEntities = new ArrayList<>();
- for (Element elem : n) {
- modEntities.add(new Entity((docRootFound || this.docRoot), elem, di, config, this));
- }
- this.children = Collections.unmodifiableList(modEntities);
- }
-
- public String getPk() {
- return pk == null ? pkMappingFromSchema : pk;
- }
-
- public String getSchemaPk() {
- return pkMappingFromSchema != null ? pkMappingFromSchema : pk;
- }
-
- public String getName() {
- return name;
- }
-
- public String getPkMappingFromSchema() {
- return pkMappingFromSchema;
- }
-
- public String getDataSourceName() {
- return dataSourceName;
- }
-
- public String getProcessorName() {
- return processorName;
- }
-
- public Entity getParentEntity() {
- return parentEntity;
- }
-
- public boolean isDocRoot() {
- return docRoot;
- }
-
- public List<Entity> getChildren() {
- return children;
- }
-
- public List<EntityField> getFields() {
- return fields;
- }
-
- public Map<String,Set<EntityField>> getColNameVsField() {
- return colNameVsField;
- }
-
- public Map<String,String> getAllAttributes() {
- return allAttributes;
- }
-
- public List<Map<String,String>> getAllFieldsList() {
- return allFieldAttributes;
- }
-
- public boolean isChild() {
- return child;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/EntityField.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/EntityField.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/EntityField.java
deleted file mode 100644
index 2b28cb7..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/EntityField.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport.config;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.solr.handler.dataimport.ConfigParseUtil;
-import org.apache.solr.handler.dataimport.DataImportHandlerException;
-import org.apache.solr.handler.dataimport.DataImporter;
-import org.w3c.dom.Element;
-
-public class EntityField {
- private final String column;
- private final String name;
- private final boolean toWrite;
- private final boolean multiValued;
- private final boolean dynamicName;
- private final Entity entity;
- private final Map<String, String> allAttributes;
-
- public EntityField(Builder b) {
- this.column = b.column;
- this.name = b.name;
- this.toWrite = b.toWrite;
- this.multiValued = b.multiValued;
- this.dynamicName = b.dynamicName;
- this.entity = b.entity;
- this.allAttributes = Collections.unmodifiableMap(new HashMap<>(b.allAttributes));
- }
-
- public String getName() {
- return name == null ? column : name;
- }
-
- public Entity getEntity() {
- return entity;
- }
-
- public String getColumn() {
- return column;
- }
-
- public boolean isToWrite() {
- return toWrite;
- }
-
- public boolean isMultiValued() {
- return multiValued;
- }
-
- public boolean isDynamicName() {
- return dynamicName;
- }
-
- public Map<String,String> getAllAttributes() {
- return allAttributes;
- }
-
- public static class Builder {
- public String column;
- public String name;
- public float boost;
- public boolean toWrite = true;
- public boolean multiValued = false;
- public boolean dynamicName = false;
- public Entity entity;
- public Map<String, String> allAttributes = new HashMap<>();
-
- public Builder(Element e) {
- this.name = ConfigParseUtil.getStringAttribute(e, DataImporter.NAME, null);
- this.column = ConfigParseUtil.getStringAttribute(e, DataImporter.COLUMN, null);
- if (column == null) {
- throw new DataImportHandlerException(SEVERE, "Field must have a column attribute");
- }
- this.boost = Float.parseFloat(ConfigParseUtil.getStringAttribute(e, "boost", "1.0f"));
- this.allAttributes = new HashMap<>(ConfigParseUtil.getAllAttributes(e));
- }
-
- public String getNameOrColumn() {
- return name==null ? column : name;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Field.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Field.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Field.java
deleted file mode 100644
index 7ff4832..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Field.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport.config;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.solr.handler.dataimport.ConfigParseUtil;
-import org.apache.solr.handler.dataimport.DataImportHandlerException;
-import org.apache.solr.handler.dataimport.DataImporter;
-import org.w3c.dom.Element;
-
-public class Field {
- private final String column;
- private final String name;
- private final float boost;
- private final boolean toWrite;
- private final boolean multiValued;
- private final boolean dynamicName;
- private final Entity entity;
- private final Map<String, String> allAttributes;
-
- public Field(Builder b) {
- this.column = b.column;
- this.name = b.name;
- this.boost = b.boost;
- this.toWrite = b.toWrite;
- this.multiValued = b.multiValued;
- this.dynamicName = b.dynamicName;
- this.entity = b.entity;
- this.allAttributes = Collections.unmodifiableMap(new HashMap<>(b.allAttributes));
- }
-
- public String getName() {
- return name == null ? column : name;
- }
-
- public Entity getEntity() {
- return entity;
- }
-
- public String getColumn() {
- return column;
- }
-
- public float getBoost() {
- return boost;
- }
-
- public boolean isToWrite() {
- return toWrite;
- }
-
- public boolean isMultiValued() {
- return multiValued;
- }
-
- public boolean isDynamicName() {
- return dynamicName;
- }
-
- public Map<String,String> getAllAttributes() {
- return allAttributes;
- }
-
- public static class Builder {
- public String column;
- public String name;
- public float boost;
- public boolean toWrite = true;
- public boolean multiValued = false;
- public boolean dynamicName;
- public Entity entity;
- public Map<String, String> allAttributes = new HashMap<>();
-
- public Builder(Element e) {
- this.name = ConfigParseUtil.getStringAttribute(e, DataImporter.NAME, null);
- this.column = ConfigParseUtil.getStringAttribute(e, DataImporter.COLUMN, null);
- if (column == null) {
- throw new DataImportHandlerException(SEVERE, "Field must have a column attribute");
- }
- this.boost = Float.parseFloat(ConfigParseUtil.getStringAttribute(e, "boost", "1.0f"));
- this.allAttributes = new HashMap<>(ConfigParseUtil.getAllAttributes(e));
- }
-
- public String getNameOrColumn() {
- return name==null ? column : name;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/PropertyWriter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/PropertyWriter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/PropertyWriter.java
deleted file mode 100644
index 6e91a19..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/PropertyWriter.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport.config;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class PropertyWriter {
- private final String type;
- private Map<String,String> parameters;
-
- public PropertyWriter(String type, Map<String,String> parameters) {
- this.type = type;
- this.parameters = new HashMap<String,String>(parameters);
- }
-
- public Map<String,String> getParameters() {
- return parameters;
- }
-
- public String getType() {
- return type;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Script.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Script.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Script.java
deleted file mode 100644
index 9a4bc59..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Script.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport.config;
-
-import org.w3c.dom.Element;
-
-public class Script {
- private final String language;
- private final String text;
-
- public Script(Element e) {
- this.language = ConfigParseUtil.getStringAttribute(e, "language", "JavaScript");
- StringBuilder buffer = new StringBuilder();
- String script = ConfigParseUtil.getText(e, buffer);
- if (script != null) {
- this.text = script.trim();
- } else {
- this.text = null;
- }
- }
- public String getLanguage() {
- return language;
- }
- public String getText() {
- return text;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/package-info.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/package-info.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/package-info.java
deleted file mode 100644
index 50c6d4e..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Utility classes for parsing & modeling DIH configuration.
- */
-package org.apache.solr.handler.dataimport.config;
-
-
-
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/package-info.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/package-info.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/package-info.java
deleted file mode 100644
index 4a69d23..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/package-info.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * {@link org.apache.solr.handler.dataimport.DataImportHandler} and related code.
- */
-package org.apache.solr.handler.dataimport;
-
-
-
-
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/overview.html
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/overview.html b/solr/contrib/dataimporthandler/src/java/overview.html
deleted file mode 100644
index 4bb2c8b..0000000
--- a/solr/contrib/dataimporthandler/src/java/overview.html
+++ /dev/null
@@ -1,21 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<html>
-<body>
-Apache Solr Search Server: DataImportHandler contrib
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinContentStreamDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinContentStreamDataSource.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinContentStreamDataSource.java
new file mode 100644
index 0000000..f4b1d7a
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinContentStreamDataSource.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.common.util.ContentStream;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+
+import java.io.InputStream;
+import java.io.IOException;
+import java.util.Properties;
+/**
+ * <p> A data source implementation which can be used to read binary stream from content streams. </p> <p> Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a> for more
+ * details. </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 3.1
+ */
+
+public class BinContentStreamDataSource extends DataSource<InputStream> {
+ private ContextImpl context;
+ private ContentStream contentStream;
+ private InputStream in;
+
+
+ @Override
+ public void init(Context context, Properties initProps) {
+ this.context = (ContextImpl) context;
+ }
+
+ @Override
+ public InputStream getData(String query) {
+ contentStream = context.getDocBuilder().getReqParams().getContentStream();
+ if (contentStream == null)
+ throw new DataImportHandlerException(SEVERE, "No stream available. The request has no body");
+ try {
+ return in = contentStream.getStream();
+ } catch (IOException e) {
+ DataImportHandlerException.wrapAndThrow(SEVERE, e);
+ return null;
+ }
+ }
+
+ @Override
+ public void close() {
+ if (contentStream != null) {
+ try {
+ if (in == null) in = contentStream.getStream();
+ in.close();
+ } catch (IOException e) {
+ /*no op*/
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinFileDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinFileDataSource.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinFileDataSource.java
new file mode 100644
index 0000000..dc7a0f5
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinFileDataSource.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+
+import java.io.InputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.util.Properties;
+/**
+ * <p>
+ * A DataSource which reads from local files
+ * </p>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 3.1
+ */
+
+public class BinFileDataSource extends DataSource<InputStream>{
+ protected String basePath;
+ @Override
+ public void init(Context context, Properties initProps) {
+ basePath = initProps.getProperty(FileDataSource.BASE_PATH);
+ }
+
+ @Override
+ public InputStream getData(String query) {
+ File f = FileDataSource.getFile(basePath,query);
+ try {
+ return new FileInputStream(f);
+ } catch (FileNotFoundException e) {
+ wrapAndThrow(SEVERE,e,"Unable to open file "+f.getAbsolutePath());
+ return null;
+ }
+ }
+
+ @Override
+ public void close() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinURLDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinURLDataSource.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinURLDataSource.java
new file mode 100644
index 0000000..c1b4808
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinURLDataSource.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.*;
+import static org.apache.solr.handler.dataimport.URLDataSource.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.InputStream;
+import java.lang.invoke.MethodHandles;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.Properties;
+/**
+ * <p> A data source implementation which can be used to read binary streams using HTTP. </p> <p> Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a> for more
+ * details. </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 3.1
+ */
+public class BinURLDataSource extends DataSource<InputStream>{
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private String baseUrl;
+ private int connectionTimeout = CONNECTION_TIMEOUT;
+
+ private int readTimeout = READ_TIMEOUT;
+
+ private Context context;
+
+ private Properties initProps;
+
+ public BinURLDataSource() { }
+
+ @Override
+ public void init(Context context, Properties initProps) {
+ this.context = context;
+ this.initProps = initProps;
+
+ baseUrl = getInitPropWithReplacements(BASE_URL);
+ String cTimeout = getInitPropWithReplacements(CONNECTION_TIMEOUT_FIELD_NAME);
+ String rTimeout = getInitPropWithReplacements(READ_TIMEOUT_FIELD_NAME);
+ if (cTimeout != null) {
+ try {
+ connectionTimeout = Integer.parseInt(cTimeout);
+ } catch (NumberFormatException e) {
+ log.warn("Invalid connection timeout: " + cTimeout);
+ }
+ }
+ if (rTimeout != null) {
+ try {
+ readTimeout = Integer.parseInt(rTimeout);
+ } catch (NumberFormatException e) {
+ log.warn("Invalid read timeout: " + rTimeout);
+ }
+ }
+ }
+
+ @Override
+ public InputStream getData(String query) {
+ URL url = null;
+ try {
+ if (URIMETHOD.matcher(query).find()) url = new URL(query);
+ else url = new URL(baseUrl + query);
+ log.debug("Accessing URL: " + url.toString());
+ URLConnection conn = url.openConnection();
+ conn.setConnectTimeout(connectionTimeout);
+ conn.setReadTimeout(readTimeout);
+ return conn.getInputStream();
+ } catch (Exception e) {
+ log.error("Exception thrown while getting data", e);
+ wrapAndThrow (SEVERE, e, "Exception in invoking url " + url);
+ return null;//unreachable
+ }
+ }
+
+ @Override
+ public void close() { }
+
+ private String getInitPropWithReplacements(String propertyName) {
+ final String expr = initProps.getProperty(propertyName);
+ if (expr == null) {
+ return null;
+ }
+ return context.replaceTokens(expr);
+ }
+}
[16/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
new file mode 100644
index 0000000..d610d66
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
@@ -0,0 +1,622 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.common.EmptyEntityResolver;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.util.SystemIdResolver;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.XMLErrorLogger;
+import org.apache.solr.handler.dataimport.config.ConfigNameConstants;
+import org.apache.solr.handler.dataimport.config.ConfigParseUtil;
+import org.apache.solr.handler.dataimport.config.DIHConfiguration;
+import org.apache.solr.handler.dataimport.config.Entity;
+import org.apache.solr.handler.dataimport.config.PropertyWriter;
+import org.apache.solr.handler.dataimport.config.Script;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DocBuilder.loadClass;
+import static org.apache.solr.handler.dataimport.config.ConfigNameConstants.CLASS;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+import org.apache.commons.io.IOUtils;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * <p> Stores all configuration information for pulling and indexing data. </p>
+ * <p>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @since solr 1.3
+ */
+public class DataImporter {
+
+ public enum Status {
+ IDLE, RUNNING_FULL_DUMP, RUNNING_DELTA_DUMP, JOB_FAILED
+ }
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static final XMLErrorLogger XMLLOG = new XMLErrorLogger(log);
+
+ private Status status = Status.IDLE;
+ private DIHConfiguration config;
+ private Date indexStartTime;
+ private Properties store = new Properties();
+ private Map<String, Map<String,String>> requestLevelDataSourceProps = new HashMap<>();
+ private IndexSchema schema;
+ public DocBuilder docBuilder;
+ public DocBuilder.Statistics cumulativeStatistics = new DocBuilder.Statistics();
+ private SolrCore core;
+ private Map<String, Object> coreScopeSession = new ConcurrentHashMap<>();
+ private ReentrantLock importLock = new ReentrantLock();
+ private boolean isDeltaImportSupported = false;
+ private final String handlerName;
+
+ /**
+ * Only for testing purposes
+ */
+ DataImporter() {
+ this.handlerName = "dataimport" ;
+ }
+
+ DataImporter(SolrCore core, String handlerName) {
+ this.handlerName = handlerName;
+ this.core = core;
+ this.schema = core.getLatestSchema();
+ }
+
+
+
+
+ boolean maybeReloadConfiguration(RequestInfo params,
+ NamedList<?> defaultParams) throws IOException {
+ if (importLock.tryLock()) {
+ boolean success = false;
+ try {
+ if (null != params.getRequest()) {
+ if (schema != params.getRequest().getSchema()) {
+ schema = params.getRequest().getSchema();
+ }
+ }
+ String dataConfigText = params.getDataConfig();
+ String dataconfigFile = params.getConfigFile();
+ InputSource is = null;
+ if(dataConfigText!=null && dataConfigText.length()>0) {
+ is = new InputSource(new StringReader(dataConfigText));
+ } else if(dataconfigFile!=null) {
+ is = new InputSource(core.getResourceLoader().openResource(dataconfigFile));
+ is.setSystemId(SystemIdResolver.createSystemIdFromResourceName(dataconfigFile));
+ log.info("Loading DIH Configuration: " + dataconfigFile);
+ }
+ if(is!=null) {
+ config = loadDataConfig(is);
+ success = true;
+ }
+
+ Map<String,Map<String,String>> dsProps = new HashMap<>();
+ if(defaultParams!=null) {
+ int position = 0;
+ while (position < defaultParams.size()) {
+ if (defaultParams.getName(position) == null) {
+ break;
+ }
+ String name = defaultParams.getName(position);
+ if (name.equals("datasource")) {
+ success = true;
+ NamedList dsConfig = (NamedList) defaultParams.getVal(position);
+ log.info("Getting configuration for Global Datasource...");
+ Map<String,String> props = new HashMap<>();
+ for (int i = 0; i < dsConfig.size(); i++) {
+ props.put(dsConfig.getName(i), dsConfig.getVal(i).toString());
+ }
+ log.info("Adding properties to datasource: " + props);
+ dsProps.put((String) dsConfig.get("name"), props);
+ }
+ position++;
+ }
+ }
+ requestLevelDataSourceProps = Collections.unmodifiableMap(dsProps);
+ } catch(IOException ioe) {
+ throw ioe;
+ } finally {
+ importLock.unlock();
+ }
+ return success;
+ } else {
+ return false;
+ }
+ }
+
+
+
+ public String getHandlerName() {
+ return handlerName;
+ }
+
+ public IndexSchema getSchema() {
+ return schema;
+ }
+
+ /**
+ * Used by tests
+ */
+ void loadAndInit(String configStr) {
+ config = loadDataConfig(new InputSource(new StringReader(configStr)));
+ }
+
+ void loadAndInit(InputSource configFile) {
+ config = loadDataConfig(configFile);
+ }
+
+ public DIHConfiguration loadDataConfig(InputSource configFile) {
+
+ DIHConfiguration dihcfg = null;
+ try {
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ dbf.setValidating(false);
+
+ // only enable xinclude, if XML is coming from safe source (local file)
+ // and a a SolrCore and SystemId is present (makes no sense otherwise):
+ if (core != null && configFile.getSystemId() != null) {
+ try {
+ dbf.setXIncludeAware(true);
+ dbf.setNamespaceAware(true);
+ } catch( UnsupportedOperationException e ) {
+ log.warn( "XML parser doesn't support XInclude option" );
+ }
+ }
+
+ DocumentBuilder builder = dbf.newDocumentBuilder();
+ // only enable xinclude / external entities, if XML is coming from
+ // safe source (local file) and a a SolrCore and SystemId is present:
+ if (core != null && configFile.getSystemId() != null) {
+ builder.setEntityResolver(new SystemIdResolver(core.getResourceLoader()));
+ } else {
+ // Don't allow external entities without having a system ID:
+ builder.setEntityResolver(EmptyEntityResolver.SAX_INSTANCE);
+ }
+ builder.setErrorHandler(XMLLOG);
+ Document document;
+ try {
+ document = builder.parse(configFile);
+ } finally {
+ // some XML parsers are broken and don't close the byte stream (but they should according to spec)
+ IOUtils.closeQuietly(configFile.getByteStream());
+ }
+
+ dihcfg = readFromXml(document);
+ log.info("Data Configuration loaded successfully");
+ } catch (Exception e) {
+ throw new DataImportHandlerException(SEVERE,
+ "Data Config problem: " + e.getMessage(), e);
+ }
+ for (Entity e : dihcfg.getEntities()) {
+ if (e.getAllAttributes().containsKey(SqlEntityProcessor.DELTA_QUERY)) {
+ isDeltaImportSupported = true;
+ break;
+ }
+ }
+ return dihcfg;
+ }
+
+ public DIHConfiguration readFromXml(Document xmlDocument) {
+ DIHConfiguration config;
+ List<Map<String, String >> functions = new ArrayList<>();
+ Script script = null;
+ Map<String, Map<String,String>> dataSources = new HashMap<>();
+
+ NodeList dataConfigTags = xmlDocument.getElementsByTagName("dataConfig");
+ if(dataConfigTags == null || dataConfigTags.getLength() == 0) {
+ throw new DataImportHandlerException(SEVERE, "the root node '<dataConfig>' is missing");
+ }
+ Element e = (Element) dataConfigTags.item(0);
+ List<Element> documentTags = ConfigParseUtil.getChildNodes(e, "document");
+ if (documentTags.isEmpty()) {
+ throw new DataImportHandlerException(SEVERE, "DataImportHandler " +
+ "configuration file must have one <document> node.");
+ }
+
+ List<Element> scriptTags = ConfigParseUtil.getChildNodes(e, ConfigNameConstants.SCRIPT);
+ if (!scriptTags.isEmpty()) {
+ script = new Script(scriptTags.get(0));
+ }
+
+ // Add the provided evaluators
+ List<Element> functionTags = ConfigParseUtil.getChildNodes(e, ConfigNameConstants.FUNCTION);
+ if (!functionTags.isEmpty()) {
+ for (Element element : functionTags) {
+ String func = ConfigParseUtil.getStringAttribute(element, NAME, null);
+ String clz = ConfigParseUtil.getStringAttribute(element, ConfigNameConstants.CLASS, null);
+ if (func == null || clz == null){
+ throw new DataImportHandlerException(
+ SEVERE,
+ "<function> must have a 'name' and 'class' attributes");
+ } else {
+ functions.add(ConfigParseUtil.getAllAttributes(element));
+ }
+ }
+ }
+ List<Element> dataSourceTags = ConfigParseUtil.getChildNodes(e, ConfigNameConstants.DATA_SRC);
+ if (!dataSourceTags.isEmpty()) {
+ for (Element element : dataSourceTags) {
+ Map<String,String> p = new HashMap<>();
+ HashMap<String, String> attrs = ConfigParseUtil.getAllAttributes(element);
+ for (Map.Entry<String, String> entry : attrs.entrySet()) {
+ p.put(entry.getKey(), entry.getValue());
+ }
+ dataSources.put(p.get("name"), p);
+ }
+ }
+ if(dataSources.get(null) == null){
+ for (Map<String,String> properties : dataSources.values()) {
+ dataSources.put(null,properties);
+ break;
+ }
+ }
+ PropertyWriter pw = null;
+ List<Element> propertyWriterTags = ConfigParseUtil.getChildNodes(e, ConfigNameConstants.PROPERTY_WRITER);
+ if (propertyWriterTags.isEmpty()) {
+ boolean zookeeper = false;
+ if (this.core != null
+ && this.core.getCoreContainer().isZooKeeperAware()) {
+ zookeeper = true;
+ }
+ pw = new PropertyWriter(zookeeper ? "ZKPropertiesWriter"
+ : "SimplePropertiesWriter", Collections.<String,String> emptyMap());
+ } else if (propertyWriterTags.size() > 1) {
+ throw new DataImportHandlerException(SEVERE, "Only one "
+ + ConfigNameConstants.PROPERTY_WRITER + " can be configured.");
+ } else {
+ Element pwElement = propertyWriterTags.get(0);
+ String type = null;
+ Map<String,String> params = new HashMap<>();
+ for (Map.Entry<String,String> entry : ConfigParseUtil.getAllAttributes(
+ pwElement).entrySet()) {
+ if (TYPE.equals(entry.getKey())) {
+ type = entry.getValue();
+ } else {
+ params.put(entry.getKey(), entry.getValue());
+ }
+ }
+ if (type == null) {
+ throw new DataImportHandlerException(SEVERE, "The "
+ + ConfigNameConstants.PROPERTY_WRITER + " element must specify "
+ + TYPE);
+ }
+ pw = new PropertyWriter(type, params);
+ }
+ return new DIHConfiguration(documentTags.get(0), this, functions, script, dataSources, pw);
+ }
+
+ @SuppressWarnings("unchecked")
+ private DIHProperties createPropertyWriter() {
+ DIHProperties propWriter = null;
+ PropertyWriter configPw = config.getPropertyWriter();
+ try {
+ Class<DIHProperties> writerClass = DocBuilder.loadClass(configPw.getType(), this.core);
+ propWriter = writerClass.newInstance();
+ propWriter.init(this, configPw.getParameters());
+ } catch (Exception e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "Unable to PropertyWriter implementation:" + configPw.getType(), e);
+ }
+ return propWriter;
+ }
+
+ public DIHConfiguration getConfig() {
+ return config;
+ }
+
+ Date getIndexStartTime() {
+ return indexStartTime;
+ }
+
+ void setIndexStartTime(Date indextStartTime) {
+ this.indexStartTime = indextStartTime;
+ }
+
+ void store(Object key, Object value) {
+ store.put(key, value);
+ }
+
+ Object retrieve(Object key) {
+ return store.get(key);
+ }
+
+ public DataSource getDataSourceInstance(Entity key, String name, Context ctx) {
+ Map<String,String> p = requestLevelDataSourceProps.get(name);
+ if (p == null)
+ p = config.getDataSources().get(name);
+ if (p == null)
+ p = requestLevelDataSourceProps.get(null);// for default data source
+ if (p == null)
+ p = config.getDataSources().get(null);
+ if (p == null)
+ throw new DataImportHandlerException(SEVERE,
+ "No dataSource :" + name + " available for entity :" + key.getName());
+ String type = p.get(TYPE);
+ DataSource dataSrc = null;
+ if (type == null) {
+ dataSrc = new JdbcDataSource();
+ } else {
+ try {
+ dataSrc = (DataSource) DocBuilder.loadClass(type, getCore()).newInstance();
+ } catch (Exception e) {
+ wrapAndThrow(SEVERE, e, "Invalid type for data source: " + type);
+ }
+ }
+ try {
+ Properties copyProps = new Properties();
+ copyProps.putAll(p);
+ Map<String, Object> map = ctx.getRequestParameters();
+ if (map.containsKey("rows")) {
+ int rows = Integer.parseInt((String) map.get("rows"));
+ if (map.containsKey("start")) {
+ rows += Integer.parseInt((String) map.get("start"));
+ }
+ copyProps.setProperty("maxRows", String.valueOf(rows));
+ }
+ dataSrc.init(ctx, copyProps);
+ } catch (Exception e) {
+ wrapAndThrow(SEVERE, e, "Failed to initialize DataSource: " + key.getDataSourceName());
+ }
+ return dataSrc;
+ }
+
+ public Status getStatus() {
+ return status;
+ }
+
+ public void setStatus(Status status) {
+ this.status = status;
+ }
+
+ public boolean isBusy() {
+ return importLock.isLocked();
+ }
+
+ public void doFullImport(DIHWriter writer, RequestInfo requestParams) {
+ log.info("Starting Full Import");
+ setStatus(Status.RUNNING_FULL_DUMP);
+ try {
+ DIHProperties dihPropWriter = createPropertyWriter();
+ setIndexStartTime(dihPropWriter.getCurrentTimestamp());
+ docBuilder = new DocBuilder(this, writer, dihPropWriter, requestParams);
+ checkWritablePersistFile(writer, dihPropWriter);
+ docBuilder.execute();
+ if (!requestParams.isDebug())
+ cumulativeStatistics.add(docBuilder.importStatistics);
+ } catch (Exception e) {
+ SolrException.log(log, "Full Import failed", e);
+ docBuilder.handleError("Full Import failed", e);
+ } finally {
+ setStatus(Status.IDLE);
+ DocBuilder.INSTANCE.set(null);
+ }
+
+ }
+
+ private void checkWritablePersistFile(DIHWriter writer, DIHProperties dihPropWriter) {
+ if (isDeltaImportSupported && !dihPropWriter.isWritable()) {
+ throw new DataImportHandlerException(SEVERE,
+ "Properties is not writable. Delta imports are supported by data config but will not work.");
+ }
+ }
+
+ public void doDeltaImport(DIHWriter writer, RequestInfo requestParams) {
+ log.info("Starting Delta Import");
+ setStatus(Status.RUNNING_DELTA_DUMP);
+ try {
+ DIHProperties dihPropWriter = createPropertyWriter();
+ setIndexStartTime(dihPropWriter.getCurrentTimestamp());
+ docBuilder = new DocBuilder(this, writer, dihPropWriter, requestParams);
+ checkWritablePersistFile(writer, dihPropWriter);
+ docBuilder.execute();
+ if (!requestParams.isDebug())
+ cumulativeStatistics.add(docBuilder.importStatistics);
+ } catch (Exception e) {
+ log.error("Delta Import Failed", e);
+ docBuilder.handleError("Delta Import Failed", e);
+ } finally {
+ setStatus(Status.IDLE);
+ DocBuilder.INSTANCE.set(null);
+ }
+
+ }
+
+ public void runAsync(final RequestInfo reqParams, final DIHWriter sw) {
+ new Thread(() -> runCmd(reqParams, sw)).start();
+ }
+
+ void runCmd(RequestInfo reqParams, DIHWriter sw) {
+ String command = reqParams.getCommand();
+ if (command.equals(ABORT_CMD)) {
+ if (docBuilder != null) {
+ docBuilder.abort();
+ }
+ return;
+ }
+ if (!importLock.tryLock()){
+ log.warn("Import command failed . another import is running");
+ return;
+ }
+ try {
+ if (FULL_IMPORT_CMD.equals(command) || IMPORT_CMD.equals(command)) {
+ doFullImport(sw, reqParams);
+ } else if (command.equals(DELTA_IMPORT_CMD)) {
+ doDeltaImport(sw, reqParams);
+ }
+ } finally {
+ importLock.unlock();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ Map<String, String> getStatusMessages() {
+ //this map object is a Collections.synchronizedMap(new LinkedHashMap()). if we
+ // synchronize on the object it must be safe to iterate through the map
+ Map statusMessages = (Map) retrieve(STATUS_MSGS);
+ Map<String, String> result = new LinkedHashMap<>();
+ if (statusMessages != null) {
+ synchronized (statusMessages) {
+ for (Object o : statusMessages.entrySet()) {
+ Map.Entry e = (Map.Entry) o;
+ //the toString is taken because some of the Objects create the data lazily when toString() is called
+ result.put((String) e.getKey(), e.getValue().toString());
+ }
+ }
+ }
+ return result;
+
+ }
+
+ public DocBuilder getDocBuilder() {
+ return docBuilder;
+ }
+
+ public DocBuilder getDocBuilder(DIHWriter writer, RequestInfo requestParams) {
+ DIHProperties dihPropWriter = createPropertyWriter();
+ return new DocBuilder(this, writer, dihPropWriter, requestParams);
+ }
+
+ Map<String, Evaluator> getEvaluators() {
+ return getEvaluators(config.getFunctions());
+ }
+
+ /**
+ * used by tests.
+ */
+ Map<String, Evaluator> getEvaluators(List<Map<String,String>> fn) {
+ Map<String, Evaluator> evaluators = new HashMap<>();
+ evaluators.put(Evaluator.DATE_FORMAT_EVALUATOR, new DateFormatEvaluator());
+ evaluators.put(Evaluator.SQL_ESCAPE_EVALUATOR, new SqlEscapingEvaluator());
+ evaluators.put(Evaluator.URL_ENCODE_EVALUATOR, new UrlEvaluator());
+ evaluators.put(Evaluator.ESCAPE_SOLR_QUERY_CHARS, new SolrQueryEscapingEvaluator());
+ SolrCore core = docBuilder == null ? null : docBuilder.dataImporter.getCore();
+ for (Map<String, String> map : fn) {
+ try {
+ evaluators.put(map.get(NAME), (Evaluator) loadClass(map.get(CLASS), core).newInstance());
+ } catch (Exception e) {
+ wrapAndThrow(SEVERE, e, "Unable to instantiate evaluator: " + map.get(CLASS));
+ }
+ }
+ return evaluators;
+ }
+
+ static final ThreadLocal<AtomicLong> QUERY_COUNT = new ThreadLocal<AtomicLong>() {
+ @Override
+ protected AtomicLong initialValue() {
+ return new AtomicLong();
+ }
+ };
+
+
+
+ static final class MSG {
+ public static final String NO_CONFIG_FOUND = "Configuration not found";
+
+ public static final String NO_INIT = "DataImportHandler started. Not Initialized. No commands can be run";
+
+ public static final String INVALID_CONFIG = "FATAL: Could not create importer. DataImporter config invalid";
+
+ public static final String LOAD_EXP = "Exception while loading DataImporter";
+
+ public static final String JMX_DESC = "Manage data import from databases to Solr";
+
+ public static final String CMD_RUNNING = "A command is still running...";
+
+ public static final String DEBUG_NOT_ENABLED = "Debug not enabled. Add a tag <str name=\"enableDebug\">true</str> in solrconfig.xml";
+
+ public static final String CONFIG_RELOADED = "Configuration Re-loaded sucessfully";
+
+ public static final String CONFIG_NOT_RELOADED = "Configuration NOT Re-loaded...Data Importer is busy.";
+
+ public static final String TOTAL_DOC_PROCESSED = "Total Documents Processed";
+
+ public static final String TOTAL_FAILED_DOCS = "Total Documents Failed";
+
+ public static final String TOTAL_QUERIES_EXECUTED = "Total Requests made to DataSource";
+
+ public static final String TOTAL_ROWS_EXECUTED = "Total Rows Fetched";
+
+ public static final String TOTAL_DOCS_DELETED = "Total Documents Deleted";
+
+ public static final String TOTAL_DOCS_SKIPPED = "Total Documents Skipped";
+ }
+
+ public SolrCore getCore() {
+ return core;
+ }
+
+ void putToCoreScopeSession(String key, Object val) {
+ coreScopeSession.put(key, val);
+ }
+ Object getFromCoreScopeSession(String key) {
+ return coreScopeSession.get(key);
+ }
+
+ public static final String COLUMN = "column";
+
+ public static final String TYPE = "type";
+
+ public static final String DATA_SRC = "dataSource";
+
+ public static final String MULTI_VALUED = "multiValued";
+
+ public static final String NAME = "name";
+
+ public static final String STATUS_MSGS = "status-messages";
+
+ public static final String FULL_IMPORT_CMD = "full-import";
+
+ public static final String IMPORT_CMD = "import";
+
+ public static final String DELTA_IMPORT_CMD = "delta-import";
+
+ public static final String ABORT_CMD = "abort";
+
+ public static final String DEBUG_MODE = "debug";
+
+ public static final String RELOAD_CONF_CMD = "reload-config";
+
+ public static final String SHOW_CONF_CMD = "show-config";
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java
new file mode 100644
index 0000000..e217ddd
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.Properties;
+
+/**
+ * <p>
+ * Provides data from a source with a given query.
+ * </p>
+ * <p>
+ * Implementation of this abstract class must provide a default no-arg constructor
+ * </p>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 1.3
+ */
+public abstract class DataSource<T> {
+
+ /**
+ * Initializes the DataSource with the <code>Context</code> and
+ * initialization properties.
+ * <p>
+ * This is invoked by the <code>DataImporter</code> after creating an
+ * instance of this class.
+ */
+ public abstract void init(Context context, Properties initProps);
+
+ /**
+ * Get records for the given query.The return type depends on the
+ * implementation .
+ *
+ * @param query The query string. It can be a SQL for JdbcDataSource or a URL
+ * for HttpDataSource or a file location for FileDataSource or a custom
+ * format for your own custom DataSource.
+ * @return Depends on the implementation. For instance JdbcDataSource returns
+ * an Iterator<Map <String,Object>>
+ */
+ public abstract T getData(String query);
+
+ /**
+ * Cleans up resources of this DataSource after use.
+ */
+ public abstract void close();
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java
new file mode 100644
index 0000000..f4df820
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.IllformedLocaleException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+
+import org.apache.solr.common.util.SuppressForbidden;
+import org.apache.solr.handler.dataimport.config.EntityField;
+import org.apache.solr.util.DateMathParser;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+
+/**
+ * <p>Formats values using a given date format. </p>
+ * <p>Pass three parameters:
+ * <ul>
+ * <li>An {@link EntityField} or a date expression to be parsed with
+ * the {@link DateMathParser} class If the value is in a String,
+ * then it is assumed to be a datemath expression, otherwise it
+ * resolved using a {@link VariableResolver} instance</li>
+ * <li>A date format see {@link SimpleDateFormat} for the syntax.</li>
+ * <li>The {@link Locale} to parse.
+ * (optional. Defaults to the Root Locale) </li>
+ * </ul>
+ */
+public class DateFormatEvaluator extends Evaluator {
+
+ public static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
+ protected Map<String, Locale> availableLocales = new HashMap<>();
+ protected Set<String> availableTimezones = new HashSet<>();
+
+ @SuppressForbidden(reason = "Usage of outdated locale parsing with Locale#toString() because of backwards compatibility")
+ public DateFormatEvaluator() {
+ for (Locale locale : Locale.getAvailableLocales()) {
+ availableLocales.put(locale.toString(), locale);
+ }
+ for (String tz : TimeZone.getAvailableIDs()) {
+ availableTimezones.add(tz);
+ }
+ }
+
+ private SimpleDateFormat getDateFormat(String pattern, TimeZone timezone, Locale locale) {
+ final SimpleDateFormat sdf = new SimpleDateFormat(pattern, locale);
+ sdf.setTimeZone(timezone);
+ return sdf;
+ }
+
+ @Override
+ public String evaluate(String expression, Context context) {
+ List<Object> l = parseParams(expression, context.getVariableResolver());
+ if (l.size() < 2 || l.size() > 4) {
+ throw new DataImportHandlerException(SEVERE, "'formatDate()' must have two, three or four parameters ");
+ }
+ Object o = l.get(0);
+ Object format = l.get(1);
+ if (format instanceof VariableWrapper) {
+ VariableWrapper wrapper = (VariableWrapper) format;
+ o = wrapper.resolve();
+ format = o.toString();
+ }
+ Locale locale = Locale.ENGLISH; // we default to ENGLISH for dates for full Java 9 compatibility
+ if(l.size()>2) {
+ Object localeObj = l.get(2);
+ String localeStr = null;
+ if (localeObj instanceof VariableWrapper) {
+ localeStr = ((VariableWrapper) localeObj).resolve().toString();
+ } else {
+ localeStr = localeObj.toString();
+ }
+ locale = availableLocales.get(localeStr);
+ if (locale == null) try {
+ locale = new Locale.Builder().setLanguageTag(localeStr).build();
+ } catch (IllformedLocaleException ex) {
+ throw new DataImportHandlerException(SEVERE, "Malformed / non-existent locale: " + localeStr, ex);
+ }
+ }
+ TimeZone tz = TimeZone.getDefault(); // DWS TODO: is this the right default for us? Deserves explanation if so.
+ if(l.size()==4) {
+ Object tzObj = l.get(3);
+ String tzStr = null;
+ if (tzObj instanceof VariableWrapper) {
+ tzStr = ((VariableWrapper) tzObj).resolve().toString();
+ } else {
+ tzStr = tzObj.toString();
+ }
+ if(availableTimezones.contains(tzStr)) {
+ tz = TimeZone.getTimeZone(tzStr);
+ } else {
+ throw new DataImportHandlerException(SEVERE, "Unsupported Timezone: " + tzStr);
+ }
+ }
+ String dateFmt = format.toString();
+ SimpleDateFormat fmt = getDateFormat(dateFmt, tz, locale);
+ Date date = null;
+ if (o instanceof VariableWrapper) {
+ date = evaluateWrapper((VariableWrapper) o, locale, tz);
+ } else {
+ date = evaluateString(o.toString(), locale, tz);
+ }
+ return fmt.format(date);
+ }
+
+ /**
+ * NOTE: declared as a method to allow for extensibility
+ *
+ * @lucene.experimental this API is experimental and subject to change
+ * @return the result of evaluating a string
+ */
+ protected Date evaluateWrapper(VariableWrapper variableWrapper, Locale locale, TimeZone tz) {
+ Date date = null;
+ Object variableval = resolveWrapper(variableWrapper,locale,tz);
+ if (variableval instanceof Date) {
+ date = (Date) variableval;
+ } else {
+ String s = variableval.toString();
+ try {
+ date = getDateFormat(DEFAULT_DATE_FORMAT, tz, locale).parse(s);
+ } catch (ParseException exp) {
+ wrapAndThrow(SEVERE, exp, "Invalid expression for date");
+ }
+ }
+ return date;
+ }
+
+ /**
+ * NOTE: declared as a method to allow for extensibility
+ * @lucene.experimental
+ * @return the result of evaluating a string
+ */
+ protected Date evaluateString(String datemathfmt, Locale locale, TimeZone tz) {
+ // note: DMP does not use the locale but perhaps a subclass might use it, for e.g. parsing a date in a custom
+ // string that doesn't necessarily have date math?
+ //TODO refactor DateMathParser.parseMath a bit to have a static method for this logic.
+ if (datemathfmt.startsWith("NOW")) {
+ datemathfmt = datemathfmt.substring("NOW".length());
+ }
+ try {
+ DateMathParser parser = new DateMathParser(tz);
+ parser.setNow(new Date());// thus do *not* use SolrRequestInfo
+ return parser.parseMath(datemathfmt);
+ } catch (ParseException e) {
+ throw wrapAndThrow(SEVERE, e, "Invalid expression for date");
+ }
+ }
+
+ /**
+ * NOTE: declared as a method to allow for extensibility
+ * @lucene.experimental
+ * @return the result of resolving the variable wrapper
+ */
+ protected Object resolveWrapper(VariableWrapper variableWrapper, Locale locale, TimeZone tz) {
+ return variableWrapper.resolve();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java
new file mode 100644
index 0000000..6da9cc1
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.lang.invoke.MethodHandles;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>
+ * {@link Transformer} instance which creates {@link Date} instances out of {@link String}s.
+ * </p>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * <p>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @since solr 1.3
+ */
+public class DateFormatTransformer extends Transformer {
+ private Map<String, SimpleDateFormat> fmtCache = new HashMap<>();
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Object transformRow(Map<String, Object> aRow, Context context) {
+
+ for (Map<String, String> map : context.getAllEntityFields()) {
+ Locale locale = Locale.ENGLISH; // we default to ENGLISH for dates for full Java 9 compatibility
+ String customLocale = map.get(LOCALE);
+ if (customLocale != null) {
+ try {
+ locale = new Locale.Builder().setLanguageTag(customLocale).build();
+ } catch (IllformedLocaleException e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "Invalid Locale specified: " + customLocale, e);
+ }
+ }
+
+ String fmt = map.get(DATE_TIME_FMT);
+ if (fmt == null)
+ continue;
+ VariableResolver resolver = context.getVariableResolver();
+ fmt = resolver.replaceTokens(fmt);
+ String column = map.get(DataImporter.COLUMN);
+ String srcCol = map.get(RegexTransformer.SRC_COL_NAME);
+ if (srcCol == null)
+ srcCol = column;
+ try {
+ Object o = aRow.get(srcCol);
+ if (o instanceof List) {
+ List inputs = (List) o;
+ List<Date> results = new ArrayList<>();
+ for (Object input : inputs) {
+ results.add(process(input, fmt, locale));
+ }
+ aRow.put(column, results);
+ } else {
+ if (o != null) {
+ aRow.put(column, process(o, fmt, locale));
+ }
+ }
+ } catch (ParseException e) {
+ log.warn("Could not parse a Date field ", e);
+ }
+ }
+ return aRow;
+ }
+
+ private Date process(Object value, String format, Locale locale) throws ParseException {
+ if (value == null) return null;
+ String strVal = value.toString().trim();
+ if (strVal.length() == 0)
+ return null;
+ SimpleDateFormat fmt = fmtCache.get(format);
+ if (fmt == null) {
+ fmt = new SimpleDateFormat(format, locale);
+ fmtCache.put(format, fmt);
+ }
+ return fmt.parse(strVal);
+ }
+
+ public static final String DATE_TIME_FMT = "dateTimeFormat";
+
+ public static final String LOCALE = "locale";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DebugInfo.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DebugInfo.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DebugInfo.java
new file mode 100644
index 0000000..623832f
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DebugInfo.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.StrUtils;
+
+public class DebugInfo {
+
+ private static final class ChildRollupDocs extends AbstractList<SolrInputDocument> {
+
+ private List<SolrInputDocument> delegate = new ArrayList<>();
+
+ @Override
+ public SolrInputDocument get(int index) {
+ return delegate.get(index);
+ }
+
+ @Override
+ public int size() {
+ return delegate.size();
+ }
+
+ public boolean add(SolrInputDocument e) {
+ SolrInputDocument transformed = e.deepCopy();
+ if (transformed.hasChildDocuments()) {
+ ChildRollupDocs childList = new ChildRollupDocs();
+ childList.addAll(transformed.getChildDocuments());
+ transformed.addField("_childDocuments_", childList);
+ transformed.getChildDocuments().clear();
+ }
+ return delegate.add(transformed);
+ }
+ }
+
+ public List<SolrInputDocument> debugDocuments = new ChildRollupDocs();
+
+ public NamedList<String> debugVerboseOutput = null;
+ public boolean verbose;
+
+ public DebugInfo(Map<String,Object> requestParams) {
+ verbose = StrUtils.parseBool((String) requestParams.get("verbose"), false);
+ debugVerboseOutput = new NamedList<>();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DebugLogger.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DebugLogger.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DebugLogger.java
new file mode 100644
index 0000000..2fd9303
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DebugLogger.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+import org.apache.solr.common.util.NamedList;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.text.MessageFormat;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Stack;
+
+/**
+ * <p>
+ * Implements most of the interactive development functionality
+ * </p>
+ * <p/>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @since solr 1.3
+ */
+class DebugLogger {
+ private Stack<DebugInfo> debugStack;
+
+ NamedList output;
+// private final SolrWriter writer1;
+
+ private static final String LINE = "---------------------------------------------";
+
+ private MessageFormat fmt = new MessageFormat(
+ "----------- row #{0}-------------", Locale.ROOT);
+
+ boolean enabled = true;
+
+ public DebugLogger() {
+// writer = solrWriter;
+ output = new NamedList();
+ debugStack = new Stack<DebugInfo>() {
+
+ @Override
+ public DebugInfo pop() {
+ if (size() == 1)
+ throw new DataImportHandlerException(
+ DataImportHandlerException.SEVERE, "Stack is becoming empty");
+ return super.pop();
+ }
+ };
+ debugStack.push(new DebugInfo(null, DIHLogLevels.NONE, null));
+ output = debugStack.peek().lst;
+ }
+
+ private DebugInfo peekStack() {
+ return debugStack.isEmpty() ? null : debugStack.peek();
+ }
+
+ public void log(DIHLogLevels event, String name, Object row) {
+ if (event == DIHLogLevels.DISABLE_LOGGING) {
+ enabled = false;
+ return;
+ } else if (event == DIHLogLevels.ENABLE_LOGGING) {
+ enabled = true;
+ return;
+ }
+
+ if (!enabled && event != DIHLogLevels.START_ENTITY
+ && event != DIHLogLevels.END_ENTITY) {
+ return;
+ }
+
+ if (event == DIHLogLevels.START_DOC) {
+ debugStack.push(new DebugInfo(null, DIHLogLevels.START_DOC, peekStack()));
+ } else if (DIHLogLevels.START_ENTITY == event) {
+ debugStack
+ .push(new DebugInfo(name, DIHLogLevels.START_ENTITY, peekStack()));
+ } else if (DIHLogLevels.ENTITY_OUT == event
+ || DIHLogLevels.PRE_TRANSFORMER_ROW == event) {
+ if (debugStack.peek().type == DIHLogLevels.START_ENTITY
+ || debugStack.peek().type == DIHLogLevels.START_DOC) {
+ debugStack.peek().lst.add(null, fmt.format(new Object[]{++debugStack
+ .peek().rowCount}));
+ addToNamedList(debugStack.peek().lst, row);
+ debugStack.peek().lst.add(null, LINE);
+ }
+ } else if (event == DIHLogLevels.ROW_END) {
+ popAllTransformers();
+ } else if (DIHLogLevels.END_ENTITY == event) {
+ while (debugStack.pop().type != DIHLogLevels.START_ENTITY)
+ ;
+ } else if (DIHLogLevels.END_DOC == event) {
+ while (debugStack.pop().type != DIHLogLevels.START_DOC)
+ ;
+ } else if (event == DIHLogLevels.TRANSFORMER_EXCEPTION) {
+ debugStack.push(new DebugInfo(name, event, peekStack()));
+ debugStack.peek().lst.add("EXCEPTION",
+ getStacktraceString((Exception) row));
+ } else if (DIHLogLevels.TRANSFORMED_ROW == event) {
+ debugStack.push(new DebugInfo(name, event, peekStack()));
+ debugStack.peek().lst.add(null, LINE);
+ addToNamedList(debugStack.peek().lst, row);
+ debugStack.peek().lst.add(null, LINE);
+ if (row instanceof DataImportHandlerException) {
+ DataImportHandlerException dataImportHandlerException = (DataImportHandlerException) row;
+ dataImportHandlerException.debugged = true;
+ }
+ } else if (DIHLogLevels.ENTITY_META == event) {
+ popAllTransformers();
+ debugStack.peek().lst.add(name, row);
+ } else if (DIHLogLevels.ENTITY_EXCEPTION == event) {
+ if (row instanceof DataImportHandlerException) {
+ DataImportHandlerException dihe = (DataImportHandlerException) row;
+ if (dihe.debugged)
+ return;
+ dihe.debugged = true;
+ }
+
+ popAllTransformers();
+ debugStack.peek().lst.add("EXCEPTION",
+ getStacktraceString((Exception) row));
+ }
+ }
+
+ private void popAllTransformers() {
+ while (true) {
+ DIHLogLevels type = debugStack.peek().type;
+ if (type == DIHLogLevels.START_DOC || type == DIHLogLevels.START_ENTITY)
+ break;
+ debugStack.pop();
+ }
+ }
+
+ private void addToNamedList(NamedList nl, Object row) {
+ if (row instanceof List) {
+ List list = (List) row;
+ NamedList l = new NamedList();
+ nl.add(null, l);
+ for (Object o : list) {
+ Map<String, Object> map = (Map<String, Object>) o;
+ for (Map.Entry<String, Object> entry : map.entrySet())
+ nl.add(entry.getKey(), entry.getValue());
+ }
+ } else if (row instanceof Map) {
+ Map<String, Object> map = (Map<String, Object>) row;
+ for (Map.Entry<String, Object> entry : map.entrySet())
+ nl.add(entry.getKey(), entry.getValue());
+ }
+ }
+
+ DataSource wrapDs(final DataSource ds) {
+ return new DataSource() {
+ @Override
+ public void init(Context context, Properties initProps) {
+ ds.init(context, initProps);
+ }
+
+ @Override
+ public void close() {
+ ds.close();
+ }
+
+ @Override
+ public Object getData(String query) {
+ log(DIHLogLevels.ENTITY_META, "query", query);
+ long start = System.nanoTime();
+ try {
+ return ds.getData(query);
+ } catch (DataImportHandlerException de) {
+ log(DIHLogLevels.ENTITY_EXCEPTION,
+ null, de);
+ throw de;
+ } catch (Exception e) {
+ log(DIHLogLevels.ENTITY_EXCEPTION,
+ null, e);
+ DataImportHandlerException de = new DataImportHandlerException(
+ DataImportHandlerException.SEVERE, "", e);
+ de.debugged = true;
+ throw de;
+ } finally {
+ log(DIHLogLevels.ENTITY_META, "time-taken", DocBuilder
+ .getTimeElapsedSince(start));
+ }
+ }
+ };
+ }
+
+ Transformer wrapTransformer(final Transformer t) {
+ return new Transformer() {
+ @Override
+ public Object transformRow(Map<String, Object> row, Context context) {
+ log(DIHLogLevels.PRE_TRANSFORMER_ROW, null, row);
+ String tName = getTransformerName(t);
+ Object result = null;
+ try {
+ result = t.transformRow(row, context);
+ log(DIHLogLevels.TRANSFORMED_ROW, tName, result);
+ } catch (DataImportHandlerException de) {
+ log(DIHLogLevels.TRANSFORMER_EXCEPTION, tName, de);
+ de.debugged = true;
+ throw de;
+ } catch (Exception e) {
+ log(DIHLogLevels.TRANSFORMER_EXCEPTION, tName, e);
+ DataImportHandlerException de = new DataImportHandlerException(DataImportHandlerException.SEVERE, "", e);
+ de.debugged = true;
+ throw de;
+ }
+ return result;
+ }
+ };
+ }
+
+ public static String getStacktraceString(Exception e) {
+ StringWriter sw = new StringWriter();
+ e.printStackTrace(new PrintWriter(sw));
+ return sw.toString();
+ }
+
+ static String getTransformerName(Transformer t) {
+ Class transClass = t.getClass();
+ if (t instanceof EntityProcessorWrapper.ReflectionTransformer) {
+ return ((EntityProcessorWrapper.ReflectionTransformer) t).trans;
+ }
+ if (t instanceof ScriptTransformer) {
+ ScriptTransformer scriptTransformer = (ScriptTransformer) t;
+ return "script:" + scriptTransformer.getFunctionName();
+ }
+ if (transClass.getPackage().equals(DebugLogger.class.getPackage())) {
+ return transClass.getSimpleName();
+ } else {
+ return transClass.getName();
+ }
+ }
+
+ private static class DebugInfo {
+ String name;
+
+ int tCount, rowCount;
+
+ NamedList lst;
+
+ DIHLogLevels type;
+
+ DebugInfo parent;
+
+ public DebugInfo(String name, DIHLogLevels type, DebugInfo parent) {
+ this.name = name;
+ this.type = type;
+ this.parent = parent;
+ lst = new NamedList();
+ if (parent != null) {
+ String displayName = null;
+ if (type == DIHLogLevels.START_ENTITY) {
+ displayName = "entity:" + name;
+ } else if (type == DIHLogLevels.TRANSFORMED_ROW
+ || type == DIHLogLevels.TRANSFORMER_EXCEPTION) {
+ displayName = "transformer:" + name;
+ } else if (type == DIHLogLevels.START_DOC) {
+ this.name = displayName = "document#" + SolrWriter.getDocCount();
+ }
+ parent.lst.add(displayName, lst);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
new file mode 100644
index 0000000..164cf70
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
@@ -0,0 +1,988 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.dataimport.config.ConfigNameConstants;
+import org.apache.solr.handler.dataimport.config.DIHConfiguration;
+import org.apache.solr.handler.dataimport.config.Entity;
+import org.apache.solr.handler.dataimport.config.EntityField;
+
+import static org.apache.solr.handler.dataimport.SolrWriter.LAST_INDEX_KEY;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * <p> {@link DocBuilder} is responsible for creating Solr documents out of the given configuration. It also maintains
+ * statistics information. It depends on the {@link EntityProcessor} implementations to fetch data. </p>
+ * <p>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @since solr 1.3
+ */
+public class DocBuilder {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static final AtomicBoolean WARNED_ABOUT_INDEX_TIME_BOOSTS = new AtomicBoolean();
+
+ private static final Date EPOCH = new Date(0);
+ public static final String DELETE_DOC_BY_ID = "$deleteDocById";
+ public static final String DELETE_DOC_BY_QUERY = "$deleteDocByQuery";
+ public static final String DOC_BOOST = "$docBoost";
+ public static final String SKIP_DOC = "$skipDoc";
+ public static final String SKIP_ROW = "$skipRow";
+
+ DataImporter dataImporter;
+
+ private DIHConfiguration config;
+
+ private EntityProcessorWrapper currentEntityProcessorWrapper;
+
+ @SuppressWarnings("unchecked")
+ private Map statusMessages = Collections.synchronizedMap(new LinkedHashMap());
+
+ public Statistics importStatistics = new Statistics();
+
+ DIHWriter writer;
+
+ boolean verboseDebug = false;
+
+ Map<String, Object> session = new HashMap<>();
+
+ static final ThreadLocal<DocBuilder> INSTANCE = new ThreadLocal<>();
+ private Map<String, Object> persistedProperties;
+
+ private DIHProperties propWriter;
+ private DebugLogger debugLogger;
+ private final RequestInfo reqParams;
+
+ public DocBuilder(DataImporter dataImporter, DIHWriter solrWriter, DIHProperties propWriter, RequestInfo reqParams) {
+ INSTANCE.set(this);
+ this.dataImporter = dataImporter;
+ this.reqParams = reqParams;
+ this.propWriter = propWriter;
+ DataImporter.QUERY_COUNT.set(importStatistics.queryCount);
+ verboseDebug = reqParams.isDebug() && reqParams.getDebugInfo().verbose;
+ persistedProperties = propWriter.readIndexerProperties();
+
+ writer = solrWriter;
+ ContextImpl ctx = new ContextImpl(null, null, null, null, reqParams.getRawParams(), null, this);
+ if (writer != null) {
+ writer.init(ctx);
+ }
+ }
+
+
+ DebugLogger getDebugLogger(){
+ if (debugLogger == null) {
+ debugLogger = new DebugLogger();
+ }
+ return debugLogger;
+ }
+
+ private VariableResolver getVariableResolver() {
+ try {
+ VariableResolver resolver = null;
+ String epoch = propWriter.convertDateToString(EPOCH);
+ if(dataImporter != null && dataImporter.getCore() != null
+ && dataImporter.getCore().getResourceLoader().getCoreProperties() != null){
+ resolver = new VariableResolver(dataImporter.getCore().getResourceLoader().getCoreProperties());
+ } else {
+ resolver = new VariableResolver();
+ }
+ resolver.setEvaluators(dataImporter.getEvaluators());
+ Map<String, Object> indexerNamespace = new HashMap<>();
+ if (persistedProperties.get(LAST_INDEX_TIME) != null) {
+ indexerNamespace.put(LAST_INDEX_TIME, persistedProperties.get(LAST_INDEX_TIME));
+ } else {
+ // set epoch
+ indexerNamespace.put(LAST_INDEX_TIME, epoch);
+ }
+ indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime());
+ indexerNamespace.put("request", new HashMap<>(reqParams.getRawParams()));
+ indexerNamespace.put("handlerName", dataImporter.getHandlerName());
+ for (Entity entity : dataImporter.getConfig().getEntities()) {
+ Map<String, Object> entityNamespace = new HashMap<>();
+ String key = SolrWriter.LAST_INDEX_KEY;
+ Object lastIndex = persistedProperties.get(entity.getName() + "." + key);
+ if (lastIndex != null) {
+ entityNamespace.put(SolrWriter.LAST_INDEX_KEY, lastIndex);
+ } else {
+ entityNamespace.put(SolrWriter.LAST_INDEX_KEY, epoch);
+ }
+ indexerNamespace.put(entity.getName(), entityNamespace);
+ }
+ resolver.addNamespace(ConfigNameConstants.IMPORTER_NS_SHORT, indexerNamespace);
+ resolver.addNamespace(ConfigNameConstants.IMPORTER_NS, indexerNamespace);
+ return resolver;
+ } catch (Exception e) {
+ wrapAndThrow(SEVERE, e);
+ // unreachable statement
+ return null;
+ }
+ }
+
+ private void invokeEventListener(String className) {
+ invokeEventListener(className, null);
+ }
+
+
+ private void invokeEventListener(String className, Exception lastException) {
+ try {
+ EventListener listener = (EventListener) loadClass(className, dataImporter.getCore()).newInstance();
+ notifyListener(listener, lastException);
+ } catch (Exception e) {
+ wrapAndThrow(SEVERE, e, "Unable to load class : " + className);
+ }
+ }
+
+ private void notifyListener(EventListener listener, Exception lastException) {
+ String currentProcess;
+ if (dataImporter.getStatus() == DataImporter.Status.RUNNING_DELTA_DUMP) {
+ currentProcess = Context.DELTA_DUMP;
+ } else {
+ currentProcess = Context.FULL_DUMP;
+ }
+ ContextImpl ctx = new ContextImpl(null, getVariableResolver(), null, currentProcess, session, null, this);
+ ctx.setLastException(lastException);
+ listener.onEvent(ctx);
+ }
+
+ @SuppressWarnings("unchecked")
+ public void execute() {
+ List<EntityProcessorWrapper> epwList = null;
+ try {
+ dataImporter.store(DataImporter.STATUS_MSGS, statusMessages);
+ config = dataImporter.getConfig();
+ final AtomicLong startTime = new AtomicLong(System.nanoTime());
+ statusMessages.put(TIME_ELAPSED, new Object() {
+ @Override
+ public String toString() {
+ return getTimeElapsedSince(startTime.get());
+ }
+ });
+
+ statusMessages.put(DataImporter.MSG.TOTAL_QUERIES_EXECUTED,
+ importStatistics.queryCount);
+ statusMessages.put(DataImporter.MSG.TOTAL_ROWS_EXECUTED,
+ importStatistics.rowsCount);
+ statusMessages.put(DataImporter.MSG.TOTAL_DOC_PROCESSED,
+ importStatistics.docCount);
+ statusMessages.put(DataImporter.MSG.TOTAL_DOCS_SKIPPED,
+ importStatistics.skipDocCount);
+
+ List<String> entities = reqParams.getEntitiesToRun();
+
+ // Trigger onImportStart
+ if (config.getOnImportStart() != null) {
+ invokeEventListener(config.getOnImportStart());
+ }
+ AtomicBoolean fullCleanDone = new AtomicBoolean(false);
+ //we must not do a delete of *:* multiple times if there are multiple root entities to be run
+ Map<String,Object> lastIndexTimeProps = new HashMap<>();
+ lastIndexTimeProps.put(LAST_INDEX_KEY, dataImporter.getIndexStartTime());
+
+ epwList = new ArrayList<>(config.getEntities().size());
+ for (Entity e : config.getEntities()) {
+ epwList.add(getEntityProcessorWrapper(e));
+ }
+ for (EntityProcessorWrapper epw : epwList) {
+ if (entities != null && !entities.contains(epw.getEntity().getName()))
+ continue;
+ lastIndexTimeProps.put(epw.getEntity().getName() + "." + LAST_INDEX_KEY, propWriter.getCurrentTimestamp());
+ currentEntityProcessorWrapper = epw;
+ String delQuery = epw.getEntity().getAllAttributes().get("preImportDeleteQuery");
+ if (dataImporter.getStatus() == DataImporter.Status.RUNNING_DELTA_DUMP) {
+ cleanByQuery(delQuery, fullCleanDone);
+ doDelta();
+ delQuery = epw.getEntity().getAllAttributes().get("postImportDeleteQuery");
+ if (delQuery != null) {
+ fullCleanDone.set(false);
+ cleanByQuery(delQuery, fullCleanDone);
+ }
+ } else {
+ cleanByQuery(delQuery, fullCleanDone);
+ doFullDump();
+ delQuery = epw.getEntity().getAllAttributes().get("postImportDeleteQuery");
+ if (delQuery != null) {
+ fullCleanDone.set(false);
+ cleanByQuery(delQuery, fullCleanDone);
+ }
+ }
+ }
+
+ if (stop.get()) {
+ // Dont commit if aborted using command=abort
+ statusMessages.put("Aborted", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).format(new Date()));
+ handleError("Aborted", null);
+ } else {
+ // Do not commit unnecessarily if this is a delta-import and no documents were created or deleted
+ if (!reqParams.isClean()) {
+ if (importStatistics.docCount.get() > 0 || importStatistics.deletedDocCount.get() > 0) {
+ finish(lastIndexTimeProps);
+ }
+ } else {
+ // Finished operation normally, commit now
+ finish(lastIndexTimeProps);
+ }
+
+ if (config.getOnImportEnd() != null) {
+ invokeEventListener(config.getOnImportEnd());
+ }
+ }
+
+ statusMessages.remove(TIME_ELAPSED);
+ statusMessages.put(DataImporter.MSG.TOTAL_DOC_PROCESSED, ""+ importStatistics.docCount.get());
+ if(importStatistics.failedDocCount.get() > 0)
+ statusMessages.put(DataImporter.MSG.TOTAL_FAILED_DOCS, ""+ importStatistics.failedDocCount.get());
+
+ statusMessages.put("Time taken", getTimeElapsedSince(startTime.get()));
+ log.info("Time taken = " + getTimeElapsedSince(startTime.get()));
+ } catch(Exception e)
+ {
+ throw new RuntimeException(e);
+ } finally
+ {
+ if (writer != null) {
+ writer.close();
+ }
+ if (epwList != null) {
+ closeEntityProcessorWrappers(epwList);
+ }
+ if(reqParams.isDebug()) {
+ reqParams.getDebugInfo().debugVerboseOutput = getDebugLogger().output;
+ }
+ }
+ }
+ private void closeEntityProcessorWrappers(List<EntityProcessorWrapper> epwList) {
+ for(EntityProcessorWrapper epw : epwList) {
+ epw.close();
+ if(epw.getDatasource()!=null) {
+ epw.getDatasource().close();
+ }
+ closeEntityProcessorWrappers(epw.getChildren());
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void finish(Map<String,Object> lastIndexTimeProps) {
+ log.info("Import completed successfully");
+ statusMessages.put("", "Indexing completed. Added/Updated: "
+ + importStatistics.docCount + " documents. Deleted "
+ + importStatistics.deletedDocCount + " documents.");
+ if(reqParams.isCommit()) {
+ writer.commit(reqParams.isOptimize());
+ addStatusMessage("Committed");
+ if (reqParams.isOptimize())
+ addStatusMessage("Optimized");
+ }
+ try {
+ propWriter.persist(lastIndexTimeProps);
+ } catch (Exception e) {
+ log.error("Could not write property file", e);
+ statusMessages.put("error", "Could not write property file. Delta imports will not work. " +
+ "Make sure your conf directory is writable");
+ }
+ }
+
+ void handleError(String message, Exception e) {
+ if (!dataImporter.getCore().getCoreContainer().isZooKeeperAware()) {
+ writer.rollback();
+ }
+
+ statusMessages.put(message, "Indexing error");
+ addStatusMessage(message);
+ if ((config != null) && (config.getOnError() != null)) {
+ invokeEventListener(config.getOnError(), e);
+ }
+ }
+
+ private void doFullDump() {
+ addStatusMessage("Full Dump Started");
+ buildDocument(getVariableResolver(), null, null, currentEntityProcessorWrapper, true, null);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void doDelta() {
+ addStatusMessage("Delta Dump started");
+ VariableResolver resolver = getVariableResolver();
+
+ if (config.getDeleteQuery() != null) {
+ writer.deleteByQuery(config.getDeleteQuery());
+ }
+
+ addStatusMessage("Identifying Delta");
+ log.info("Starting delta collection.");
+ Set<Map<String, Object>> deletedKeys = new HashSet<>();
+ Set<Map<String, Object>> allPks = collectDelta(currentEntityProcessorWrapper, resolver, deletedKeys);
+ if (stop.get())
+ return;
+ addStatusMessage("Deltas Obtained");
+ addStatusMessage("Building documents");
+ if (!deletedKeys.isEmpty()) {
+ allPks.removeAll(deletedKeys);
+ deleteAll(deletedKeys);
+ // Make sure that documents are not re-created
+ }
+ deletedKeys = null;
+ writer.setDeltaKeys(allPks);
+
+ statusMessages.put("Total Changed Documents", allPks.size());
+ VariableResolver vri = getVariableResolver();
+ Iterator<Map<String, Object>> pkIter = allPks.iterator();
+ while (pkIter.hasNext()) {
+ Map<String, Object> map = pkIter.next();
+ vri.addNamespace(ConfigNameConstants.IMPORTER_NS_SHORT + ".delta", map);
+ buildDocument(vri, null, map, currentEntityProcessorWrapper, true, null);
+ pkIter.remove();
+ // check for abort
+ if (stop.get())
+ break;
+ }
+
+ if (!stop.get()) {
+ log.info("Delta Import completed successfully");
+ }
+ }
+
+ private void deleteAll(Set<Map<String, Object>> deletedKeys) {
+ log.info("Deleting stale documents ");
+ Iterator<Map<String, Object>> iter = deletedKeys.iterator();
+ while (iter.hasNext()) {
+ Map<String, Object> map = iter.next();
+ String keyName = currentEntityProcessorWrapper.getEntity().isDocRoot() ? currentEntityProcessorWrapper.getEntity().getPk() : currentEntityProcessorWrapper.getEntity().getSchemaPk();
+ Object key = map.get(keyName);
+ if(key == null) {
+ keyName = findMatchingPkColumn(keyName, map);
+ key = map.get(keyName);
+ }
+ if(key == null) {
+ log.warn("no key was available for deleted pk query. keyName = " + keyName);
+ continue;
+ }
+ writer.deleteDoc(key);
+ importStatistics.deletedDocCount.incrementAndGet();
+ iter.remove();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public void addStatusMessage(String msg) {
+ statusMessages.put(msg, new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).format(new Date()));
+ }
+
+ private void resetEntity(EntityProcessorWrapper epw) {
+ epw.setInitialized(false);
+ for (EntityProcessorWrapper child : epw.getChildren()) {
+ resetEntity(child);
+ }
+
+ }
+
+ private void buildDocument(VariableResolver vr, DocWrapper doc,
+ Map<String,Object> pk, EntityProcessorWrapper epw, boolean isRoot,
+ ContextImpl parentCtx) {
+ List<EntityProcessorWrapper> entitiesToDestroy = new ArrayList<>();
+ try {
+ buildDocument(vr, doc, pk, epw, isRoot, parentCtx, entitiesToDestroy);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ for (EntityProcessorWrapper entityWrapper : entitiesToDestroy) {
+ entityWrapper.destroy();
+ }
+ resetEntity(epw);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void buildDocument(VariableResolver vr, DocWrapper doc,
+ Map<String, Object> pk, EntityProcessorWrapper epw, boolean isRoot,
+ ContextImpl parentCtx, List<EntityProcessorWrapper> entitiesToDestroy) {
+
+ ContextImpl ctx = new ContextImpl(epw, vr, null,
+ pk == null ? Context.FULL_DUMP : Context.DELTA_DUMP,
+ session, parentCtx, this);
+ epw.init(ctx);
+ if (!epw.isInitialized()) {
+ entitiesToDestroy.add(epw);
+ epw.setInitialized(true);
+ }
+
+ if (reqParams.getStart() > 0) {
+ getDebugLogger().log(DIHLogLevels.DISABLE_LOGGING, null, null);
+ }
+
+ if (verboseDebug) {
+ getDebugLogger().log(DIHLogLevels.START_ENTITY, epw.getEntity().getName(), null);
+ }
+
+ int seenDocCount = 0;
+
+ try {
+ while (true) {
+ if (stop.get())
+ return;
+ if(importStatistics.docCount.get() > (reqParams.getStart() + reqParams.getRows())) break;
+ try {
+ seenDocCount++;
+
+ if (seenDocCount > reqParams.getStart()) {
+ getDebugLogger().log(DIHLogLevels.ENABLE_LOGGING, null, null);
+ }
+
+ if (verboseDebug && epw.getEntity().isDocRoot()) {
+ getDebugLogger().log(DIHLogLevels.START_DOC, epw.getEntity().getName(), null);
+ }
+ if (doc == null && epw.getEntity().isDocRoot()) {
+ doc = new DocWrapper();
+ ctx.setDoc(doc);
+ Entity e = epw.getEntity();
+ while (e.getParentEntity() != null) {
+ addFields(e.getParentEntity(), doc, (Map<String, Object>) vr
+ .resolve(e.getParentEntity().getName()), vr);
+ e = e.getParentEntity();
+ }
+ }
+
+ Map<String, Object> arow = epw.nextRow();
+ if (arow == null) {
+ break;
+ }
+
+ // Support for start parameter in debug mode
+ if (epw.getEntity().isDocRoot()) {
+ if (seenDocCount <= reqParams.getStart())
+ continue;
+ if (seenDocCount > reqParams.getStart() + reqParams.getRows()) {
+ log.info("Indexing stopped at docCount = " + importStatistics.docCount);
+ break;
+ }
+ }
+
+ if (verboseDebug) {
+ getDebugLogger().log(DIHLogLevels.ENTITY_OUT, epw.getEntity().getName(), arow);
+ }
+ importStatistics.rowsCount.incrementAndGet();
+
+ DocWrapper childDoc = null;
+ if (doc != null) {
+ if (epw.getEntity().isChild()) {
+ childDoc = new DocWrapper();
+ handleSpecialCommands(arow, childDoc);
+ addFields(epw.getEntity(), childDoc, arow, vr);
+ doc.addChildDocument(childDoc);
+ } else {
+ handleSpecialCommands(arow, doc);
+ vr.addNamespace(epw.getEntity().getName(), arow);
+ addFields(epw.getEntity(), doc, arow, vr);
+ vr.removeNamespace(epw.getEntity().getName());
+ }
+ }
+ if (epw.getEntity().getChildren() != null) {
+ vr.addNamespace(epw.getEntity().getName(), arow);
+ for (EntityProcessorWrapper child : epw.getChildren()) {
+ if (childDoc != null) {
+ buildDocument(vr, childDoc,
+ child.getEntity().isDocRoot() ? pk : null, child, false, ctx, entitiesToDestroy);
+ } else {
+ buildDocument(vr, doc,
+ child.getEntity().isDocRoot() ? pk : null, child, false, ctx, entitiesToDestroy);
+ }
+ }
+ vr.removeNamespace(epw.getEntity().getName());
+ }
+ if (epw.getEntity().isDocRoot()) {
+ if (stop.get())
+ return;
+ if (!doc.isEmpty()) {
+ boolean result = writer.upload(doc);
+ if(reqParams.isDebug()) {
+ reqParams.getDebugInfo().debugDocuments.add(doc);
+ }
+ doc = null;
+ if (result){
+ importStatistics.docCount.incrementAndGet();
+ } else {
+ importStatistics.failedDocCount.incrementAndGet();
+ }
+ }
+ }
+ } catch (DataImportHandlerException e) {
+ if (verboseDebug) {
+ getDebugLogger().log(DIHLogLevels.ENTITY_EXCEPTION, epw.getEntity().getName(), e);
+ }
+ if(e.getErrCode() == DataImportHandlerException.SKIP_ROW){
+ continue;
+ }
+ if (isRoot) {
+ if (e.getErrCode() == DataImportHandlerException.SKIP) {
+ importStatistics.skipDocCount.getAndIncrement();
+ doc = null;
+ } else {
+ SolrException.log(log, "Exception while processing: "
+ + epw.getEntity().getName() + " document : " + doc, e);
+ }
+ if (e.getErrCode() == DataImportHandlerException.SEVERE)
+ throw e;
+ } else
+ throw e;
+ } catch (Exception t) {
+ if (verboseDebug) {
+ getDebugLogger().log(DIHLogLevels.ENTITY_EXCEPTION, epw.getEntity().getName(), t);
+ }
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE, t);
+ } finally {
+ if (verboseDebug) {
+ getDebugLogger().log(DIHLogLevels.ROW_END, epw.getEntity().getName(), null);
+ if (epw.getEntity().isDocRoot())
+ getDebugLogger().log(DIHLogLevels.END_DOC, null, null);
+ }
+ }
+ }
+ } finally {
+ if (verboseDebug) {
+ getDebugLogger().log(DIHLogLevels.END_ENTITY, null, null);
+ }
+ }
+ }
+
+ static class DocWrapper extends SolrInputDocument {
+ //final SolrInputDocument solrDocument = new SolrInputDocument();
+ Map<String ,Object> session;
+
+ public void setSessionAttribute(String key, Object val){
+ if(session == null) session = new HashMap<>();
+ session.put(key, val);
+ }
+
+ public Object getSessionAttribute(String key) {
+ return session == null ? null : session.get(key);
+ }
+ }
+
+ private void handleSpecialCommands(Map<String, Object> arow, DocWrapper doc) {
+ Object value = arow.get(DELETE_DOC_BY_ID);
+ if (value != null) {
+ if (value instanceof Collection) {
+ Collection collection = (Collection) value;
+ for (Object o : collection) {
+ writer.deleteDoc(o.toString());
+ importStatistics.deletedDocCount.incrementAndGet();
+ }
+ } else {
+ writer.deleteDoc(value);
+ importStatistics.deletedDocCount.incrementAndGet();
+ }
+ }
+ value = arow.get(DELETE_DOC_BY_QUERY);
+ if (value != null) {
+ if (value instanceof Collection) {
+ Collection collection = (Collection) value;
+ for (Object o : collection) {
+ writer.deleteByQuery(o.toString());
+ importStatistics.deletedDocCount.incrementAndGet();
+ }
+ } else {
+ writer.deleteByQuery(value.toString());
+ importStatistics.deletedDocCount.incrementAndGet();
+ }
+ }
+ value = arow.get(DOC_BOOST);
+ if (value != null) {
+ String message = "Ignoring document boost: " + value + " as index-time boosts are not supported anymore";
+ if (WARNED_ABOUT_INDEX_TIME_BOOSTS.compareAndSet(false, true)) {
+ log.warn(message);
+ } else {
+ log.debug(message);
+ }
+ }
+
+ value = arow.get(SKIP_DOC);
+ if (value != null) {
+ if (Boolean.parseBoolean(value.toString())) {
+ throw new DataImportHandlerException(DataImportHandlerException.SKIP,
+ "Document skipped :" + arow);
+ }
+ }
+
+ value = arow.get(SKIP_ROW);
+ if (value != null) {
+ if (Boolean.parseBoolean(value.toString())) {
+ throw new DataImportHandlerException(DataImportHandlerException.SKIP_ROW);
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void addFields(Entity entity, DocWrapper doc,
+ Map<String, Object> arow, VariableResolver vr) {
+ for (Map.Entry<String, Object> entry : arow.entrySet()) {
+ String key = entry.getKey();
+ Object value = entry.getValue();
+ if (value == null) continue;
+ if (key.startsWith("$")) continue;
+ Set<EntityField> field = entity.getColNameVsField().get(key);
+ IndexSchema schema = null == reqParams.getRequest() ? null : reqParams.getRequest().getSchema();
+ if (field == null && schema != null) {
+ // This can be a dynamic field or a field which does not have an entry in data-config ( an implicit field)
+ SchemaField sf = schema.getFieldOrNull(key);
+ if (sf == null) {
+ sf = config.getSchemaField(key);
+ }
+ if (sf != null) {
+ addFieldToDoc(entry.getValue(), sf.getName(), sf.multiValued(), doc);
+ }
+ //else do nothing. if we add it it may fail
+ } else {
+ if (field != null) {
+ for (EntityField f : field) {
+ String name = f.getName();
+ boolean multiValued = f.isMultiValued();
+ boolean toWrite = f.isToWrite();
+ if(f.isDynamicName()){
+ name = vr.replaceTokens(name);
+ SchemaField schemaField = config.getSchemaField(name);
+ if(schemaField == null) {
+ toWrite = false;
+ } else {
+ multiValued = schemaField.multiValued();
+ toWrite = true;
+ }
+ }
+ if (toWrite) {
+ addFieldToDoc(entry.getValue(), name, multiValued, doc);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private void addFieldToDoc(Object value, String name, boolean multiValued, DocWrapper doc) {
+ if (value instanceof Collection) {
+ Collection collection = (Collection) value;
+ if (multiValued) {
+ for (Object o : collection) {
+ if (o != null)
+ doc.addField(name, o);
+ }
+ } else {
+ if (doc.getField(name) == null)
+ for (Object o : collection) {
+ if (o != null) {
+ doc.addField(name, o);
+ break;
+ }
+ }
+ }
+ } else if (multiValued) {
+ if (value != null) {
+ doc.addField(name, value);
+ }
+ } else {
+ if (doc.getField(name) == null && value != null)
+ doc.addField(name, value);
+ }
+ }
+
+ public EntityProcessorWrapper getEntityProcessorWrapper(Entity entity) {
+ EntityProcessor entityProcessor = null;
+ if (entity.getProcessorName() == null) {
+ entityProcessor = new SqlEntityProcessor();
+ } else {
+ try {
+ entityProcessor = (EntityProcessor) loadClass(entity.getProcessorName(), dataImporter.getCore())
+ .newInstance();
+ } catch (Exception e) {
+ wrapAndThrow (SEVERE,e,
+ "Unable to load EntityProcessor implementation for entity:" + entity.getName());
+ }
+ }
+ EntityProcessorWrapper epw = new EntityProcessorWrapper(entityProcessor, entity, this);
+ for(Entity e1 : entity.getChildren()) {
+ epw.getChildren().add(getEntityProcessorWrapper(e1));
+ }
+
+ return epw;
+ }
+
+ private String findMatchingPkColumn(String pk, Map<String, Object> row) {
+ if (row.containsKey(pk)) {
+ throw new IllegalArgumentException(String.format(Locale.ROOT,
+ "deltaQuery returned a row with null for primary key %s", pk));
+ }
+ String resolvedPk = null;
+ for (String columnName : row.keySet()) {
+ if (columnName.endsWith("." + pk) || pk.endsWith("." + columnName)) {
+ if (resolvedPk != null)
+ throw new IllegalArgumentException(
+ String.format(Locale.ROOT,
+ "deltaQuery has more than one column (%s and %s) that might resolve to declared primary key pk='%s'",
+ resolvedPk, columnName, pk));
+ resolvedPk = columnName;
+ }
+ }
+ if (resolvedPk == null) {
+ throw new IllegalArgumentException(
+ String
+ .format(
+ Locale.ROOT,
+ "deltaQuery has no column to resolve to declared primary key pk='%s'",
+ pk));
+ }
+ log.info(String.format(Locale.ROOT,
+ "Resolving deltaQuery column '%s' to match entity's declared pk '%s'",
+ resolvedPk, pk));
+ return resolvedPk;
+ }
+
+ /**
+ * <p> Collects unique keys of all Solr documents for whom one or more source tables have been changed since the last
+ * indexed time. </p> <p> Note: In our definition, unique key of Solr document is the primary key of the top level
+ * entity (unless skipped using docRoot=false) in the Solr document in data-config.xml </p>
+ *
+ * @return an iterator to the list of keys for which Solr documents should be updated.
+ */
+ @SuppressWarnings("unchecked")
+ public Set<Map<String, Object>> collectDelta(EntityProcessorWrapper epw, VariableResolver resolver,
+ Set<Map<String, Object>> deletedRows) {
+ //someone called abort
+ if (stop.get())
+ return new HashSet();
+
+ ContextImpl context1 = new ContextImpl(epw, resolver, null, Context.FIND_DELTA, session, null, this);
+ epw.init(context1);
+
+ Set<Map<String, Object>> myModifiedPks = new HashSet<>();
+
+
+
+ for (EntityProcessorWrapper childEpw : epw.getChildren()) {
+ //this ensures that we start from the leaf nodes
+ myModifiedPks.addAll(collectDelta(childEpw, resolver, deletedRows));
+ //someone called abort
+ if (stop.get())
+ return new HashSet();
+ }
+
+ // identifying the modified rows for this entity
+ Map<String, Map<String, Object>> deltaSet = new HashMap<>();
+ log.info("Running ModifiedRowKey() for Entity: " + epw.getEntity().getName());
+ //get the modified rows in this entity
+ String pk = epw.getEntity().getPk();
+ while (true) {
+ Map<String, Object> row = epw.nextModifiedRowKey();
+
+ if (row == null)
+ break;
+
+ Object pkValue = row.get(pk);
+ if (pkValue == null) {
+ pk = findMatchingPkColumn(pk, row);
+ pkValue = row.get(pk);
+ }
+
+ deltaSet.put(pkValue.toString(), row);
+ importStatistics.rowsCount.incrementAndGet();
+ // check for abort
+ if (stop.get())
+ return new HashSet();
+ }
+ //get the deleted rows for this entity
+ Set<Map<String, Object>> deletedSet = new HashSet<>();
+ while (true) {
+ Map<String, Object> row = epw.nextDeletedRowKey();
+ if (row == null)
+ break;
+
+ deletedSet.add(row);
+
+ Object pkValue = row.get(pk);
+ if (pkValue == null) {
+ pk = findMatchingPkColumn(pk, row);
+ pkValue = row.get(pk);
+ }
+
+ // Remove deleted rows from the delta rows
+ String deletedRowPk = pkValue.toString();
+ if (deltaSet.containsKey(deletedRowPk)) {
+ deltaSet.remove(deletedRowPk);
+ }
+
+ importStatistics.rowsCount.incrementAndGet();
+ // check for abort
+ if (stop.get())
+ return new HashSet();
+ }
+
+ log.info("Completed ModifiedRowKey for Entity: " + epw.getEntity().getName() + " rows obtained : " + deltaSet.size());
+ log.info("Completed DeletedRowKey for Entity: " + epw.getEntity().getName() + " rows obtained : " + deletedSet.size());
+
+ myModifiedPks.addAll(deltaSet.values());
+ Set<Map<String, Object>> parentKeyList = new HashSet<>();
+ //all that we have captured is useless (in a sub-entity) if no rows in the parent is modified because of these
+ //propogate up the changes in the chain
+ if (epw.getEntity().getParentEntity() != null) {
+ // identifying deleted rows with deltas
+
+ for (Map<String, Object> row : myModifiedPks) {
+ resolver.addNamespace(epw.getEntity().getName(), row);
+ getModifiedParentRows(resolver, epw.getEntity().getName(), epw, parentKeyList);
+ // check for abort
+ if (stop.get())
+ return new HashSet();
+ }
+ // running the same for deletedrows
+ for (Map<String, Object> row : deletedSet) {
+ resolver.addNamespace(epw.getEntity().getName(), row);
+ getModifiedParentRows(resolver, epw.getEntity().getName(), epw, parentKeyList);
+ // check for abort
+ if (stop.get())
+ return new HashSet();
+ }
+ }
+ log.info("Completed parentDeltaQuery for Entity: " + epw.getEntity().getName());
+ if (epw.getEntity().isDocRoot())
+ deletedRows.addAll(deletedSet);
+
+ // Do not use entity.isDocRoot here because one of descendant entities may set rootEntity="true"
+ return epw.getEntity().getParentEntity() == null ?
+ myModifiedPks : new HashSet<>(parentKeyList);
+ }
+
+ private void getModifiedParentRows(VariableResolver resolver,
+ String entity, EntityProcessor entityProcessor,
+ Set<Map<String, Object>> parentKeyList) {
+ try {
+ while (true) {
+ Map<String, Object> parentRow = entityProcessor
+ .nextModifiedParentRowKey();
+ if (parentRow == null)
+ break;
+
+ parentKeyList.add(parentRow);
+ importStatistics.rowsCount.incrementAndGet();
+ // check for abort
+ if (stop.get())
+ return;
+ }
+
+ } finally {
+ resolver.removeNamespace(entity);
+ }
+ }
+
+ public void abort() {
+ stop.set(true);
+ }
+
+ private AtomicBoolean stop = new AtomicBoolean(false);
+
+ public static final String TIME_ELAPSED = "Time Elapsed";
+
+ static String getTimeElapsedSince(long l) {
+ l = TimeUnit.MILLISECONDS.convert(System.nanoTime() - l, TimeUnit.NANOSECONDS);
+ return (l / (60000 * 60)) + ":" + (l / 60000) % 60 + ":" + (l / 1000)
+ % 60 + "." + l % 1000;
+ }
+
+ public RequestInfo getReqParams() {
+ return reqParams;
+ }
+
+ @SuppressWarnings("unchecked")
+ static Class loadClass(String name, SolrCore core) throws ClassNotFoundException {
+ try {
+ return core != null ?
+ core.getResourceLoader().findClass(name, Object.class) :
+ Class.forName(name);
+ } catch (Exception e) {
+ try {
+ String n = DocBuilder.class.getPackage().getName() + "." + name;
+ return core != null ?
+ core.getResourceLoader().findClass(n, Object.class) :
+ Class.forName(n);
+ } catch (Exception e1) {
+ throw new ClassNotFoundException("Unable to load " + name + " or " + DocBuilder.class.getPackage().getName() + "." + name, e);
+ }
+ }
+ }
+
+ public static class Statistics {
+ public AtomicLong docCount = new AtomicLong();
+
+ public AtomicLong deletedDocCount = new AtomicLong();
+
+ public AtomicLong failedDocCount = new AtomicLong();
+
+ public AtomicLong rowsCount = new AtomicLong();
+
+ public AtomicLong queryCount = new AtomicLong();
+
+ public AtomicLong skipDocCount = new AtomicLong();
+
+ public Statistics add(Statistics stats) {
+ this.docCount.addAndGet(stats.docCount.get());
+ this.deletedDocCount.addAndGet(stats.deletedDocCount.get());
+ this.rowsCount.addAndGet(stats.rowsCount.get());
+ this.queryCount.addAndGet(stats.queryCount.get());
+
+ return this;
+ }
+
+ public Map<String, Object> getStatsSnapshot() {
+ Map<String, Object> result = new HashMap<>();
+ result.put("docCount", docCount.get());
+ result.put("deletedDocCount", deletedDocCount.get());
+ result.put("rowCount", rowsCount.get());
+ result.put("queryCount", rowsCount.get());
+ result.put("skipDocCount", skipDocCount.get());
+ return result;
+ }
+
+ }
+
+ private void cleanByQuery(String delQuery, AtomicBoolean completeCleanDone) {
+ delQuery = getVariableResolver().replaceTokens(delQuery);
+ if (reqParams.isClean()) {
+ if (delQuery == null && !completeCleanDone.get()) {
+ writer.doDeleteAll();
+ completeCleanDone.set(true);
+ } else if (delQuery != null) {
+ writer.deleteByQuery(delQuery);
+ }
+ }
+ }
+
+ public static final String LAST_INDEX_TIME = "last_index_time";
+ public static final String INDEX_START_TIME = "index_start_time";
+}
[14/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/MockDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/MockDataSource.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/MockDataSource.java
new file mode 100644
index 0000000..8989ea2
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/MockDataSource.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * <p>
+ * A mock DataSource implementation which can be used for testing.
+ * </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 1.3
+ */
+public class MockDataSource extends
+ DataSource<Iterator<Map<String, Object>>> {
+
+ private static Map<String, Iterator<Map<String, Object>>> cache = new HashMap<>();
+
+ public static void setIterator(String query,
+ Iterator<Map<String, Object>> iter) {
+ cache.put(query, iter);
+ }
+
+ public static void clearCache() {
+ cache.clear();
+ }
+
+ @Override
+ public void init(Context context, Properties initProps) {
+ }
+
+ @Override
+ public Iterator<Map<String, Object>> getData(String query) {
+ return cache.get(query);
+ }
+
+ @Override
+ public void close() {
+ cache.clear();
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java
new file mode 100644
index 0000000..349b14e
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+import java.text.NumberFormat;
+import java.text.ParseException;
+import java.text.ParsePosition;
+import java.util.ArrayList;
+import java.util.IllformedLocaleException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+/**
+ * <p>
+ * A {@link Transformer} instance which can extract numbers out of strings. It uses
+ * {@link NumberFormat} class to parse strings and supports
+ * Number, Integer, Currency and Percent styles as supported by
+ * {@link NumberFormat} with configurable locales.
+ * </p>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 1.3
+ */
+public class NumberFormatTransformer extends Transformer {
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Object transformRow(Map<String, Object> row, Context context) {
+ for (Map<String, String> fld : context.getAllEntityFields()) {
+ String style = context.replaceTokens(fld.get(FORMAT_STYLE));
+ if (style != null) {
+ String column = fld.get(DataImporter.COLUMN);
+ String srcCol = fld.get(RegexTransformer.SRC_COL_NAME);
+ String localeStr = context.replaceTokens(fld.get(LOCALE));
+ if (srcCol == null)
+ srcCol = column;
+ Locale locale = Locale.ROOT;
+ if (localeStr != null) {
+ try {
+ locale = new Locale.Builder().setLanguageTag(localeStr).build();
+ } catch (IllformedLocaleException e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Invalid Locale '" + localeStr + "' specified for field: " + fld, e);
+ }
+ }
+
+ Object val = row.get(srcCol);
+ String styleSmall = style.toLowerCase(Locale.ROOT);
+
+ if (val instanceof List) {
+ List<String> inputs = (List) val;
+ List results = new ArrayList();
+ for (String input : inputs) {
+ try {
+ results.add(process(input, styleSmall, locale));
+ } catch (ParseException e) {
+ throw new DataImportHandlerException(
+ DataImportHandlerException.SEVERE,
+ "Failed to apply NumberFormat on column: " + column, e);
+ }
+ }
+ row.put(column, results);
+ } else {
+ if (val == null || val.toString().trim().equals(""))
+ continue;
+ try {
+ row.put(column, process(val.toString(), styleSmall, locale));
+ } catch (ParseException e) {
+ throw new DataImportHandlerException(
+ DataImportHandlerException.SEVERE,
+ "Failed to apply NumberFormat on column: " + column, e);
+ }
+ }
+ }
+ }
+ return row;
+ }
+
+ private Number process(String val, String style, Locale locale) throws ParseException {
+ if (INTEGER.equals(style)) {
+ return parseNumber(val, NumberFormat.getIntegerInstance(locale));
+ } else if (NUMBER.equals(style)) {
+ return parseNumber(val, NumberFormat.getNumberInstance(locale));
+ } else if (CURRENCY.equals(style)) {
+ return parseNumber(val, NumberFormat.getCurrencyInstance(locale));
+ } else if (PERCENT.equals(style)) {
+ return parseNumber(val, NumberFormat.getPercentInstance(locale));
+ }
+
+ return null;
+ }
+
+ private Number parseNumber(String val, NumberFormat numFormat) throws ParseException {
+ ParsePosition parsePos = new ParsePosition(0);
+ Number num = numFormat.parse(val, parsePos);
+ if (parsePos.getIndex() != val.length()) {
+ throw new ParseException("illegal number format", parsePos.getIndex());
+ }
+ return num;
+ }
+
+ public static final String FORMAT_STYLE = "formatStyle";
+
+ public static final String LOCALE = "locale";
+
+ public static final String NUMBER = "number";
+
+ public static final String PERCENT = "percent";
+
+ public static final String INTEGER = "integer";
+
+ public static final String CURRENCY = "currency";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/PlainTextEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/PlainTextEntityProcessor.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/PlainTextEntityProcessor.java
new file mode 100644
index 0000000..c75608c
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/PlainTextEntityProcessor.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+import static org.apache.solr.handler.dataimport.XPathEntityProcessor.URL;
+import org.apache.commons.io.IOUtils;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringWriter;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * <p>An implementation of {@link EntityProcessor} which reads data from a url/file and give out a row which contains one String
+ * value. The name of the field is 'plainText'.
+ *
+ * @since solr 1.4
+ */
+public class PlainTextEntityProcessor extends EntityProcessorBase {
+ private boolean ended = false;
+
+ @Override
+ public void init(Context context) {
+ super.init(context);
+ ended = false;
+ }
+
+ @Override
+ public Map<String, Object> nextRow() {
+ if (ended) return null;
+ DataSource<Reader> ds = context.getDataSource();
+ String url = context.replaceTokens(context.getEntityAttribute(URL));
+ Reader r = null;
+ try {
+ r = ds.getData(url);
+ } catch (Exception e) {
+ wrapAndThrow(SEVERE, e, "Exception reading url : " + url);
+ }
+ StringWriter sw = new StringWriter();
+ char[] buf = new char[1024];
+ while (true) {
+ int len = 0;
+ try {
+ len = r.read(buf);
+ } catch (IOException e) {
+ IOUtils.closeQuietly(r);
+ wrapAndThrow(SEVERE, e, "Exception reading url : " + url);
+ }
+ if (len <= 0) break;
+ sw.append(new String(buf, 0, len));
+ }
+ Map<String, Object> row = new HashMap<>();
+ row.put(PLAIN_TEXT, sw.toString());
+ ended = true;
+ IOUtils.closeQuietly(r);
+ return row;
+ }
+
+ public static final String PLAIN_TEXT = "plainText";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/RegexTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/RegexTransformer.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/RegexTransformer.java
new file mode 100644
index 0000000..7a919de
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/RegexTransformer.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.lang.invoke.MethodHandles;
+import java.util.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * <p>
+ * A {@link Transformer} implementation which uses Regular Expressions to extract, split
+ * and replace data in fields.
+ * </p>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 1.3
+ * @see Pattern
+ */
+public class RegexTransformer extends Transformer {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Map<String, Object> transformRow(Map<String, Object> row,
+ Context ctx) {
+ List<Map<String, String>> fields = ctx.getAllEntityFields();
+ for (Map<String, String> field : fields) {
+ String col = field.get(DataImporter.COLUMN);
+ String reStr = ctx.replaceTokens(field.get(REGEX));
+ String splitBy = ctx.replaceTokens(field.get(SPLIT_BY));
+ String replaceWith = ctx.replaceTokens(field.get(REPLACE_WITH));
+ String groupNames = ctx.replaceTokens(field.get(GROUP_NAMES));
+ if (reStr != null || splitBy != null) {
+ String srcColName = field.get(SRC_COL_NAME);
+ if (srcColName == null) {
+ srcColName = col;
+ }
+ Object tmpVal = row.get(srcColName);
+ if (tmpVal == null)
+ continue;
+
+ if (tmpVal instanceof List) {
+ List<String> inputs = (List<String>) tmpVal;
+ List results = new ArrayList();
+ Map<String,List> otherVars= null;
+ for (String input : inputs) {
+ Object o = process(col, reStr, splitBy, replaceWith, input, groupNames);
+ if (o != null){
+ if (o instanceof Map) {
+ Map map = (Map) o;
+ for (Object e : map.entrySet()) {
+ Map.Entry<String ,Object> entry = (Map.Entry<String, Object>) e;
+ List l = results;
+ if(!col.equals(entry.getKey())){
+ if(otherVars == null) otherVars = new HashMap<>();
+ l = otherVars.get(entry.getKey());
+ if(l == null){
+ l = new ArrayList();
+ otherVars.put(entry.getKey(), l);
+ }
+ }
+ if (entry.getValue() instanceof Collection) {
+ l.addAll((Collection) entry.getValue());
+ } else {
+ l.add(entry.getValue());
+ }
+ }
+ } else {
+ if (o instanceof Collection) {
+ results.addAll((Collection) o);
+ } else {
+ results.add(o);
+ }
+ }
+ }
+ }
+ row.put(col, results);
+ if(otherVars != null) row.putAll(otherVars);
+ } else {
+ String value = tmpVal.toString();
+ Object o = process(col, reStr, splitBy, replaceWith, value, groupNames);
+ if (o != null){
+ if (o instanceof Map) {
+ row.putAll((Map) o);
+ } else{
+ row.put(col, o);
+ }
+ }
+ }
+ }
+ }
+ return row;
+ }
+
+ private Object process(String col, String reStr, String splitBy,
+ String replaceWith, String value, String groupNames) {
+ if (splitBy != null) {
+ return readBySplit(splitBy, value);
+ } else if (replaceWith != null) {
+ Pattern p = getPattern(reStr);
+ Matcher m = p.matcher(value);
+ return m.find() ? m.replaceAll(replaceWith) : value;
+ } else {
+ return readfromRegExp(reStr, value, col, groupNames);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<String> readBySplit(String splitBy, String value) {
+ String[] vals = value.split(splitBy);
+ List<String> l = new ArrayList<>();
+ l.addAll(Arrays.asList(vals));
+ return l;
+ }
+
+ @SuppressWarnings("unchecked")
+ private Object readfromRegExp(String reStr, String value, String columnName, String gNames) {
+ String[] groupNames = null;
+ if(gNames != null && gNames.trim().length() >0){
+ groupNames = gNames.split(",");
+ }
+ Pattern regexp = getPattern(reStr);
+ Matcher m = regexp.matcher(value);
+ if (m.find() && m.groupCount() > 0) {
+ if (m.groupCount() > 1) {
+ List l = null;
+ Map<String ,String > map = null;
+ if(groupNames == null){
+ l = new ArrayList();
+ } else {
+ map = new HashMap<>();
+ }
+ for (int i = 1; i <= m.groupCount(); i++) {
+ try {
+ if(l != null){
+ l.add(m.group(i));
+ } else if (map != null ){
+ if(i <= groupNames.length){
+ String nameOfGroup = groupNames[i-1];
+ if(nameOfGroup != null && nameOfGroup.trim().length() >0){
+ map.put(nameOfGroup, m.group(i));
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.warn("Parsing failed for field : " + columnName, e);
+ }
+ }
+ return l == null ? map: l;
+ } else {
+ return m.group(1);
+ }
+ }
+
+ return null;
+ }
+
+ private Pattern getPattern(String reStr) {
+ Pattern result = PATTERN_CACHE.get(reStr);
+ if (result == null) {
+ PATTERN_CACHE.put(reStr, result = Pattern.compile(reStr));
+ }
+ return result;
+ }
+
+ private HashMap<String, Pattern> PATTERN_CACHE = new HashMap<>();
+
+ public static final String REGEX = "regex";
+
+ public static final String REPLACE_WITH = "replaceWith";
+
+ public static final String SPLIT_BY = "splitBy";
+
+ public static final String SRC_COL_NAME = "sourceColName";
+
+ public static final String GROUP_NAMES = "groupNames";
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/RequestInfo.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/RequestInfo.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/RequestInfo.java
new file mode 100644
index 0000000..d3f1a56
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/RequestInfo.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.request.SolrQueryRequest;
+
+public class RequestInfo {
+ private final String command;
+ private final boolean debug;
+ private final boolean syncMode;
+ private final boolean commit;
+ private final boolean optimize;
+ private final int start;
+ private final long rows;
+ private final boolean clean;
+ private final List<String> entitiesToRun;
+ private final Map<String,Object> rawParams;
+ private final String configFile;
+ private final String dataConfig;
+ private final SolrQueryRequest request;
+
+ //TODO: find a different home for these two...
+ private final ContentStream contentStream;
+ private final DebugInfo debugInfo;
+
+ public RequestInfo(SolrQueryRequest request, Map<String,Object> requestParams, ContentStream stream) {
+ this.request = request;
+ this.contentStream = stream;
+ if (requestParams.containsKey("command")) {
+ command = (String) requestParams.get("command");
+ } else {
+ command = null;
+ }
+ boolean debugMode = StrUtils.parseBool((String) requestParams.get("debug"), false);
+ if (debugMode) {
+ debug = true;
+ debugInfo = new DebugInfo(requestParams);
+ } else {
+ debug = false;
+ debugInfo = null;
+ }
+ if (requestParams.containsKey("clean")) {
+ clean = StrUtils.parseBool( (String) requestParams.get("clean"), true);
+ } else if (DataImporter.DELTA_IMPORT_CMD.equals(command) || DataImporter.IMPORT_CMD.equals(command)) {
+ clean = false;
+ } else {
+ clean = debug ? false : true;
+ }
+ optimize = StrUtils.parseBool((String) requestParams.get("optimize"), false);
+ if(optimize) {
+ commit = true;
+ } else {
+ commit = StrUtils.parseBool( (String) requestParams.get("commit"), (debug ? false : true));
+ }
+ if (requestParams.containsKey("rows")) {
+ rows = Integer.parseInt((String) requestParams.get("rows"));
+ } else {
+ rows = debug ? 10 : Long.MAX_VALUE;
+ }
+
+ if (requestParams.containsKey("start")) {
+ start = Integer.parseInt((String) requestParams.get("start"));
+ } else {
+ start = 0;
+ }
+ syncMode = StrUtils.parseBool((String) requestParams.get("synchronous"), false);
+
+ Object o = requestParams.get("entity");
+ List<String> modifiableEntities = null;
+ if(o != null) {
+ if (o instanceof String) {
+ modifiableEntities = new ArrayList<>();
+ modifiableEntities.add((String) o);
+ } else if (o instanceof List<?>) {
+ @SuppressWarnings("unchecked")
+ List<String> modifiableEntities1 = new ArrayList<>((List<String>) o);
+ modifiableEntities = modifiableEntities1;
+ }
+ entitiesToRun = Collections.unmodifiableList(modifiableEntities);
+ } else {
+ entitiesToRun = null;
+ }
+ String configFileParam = (String) requestParams.get("config");
+ configFile = configFileParam;
+ String dataConfigParam = (String) requestParams.get("dataConfig");
+ if (dataConfigParam != null && dataConfigParam.trim().length() == 0) {
+ // Empty data-config param is not valid, change it to null
+ dataConfigParam = null;
+ }
+ dataConfig = dataConfigParam;
+ this.rawParams = Collections.unmodifiableMap(new HashMap<>(requestParams));
+ }
+
+ public String getCommand() {
+ return command;
+ }
+
+ public boolean isDebug() {
+ return debug;
+ }
+
+ public boolean isSyncMode() {
+ return syncMode;
+ }
+
+ public boolean isCommit() {
+ return commit;
+ }
+
+ public boolean isOptimize() {
+ return optimize;
+ }
+
+ public int getStart() {
+ return start;
+ }
+
+ public long getRows() {
+ return rows;
+ }
+
+ public boolean isClean() {
+ return clean;
+ }
+ /**
+ * Returns null if we are to run all entities, otherwise just run the entities named in the list.
+ */
+ public List<String> getEntitiesToRun() {
+ return entitiesToRun;
+ }
+
+ public String getDataConfig() {
+ return dataConfig;
+ }
+
+ public Map<String,Object> getRawParams() {
+ return rawParams;
+ }
+
+ public ContentStream getContentStream() {
+ return contentStream;
+ }
+
+ public DebugInfo getDebugInfo() {
+ return debugInfo;
+ }
+
+ public String getConfigFile() {
+ return configFile;
+ }
+
+ public SolrQueryRequest getRequest() {
+ return request;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ScriptTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ScriptTransformer.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ScriptTransformer.java
new file mode 100644
index 0000000..165d76d
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ScriptTransformer.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+
+import java.util.Map;
+
+import javax.script.Invocable;
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineManager;
+import javax.script.ScriptException;
+
+/**
+ * <p>
+ * A {@link Transformer} instance capable of executing functions written in scripting
+ * languages as a {@link Transformer} instance.
+ * </p>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 1.3
+ */
+public class ScriptTransformer extends Transformer {
+ private Invocable engine;
+ private String functionName;
+
+ @Override
+ public Object transformRow(Map<String, Object> row, Context context) {
+ try {
+ if (engine == null)
+ initEngine(context);
+ if (engine == null)
+ return row;
+ return engine.invokeFunction(functionName, new Object[]{row, context});
+ } catch (DataImportHandlerException e) {
+ throw e;
+ } catch (Exception e) {
+ wrapAndThrow(SEVERE,e, "Error invoking script for entity " + context.getEntityAttribute("name"));
+ }
+ //will not reach here
+ return null;
+ }
+
+ private void initEngine(Context context) {
+ String scriptText = context.getScript();
+ String scriptLang = context.getScriptLanguage();
+ if (scriptText == null) {
+ throw new DataImportHandlerException(SEVERE,
+ "<script> tag is not present under <dataConfig>");
+ }
+ ScriptEngineManager scriptEngineMgr = new ScriptEngineManager();
+ ScriptEngine scriptEngine = scriptEngineMgr.getEngineByName(scriptLang);
+ if (scriptEngine == null) {
+ throw new DataImportHandlerException(SEVERE,
+ "Cannot load Script Engine for language: " + scriptLang);
+ }
+ if (scriptEngine instanceof Invocable) {
+ engine = (Invocable) scriptEngine;
+ } else {
+ throw new DataImportHandlerException(SEVERE,
+ "The installed ScriptEngine for: " + scriptLang
+ + " does not implement Invocable. Class is "
+ + scriptEngine.getClass().getName());
+ }
+ try {
+ scriptEngine.eval(scriptText);
+ } catch (ScriptException e) {
+ wrapAndThrow(SEVERE, e, "'eval' failed with language: " + scriptLang
+ + " and script: \n" + scriptText);
+ }
+ }
+
+ public void setFunctionName(String methodName) {
+ this.functionName = methodName;
+ }
+
+ public String getFunctionName() {
+ return functionName;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java
new file mode 100644
index 0000000..1ee18ef
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.security.AccessControlException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.IllformedLocaleException;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.lucene.util.IOUtils;
+import org.apache.solr.common.util.SuppressForbidden;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrResourceLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+/**
+ * <p>
+ * Writes properties using {@link Properties#store} .
+ * The special property "last_index_time" is converted to a formatted date.
+ * Users can configure the location, filename, locale and date format to use.
+ * </p>
+ */
+public class SimplePropertiesWriter extends DIHProperties {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ static final String LAST_INDEX_KEY = "last_index_time";
+
+ protected String filename = null;
+
+ protected String configDir = null;
+
+ protected Locale locale = null;
+
+ protected SimpleDateFormat dateFormat = null;
+
+ /**
+ * The locale to use when writing the properties file. Default is {@link Locale#ROOT}
+ */
+ public static final String LOCALE = "locale";
+ /**
+ * The date format to use when writing values for "last_index_time" to the properties file.
+ * See {@link SimpleDateFormat} for patterns. Default is yyyy-MM-dd HH:mm:ss .
+ */
+ public static final String DATE_FORMAT = "dateFormat";
+ /**
+ * The directory to save the properties file in. Default is the current core's "config" directory.
+ */
+ public static final String DIRECTORY = "directory";
+ /**
+ * The filename to save the properties file to. Default is this Handler's name from solrconfig.xml.
+ */
+ public static final String FILENAME = "filename";
+
+ @Override
+ public void init(DataImporter dataImporter, Map<String, String> params) {
+ if(params.get(FILENAME) != null) {
+ filename = params.get(FILENAME);
+ } else if(dataImporter.getHandlerName()!=null) {
+ filename = dataImporter.getHandlerName() + ".properties";
+ } else {
+ filename = "dataimport.properties";
+ }
+ findDirectory(dataImporter, params);
+ if(params.get(LOCALE) != null) {
+ locale = getLocale(params.get(LOCALE));
+ } else {
+ locale = Locale.ROOT;
+ }
+ if(params.get(DATE_FORMAT) != null) {
+ dateFormat = new SimpleDateFormat(params.get(DATE_FORMAT), locale);
+ } else {
+ dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", locale);
+ }
+ }
+
+ @SuppressForbidden(reason = "Usage of outdated locale parsing with Locale#toString() because of backwards compatibility")
+ private Locale getLocale(String name) {
+ if (name == null) {
+ return Locale.ROOT;
+ }
+ for (final Locale l : Locale.getAvailableLocales()) {
+ if(name.equals(l.toString()) || name.equals(l.getDisplayName(Locale.ROOT))) {
+ return locale;
+ }
+ }
+ try {
+ return new Locale.Builder().setLanguageTag(name).build();
+ } catch (IllformedLocaleException ex) {
+ throw new DataImportHandlerException(SEVERE, "Unsupported locale for PropertyWriter: " + name);
+ }
+ }
+
+ protected void findDirectory(DataImporter dataImporter, Map<String, String> params) {
+ if(params.get(DIRECTORY) != null) {
+ configDir = params.get(DIRECTORY);
+ } else {
+ SolrCore core = dataImporter.getCore();
+ if (core == null) {
+ configDir = SolrResourceLoader.locateSolrHome().toString();
+ } else {
+ configDir = core.getResourceLoader().getConfigDir();
+ }
+ }
+ }
+
+ private File getPersistFile() {
+ final File filePath;
+ if (new File(filename).isAbsolute() || configDir == null) {
+ filePath = new File(filename);
+ } else {
+ filePath = new File(new File(configDir), filename);
+ }
+ return filePath;
+ }
+
+ @Override
+ public boolean isWritable() {
+ File persistFile = getPersistFile();
+ try {
+ return persistFile.exists()
+ ? persistFile.canWrite()
+ : persistFile.getParentFile().canWrite();
+ } catch (AccessControlException e) {
+ return false;
+ }
+ }
+
+ @Override
+ public String convertDateToString(Date d) {
+ return dateFormat.format(d);
+ }
+ protected Date convertStringToDate(String s) {
+ try {
+ return dateFormat.parse(s);
+ } catch (ParseException e) {
+ throw new DataImportHandlerException(SEVERE, "Value for "
+ + LAST_INDEX_KEY + " is invalid for date format "
+ + dateFormat.toLocalizedPattern() + " : " + s);
+ }
+ }
+ /**
+ * {@link DocBuilder} sends the date as an Object because
+ * this class knows how to convert it to a String
+ */
+ protected Properties mapToProperties(Map<String,Object> propObjs) {
+ Properties p = new Properties();
+ for(Map.Entry<String,Object> entry : propObjs.entrySet()) {
+ String key = entry.getKey();
+ String val = null;
+ String lastKeyPart = key;
+ int lastDotPos = key.lastIndexOf('.');
+ if(lastDotPos!=-1 && key.length() > lastDotPos+1) {
+ lastKeyPart = key.substring(lastDotPos + 1);
+ }
+ if(LAST_INDEX_KEY.equals(lastKeyPart) && entry.getValue() instanceof Date) {
+ val = convertDateToString((Date) entry.getValue());
+ } else {
+ val = entry.getValue().toString();
+ }
+ p.put(key, val);
+ }
+ return p;
+ }
+ /**
+ * We'll send everything back as Strings as this class has
+ * already converted them.
+ */
+ protected Map<String,Object> propertiesToMap(Properties p) {
+ Map<String,Object> theMap = new HashMap<>();
+ for(Map.Entry<Object,Object> entry : p.entrySet()) {
+ String key = entry.getKey().toString();
+ Object val = entry.getValue().toString();
+ theMap.put(key, val);
+ }
+ return theMap;
+ }
+
+ @Override
+ public void persist(Map<String, Object> propObjs) {
+ Writer propOutput = null;
+ Properties existingProps = mapToProperties(readIndexerProperties());
+ Properties newProps = mapToProperties(propObjs);
+ try {
+ existingProps.putAll(newProps);
+ propOutput = new OutputStreamWriter(new FileOutputStream(getPersistFile()), StandardCharsets.UTF_8);
+ existingProps.store(propOutput, null);
+ log.info("Wrote last indexed time to " + filename);
+ } catch (Exception e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Unable to persist Index Start Time", e);
+ } finally {
+ IOUtils.closeWhileHandlingException(propOutput);
+ }
+ }
+
+ @Override
+ public Map<String, Object> readIndexerProperties() {
+ Properties props = new Properties();
+ InputStream propInput = null;
+ try {
+ String filePath = configDir;
+ if (configDir != null && !configDir.endsWith(File.separator)) {
+ filePath += File.separator;
+ }
+ filePath += filename;
+ propInput = new FileInputStream(filePath);
+ props.load(new InputStreamReader(propInput, StandardCharsets.UTF_8));
+ log.info("Read " + filename);
+ } catch (Exception e) {
+ log.warn("Unable to read: " + filename);
+ } finally {
+ IOUtils.closeWhileHandlingException(propInput);
+ }
+ return propertiesToMap(props);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java
new file mode 100644
index 0000000..7732673
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.http.client.HttpClient;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder;
+import org.apache.solr.client.solrj.impl.XMLResponseParser;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.CursorMarkParams;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>
+ * An implementation of {@link EntityProcessor} which fetches values from a
+ * separate Solr implementation using the SolrJ client library. Yield a row per
+ * Solr document.
+ * </p>
+ * <p>
+ * Limitations:
+ * All configuration is evaluated at the beginning;
+ * Only one query is walked;
+ * </p>
+ */
+public class SolrEntityProcessor extends EntityProcessorBase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static final String SOLR_SERVER = "url";
+ public static final String QUERY = "query";
+ public static final String TIMEOUT = "timeout";
+
+ public static final int TIMEOUT_SECS = 5 * 60; // 5 minutes
+ public static final int ROWS_DEFAULT = 50;
+
+ private SolrClient solrClient = null;
+ private String queryString;
+ private int rows = ROWS_DEFAULT;
+ private String[] filterQueries;
+ private String[] fields;
+ private String requestHandler;// 'qt' param
+ private int timeout = TIMEOUT_SECS;
+
+ @Override
+ public void destroy() {
+ try {
+ solrClient.close();
+ } catch (IOException e) {
+
+ } finally {
+ HttpClientUtil.close(((HttpSolrClient) solrClient).getHttpClient());
+ }
+ }
+
+ /**
+ * Factory method that returns a {@link HttpClient} instance used for interfacing with a source Solr service.
+ * One can override this method to return a differently configured {@link HttpClient} instance.
+ * For example configure https and http authentication.
+ *
+ * @return a {@link HttpClient} instance used for interfacing with a source Solr service
+ */
+ protected HttpClient getHttpClient() {
+ return HttpClientUtil.createClient(null);
+ }
+
+ @Override
+ protected void firstInit(Context context) {
+ super.firstInit(context);
+
+ try {
+ String serverPath = context.getResolvedEntityAttribute(SOLR_SERVER);
+ if (serverPath == null) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "SolrEntityProcessor: parameter 'url' is required");
+ }
+
+ HttpClient client = getHttpClient();
+ URL url = new URL(serverPath);
+ // (wt="javabin|xml") default is javabin
+ if ("xml".equals(context.getResolvedEntityAttribute(CommonParams.WT))) {
+ // TODO: it doesn't matter for this impl when passing a client currently, but we should close this!
+ solrClient = new Builder(url.toExternalForm())
+ .withHttpClient(client)
+ .withResponseParser(new XMLResponseParser())
+ .build();
+ log.info("using XMLResponseParser");
+ } else {
+ // TODO: it doesn't matter for this impl when passing a client currently, but we should close this!
+ solrClient = new Builder(url.toExternalForm())
+ .withHttpClient(client)
+ .build();
+ log.info("using BinaryResponseParser");
+ }
+ } catch (MalformedURLException e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE, e);
+ }
+ }
+
+ @Override
+ public Map<String,Object> nextRow() {
+ buildIterator();
+ return getNext();
+ }
+
+ /**
+ * The following method changes the rowIterator mutable field. It requires
+ * external synchronization.
+ */
+ protected void buildIterator() {
+ if (rowIterator != null) {
+ SolrDocumentListIterator documentListIterator = (SolrDocumentListIterator) rowIterator;
+ if (!documentListIterator.hasNext() && documentListIterator.hasMoreRows()) {
+ nextPage();
+ }
+ } else {
+ boolean cursor = Boolean.parseBoolean(context
+ .getResolvedEntityAttribute(CursorMarkParams.CURSOR_MARK_PARAM));
+ rowIterator = !cursor ? new SolrDocumentListIterator(new SolrDocumentList())
+ : new SolrDocumentListCursor(new SolrDocumentList(), CursorMarkParams.CURSOR_MARK_START);
+ nextPage();
+ }
+ }
+
+ protected void nextPage() {
+ ((SolrDocumentListIterator)rowIterator).doQuery();
+ }
+
+ class SolrDocumentListCursor extends SolrDocumentListIterator {
+
+ private final String cursorMark;
+
+ public SolrDocumentListCursor(SolrDocumentList solrDocumentList, String cursorMark) {
+ super(solrDocumentList);
+ this.cursorMark = cursorMark;
+ }
+
+ @Override
+ protected void passNextPage(SolrQuery solrQuery) {
+ String timeoutAsString = context.getResolvedEntityAttribute(TIMEOUT);
+ if (timeoutAsString != null) {
+ throw new DataImportHandlerException(SEVERE,"cursorMark can't be used with timeout");
+ }
+
+ solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark);
+ }
+
+ @Override
+ protected Iterator<Map<String,Object>> createNextPageIterator(QueryResponse response) {
+ return
+ new SolrDocumentListCursor(response.getResults(),
+ response.getNextCursorMark()) ;
+ }
+ }
+
+ class SolrDocumentListIterator implements Iterator<Map<String,Object>> {
+
+ private final int start;
+ private final int size;
+ private final long numFound;
+ private final Iterator<SolrDocument> solrDocumentIterator;
+
+ public SolrDocumentListIterator(SolrDocumentList solrDocumentList) {
+ this.solrDocumentIterator = solrDocumentList.iterator();
+ this.numFound = solrDocumentList.getNumFound();
+ // SolrQuery has the start field of type int while SolrDocumentList of
+ // type long. We are always querying with an int so we can't receive a
+ // long as output. That's the reason why the following cast seems safe
+ this.start = (int) solrDocumentList.getStart();
+ this.size = solrDocumentList.size();
+ }
+
+ protected QueryResponse doQuery() {
+ SolrEntityProcessor.this.queryString = context.getResolvedEntityAttribute(QUERY);
+ if (SolrEntityProcessor.this.queryString == null) {
+ throw new DataImportHandlerException(
+ DataImportHandlerException.SEVERE,
+ "SolrEntityProcessor: parameter 'query' is required"
+ );
+ }
+
+ String rowsP = context.getResolvedEntityAttribute(CommonParams.ROWS);
+ if (rowsP != null) {
+ rows = Integer.parseInt(rowsP);
+ }
+
+ String sortParam = context.getResolvedEntityAttribute(CommonParams.SORT);
+
+ String fqAsString = context.getResolvedEntityAttribute(CommonParams.FQ);
+ if (fqAsString != null) {
+ SolrEntityProcessor.this.filterQueries = fqAsString.split(",");
+ }
+
+ String fieldsAsString = context.getResolvedEntityAttribute(CommonParams.FL);
+ if (fieldsAsString != null) {
+ SolrEntityProcessor.this.fields = fieldsAsString.split(",");
+ }
+ SolrEntityProcessor.this.requestHandler = context.getResolvedEntityAttribute(CommonParams.QT);
+
+
+ SolrQuery solrQuery = new SolrQuery(queryString);
+ solrQuery.setRows(rows);
+
+ if (sortParam!=null) {
+ solrQuery.setParam(CommonParams.SORT, sortParam);
+ }
+
+ passNextPage(solrQuery);
+
+ if (fields != null) {
+ for (String field : fields) {
+ solrQuery.addField(field);
+ }
+ }
+ solrQuery.setRequestHandler(requestHandler);
+ solrQuery.setFilterQueries(filterQueries);
+
+
+ QueryResponse response = null;
+ try {
+ response = solrClient.query(solrQuery);
+ } catch (SolrServerException | IOException | SolrException e) {
+ if (ABORT.equals(onError)) {
+ wrapAndThrow(SEVERE, e);
+ } else if (SKIP.equals(onError)) {
+ wrapAndThrow(DataImportHandlerException.SKIP_ROW, e);
+ }
+ }
+
+ if (response != null) {
+ SolrEntityProcessor.this.rowIterator = createNextPageIterator(response);
+ }
+ return response;
+ }
+
+ protected Iterator<Map<String,Object>> createNextPageIterator(QueryResponse response) {
+ return new SolrDocumentListIterator(response.getResults());
+ }
+
+ protected void passNextPage(SolrQuery solrQuery) {
+ String timeoutAsString = context.getResolvedEntityAttribute(TIMEOUT);
+ if (timeoutAsString != null) {
+ SolrEntityProcessor.this.timeout = Integer.parseInt(timeoutAsString);
+ }
+
+ solrQuery.setTimeAllowed(timeout * 1000);
+
+ solrQuery.setStart(getStart() + getSize());
+ }
+
+ @Override
+ public boolean hasNext() {
+ return solrDocumentIterator.hasNext();
+ }
+
+ @Override
+ public Map<String,Object> next() {
+ SolrDocument solrDocument = solrDocumentIterator.next();
+
+ HashMap<String,Object> map = new HashMap<>();
+ Collection<String> fields = solrDocument.getFieldNames();
+ for (String field : fields) {
+ Object fieldValue = solrDocument.getFieldValue(field);
+ map.put(field, fieldValue);
+ }
+ return map;
+ }
+
+ public int getStart() {
+ return start;
+ }
+
+ public int getSize() {
+ return size;
+ }
+
+ public boolean hasMoreRows() {
+ return numFound > start + size;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrQueryEscapingEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrQueryEscapingEvaluator.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrQueryEscapingEvaluator.java
new file mode 100644
index 0000000..aece031
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrQueryEscapingEvaluator.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+
+import java.util.List;
+
+import org.apache.solr.client.solrj.util.ClientUtils;
+
+public class SolrQueryEscapingEvaluator extends Evaluator {
+ @Override
+ public String evaluate(String expression, Context context) {
+ List<Object> l = parseParams(expression, context.getVariableResolver());
+ if (l.size() != 1) {
+ throw new DataImportHandlerException(SEVERE, "'escapeQueryChars' must have at least one parameter ");
+ }
+ String s = l.get(0).toString();
+ return ClientUtils.escapeQueryChars(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java
new file mode 100644
index 0000000..3964f3f
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.update.CommitUpdateCommand;
+import org.apache.solr.update.DeleteUpdateCommand;
+import org.apache.solr.update.RollbackUpdateCommand;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * <p> Writes documents to SOLR. </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @since solr 1.3
+ */
+public class SolrWriter extends DIHWriterBase implements DIHWriter {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static final String LAST_INDEX_KEY = "last_index_time";
+
+ private final UpdateRequestProcessor processor;
+ private final int commitWithin;
+
+ SolrQueryRequest req;
+
+ public SolrWriter(UpdateRequestProcessor processor, SolrQueryRequest req) {
+ this.processor = processor;
+ this.req = req;
+ commitWithin = (req != null) ? req.getParams().getInt(UpdateParams.COMMIT_WITHIN, -1): -1;
+ }
+
+ @Override
+ public void close() {
+ try {
+ processor.finish();
+ } catch (IOException e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Unable to call finish() on UpdateRequestProcessor", e);
+ } finally {
+ deltaKeys = null;
+ try {
+ processor.close();
+ } catch (IOException e) {
+ SolrException.log(log, e);
+ }
+ }
+ }
+ @Override
+ public boolean upload(SolrInputDocument d) {
+ try {
+ AddUpdateCommand command = new AddUpdateCommand(req);
+ command.solrDoc = d;
+ command.commitWithin = commitWithin;
+ processor.processAdd(command);
+ } catch (Exception e) {
+ log.warn("Error creating document : " + d, e);
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public void deleteDoc(Object id) {
+ try {
+ log.info("Deleting document: " + id);
+ DeleteUpdateCommand delCmd = new DeleteUpdateCommand(req);
+ delCmd.setId(id.toString());
+ processor.processDelete(delCmd);
+ } catch (IOException e) {
+ log.error("Exception while deleteing: " + id, e);
+ }
+ }
+
+ @Override
+ public void deleteByQuery(String query) {
+ try {
+ log.info("Deleting documents from Solr with query: " + query);
+ DeleteUpdateCommand delCmd = new DeleteUpdateCommand(req);
+ delCmd.query = query;
+ processor.processDelete(delCmd);
+ } catch (IOException e) {
+ log.error("Exception while deleting by query: " + query, e);
+ }
+ }
+
+ @Override
+ public void commit(boolean optimize) {
+ try {
+ CommitUpdateCommand commit = new CommitUpdateCommand(req,optimize);
+ processor.processCommit(commit);
+ } catch (Exception e) {
+ log.error("Exception while solr commit.", e);
+ }
+ }
+
+ @Override
+ public void rollback() {
+ try {
+ RollbackUpdateCommand rollback = new RollbackUpdateCommand(req);
+ processor.processRollback(rollback);
+ } catch (Exception e) {
+ log.error("Exception during rollback command.", e);
+ }
+ }
+
+ @Override
+ public void doDeleteAll() {
+ try {
+ DeleteUpdateCommand deleteCommand = new DeleteUpdateCommand(req);
+ deleteCommand.query = "*:*";
+ processor.processDelete(deleteCommand);
+ } catch (IOException e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Exception in full dump while deleting all documents.", e);
+ }
+ }
+
+ static String getResourceAsString(InputStream in) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream(1024);
+ byte[] buf = new byte[1024];
+ int sz = 0;
+ try {
+ while ((sz = in.read(buf)) != -1) {
+ baos.write(buf, 0, sz);
+ }
+ } finally {
+ try {
+ in.close();
+ } catch (Exception e) {
+
+ }
+ }
+ return new String(baos.toByteArray(), StandardCharsets.UTF_8);
+ }
+
+ static String getDocCount() {
+ if (DocBuilder.INSTANCE.get() != null) {
+ return ""
+ + (DocBuilder.INSTANCE.get().importStatistics.docCount.get() + 1);
+ } else {
+ return null;
+ }
+ }
+ @Override
+ public void init(Context context) {
+ /* NO-OP */
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SortedMapBackedCache.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SortedMapBackedCache.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SortedMapBackedCache.java
new file mode 100644
index 0000000..bb84ba9
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SortedMapBackedCache.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+public class SortedMapBackedCache implements DIHCache {
+ private SortedMap<Object,List<Map<String,Object>>> theMap = null;
+ private boolean isOpen = false;
+ private boolean isReadOnly = false;
+ String primaryKeyName = null;
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void add(Map<String,Object> rec) {
+ checkOpen(true);
+ checkReadOnly();
+
+ if (rec == null || rec.size() == 0) {
+ return;
+ }
+
+ if (primaryKeyName == null) {
+ primaryKeyName = rec.keySet().iterator().next();
+ }
+
+ Object pk = rec.get(primaryKeyName);
+ if (pk instanceof Collection<?>) {
+ Collection<Object> c = (Collection<Object>) pk;
+ if (c.size() != 1) {
+ throw new RuntimeException(
+ "The primary key must have exactly 1 element.");
+ }
+ pk = c.iterator().next();
+ }
+ //Rows with null keys are not added.
+ if(pk==null) {
+ return;
+ }
+ List<Map<String,Object>> thisKeysRecs = theMap.get(pk);
+ if (thisKeysRecs == null) {
+ thisKeysRecs = new ArrayList<>();
+ theMap.put(pk, thisKeysRecs);
+ }
+ thisKeysRecs.add(rec);
+ }
+
+ private void checkOpen(boolean shouldItBe) {
+ if (!isOpen && shouldItBe) {
+ throw new IllegalStateException(
+ "Must call open() before using this cache.");
+ }
+ if (isOpen && !shouldItBe) {
+ throw new IllegalStateException("The cache is already open.");
+ }
+ }
+
+ private void checkReadOnly() {
+ if (isReadOnly) {
+ throw new IllegalStateException("Cache is read-only.");
+ }
+ }
+
+ @Override
+ public void close() {
+ isOpen = false;
+ }
+
+ @Override
+ public void delete(Object key) {
+ checkOpen(true);
+ checkReadOnly();
+ if(key==null) {
+ return;
+ }
+ theMap.remove(key);
+ }
+
+ @Override
+ public void deleteAll() {
+ deleteAll(false);
+ }
+
+ private void deleteAll(boolean readOnlyOk) {
+ if (!readOnlyOk) {
+ checkReadOnly();
+ }
+ if (theMap != null) {
+ theMap.clear();
+ }
+ }
+
+ @Override
+ public void destroy() {
+ deleteAll(true);
+ theMap = null;
+ isOpen = false;
+ }
+
+ @Override
+ public void flush() {
+ checkOpen(true);
+ checkReadOnly();
+ }
+
+ @Override
+ public Iterator<Map<String,Object>> iterator(Object key) {
+ checkOpen(true);
+ if(key==null) {
+ return null;
+ }
+ if(key instanceof Iterable<?>) {
+ List<Map<String,Object>> vals = new ArrayList<>();
+ Iterator<?> iter = ((Iterable<?>) key).iterator();
+ while(iter.hasNext()) {
+ List<Map<String,Object>> val = theMap.get(iter.next());
+ if(val!=null) {
+ vals.addAll(val);
+ }
+ }
+ if(vals.size()==0) {
+ return null;
+ }
+ return vals.iterator();
+ }
+ List<Map<String,Object>> val = theMap.get(key);
+ if (val == null) {
+ return null;
+ }
+ return val.iterator();
+ }
+
+ @Override
+ public Iterator<Map<String,Object>> iterator() {
+ return new Iterator<Map<String, Object>>() {
+ private Iterator<Map.Entry<Object,List<Map<String,Object>>>> theMapIter;
+ private List<Map<String,Object>> currentKeyResult = null;
+ private Iterator<Map<String,Object>> currentKeyResultIter = null;
+
+ {
+ theMapIter = theMap.entrySet().iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (currentKeyResultIter != null) {
+ if (currentKeyResultIter.hasNext()) {
+ return true;
+ } else {
+ currentKeyResult = null;
+ currentKeyResultIter = null;
+ }
+ }
+
+ Map.Entry<Object,List<Map<String,Object>>> next = null;
+ if (theMapIter.hasNext()) {
+ next = theMapIter.next();
+ currentKeyResult = next.getValue();
+ currentKeyResultIter = currentKeyResult.iterator();
+ if (currentKeyResultIter.hasNext()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public Map<String,Object> next() {
+ if (currentKeyResultIter != null) {
+ if (currentKeyResultIter.hasNext()) {
+ return currentKeyResultIter.next();
+ } else {
+ currentKeyResult = null;
+ currentKeyResultIter = null;
+ }
+ }
+
+ Map.Entry<Object,List<Map<String,Object>>> next = null;
+ if (theMapIter.hasNext()) {
+ next = theMapIter.next();
+ currentKeyResult = next.getValue();
+ currentKeyResultIter = currentKeyResult.iterator();
+ if (currentKeyResultIter.hasNext()) {
+ return currentKeyResultIter.next();
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ @Override
+ public void open(Context context) {
+ checkOpen(false);
+ isOpen = true;
+ if (theMap == null) {
+ theMap = new TreeMap<>();
+ }
+
+ String pkName = CachePropertyUtil.getAttributeValueAsString(context,
+ DIHCacheSupport.CACHE_PRIMARY_KEY);
+ if (pkName != null) {
+ primaryKeyName = pkName;
+ }
+ isReadOnly = false;
+ String readOnlyStr = CachePropertyUtil.getAttributeValueAsString(context,
+ DIHCacheSupport.CACHE_READ_ONLY);
+ if ("true".equalsIgnoreCase(readOnlyStr)) {
+ isReadOnly = true;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SqlEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SqlEntityProcessor.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SqlEntityProcessor.java
new file mode 100644
index 0000000..19c6d0f
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SqlEntityProcessor.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Iterator;
+import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * <p>
+ * An {@link EntityProcessor} instance which provides support for reading from
+ * databases. It is used in conjunction with {@link JdbcDataSource}. This is the default
+ * {@link EntityProcessor} if none is specified explicitly in data-config.xml
+ * </p>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ *
+ * @since solr 1.3
+ */
+public class SqlEntityProcessor extends EntityProcessorBase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ protected DataSource<Iterator<Map<String, Object>>> dataSource;
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void init(Context context) {
+ super.init(context);
+ dataSource = context.getDataSource();
+ }
+
+ protected void initQuery(String q) {
+ try {
+ DataImporter.QUERY_COUNT.get().incrementAndGet();
+ rowIterator = dataSource.getData(q);
+ this.query = q;
+ } catch (DataImportHandlerException e) {
+ throw e;
+ } catch (Exception e) {
+ log.error( "The query failed '" + q + "'", e);
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE, e);
+ }
+ }
+
+ @Override
+ public Map<String, Object> nextRow() {
+ if (rowIterator == null) {
+ String q = getQuery();
+ initQuery(context.replaceTokens(q));
+ }
+ return getNext();
+ }
+
+ @Override
+ public Map<String, Object> nextModifiedRowKey() {
+ if (rowIterator == null) {
+ String deltaQuery = context.getEntityAttribute(DELTA_QUERY);
+ if (deltaQuery == null)
+ return null;
+ initQuery(context.replaceTokens(deltaQuery));
+ }
+ return getNext();
+ }
+
+ @Override
+ public Map<String, Object> nextDeletedRowKey() {
+ if (rowIterator == null) {
+ String deletedPkQuery = context.getEntityAttribute(DEL_PK_QUERY);
+ if (deletedPkQuery == null)
+ return null;
+ initQuery(context.replaceTokens(deletedPkQuery));
+ }
+ return getNext();
+ }
+
+ @Override
+ public Map<String, Object> nextModifiedParentRowKey() {
+ if (rowIterator == null) {
+ String parentDeltaQuery = context.getEntityAttribute(PARENT_DELTA_QUERY);
+ if (parentDeltaQuery == null)
+ return null;
+ log.info("Running parentDeltaQuery for Entity: "
+ + context.getEntityAttribute("name"));
+ initQuery(context.replaceTokens(parentDeltaQuery));
+ }
+ return getNext();
+ }
+
+ public String getQuery() {
+ String queryString = context.getEntityAttribute(QUERY);
+ if (Context.FULL_DUMP.equals(context.currentProcess())) {
+ return queryString;
+ }
+ if (Context.DELTA_DUMP.equals(context.currentProcess())) {
+ String deltaImportQuery = context.getEntityAttribute(DELTA_IMPORT_QUERY);
+ if(deltaImportQuery != null) return deltaImportQuery;
+ }
+ log.warn("'deltaImportQuery' attribute is not specified for entity : "+ entityName);
+ return getDeltaImportQuery(queryString);
+ }
+
+ public String getDeltaImportQuery(String queryString) {
+ StringBuilder sb = new StringBuilder(queryString);
+ if (SELECT_WHERE_PATTERN.matcher(queryString).find()) {
+ sb.append(" and ");
+ } else {
+ sb.append(" where ");
+ }
+ boolean first = true;
+ String[] primaryKeys = context.getEntityAttribute("pk").split(",");
+ for (String primaryKey : primaryKeys) {
+ if (!first) {
+ sb.append(" and ");
+ }
+ first = false;
+ Object val = context.resolve("dataimporter.delta." + primaryKey);
+ if (val == null) {
+ Matcher m = DOT_PATTERN.matcher(primaryKey);
+ if (m.find()) {
+ val = context.resolve("dataimporter.delta." + m.group(1));
+ }
+ }
+ sb.append(primaryKey).append(" = ");
+ if (val instanceof Number) {
+ sb.append(val.toString());
+ } else {
+ sb.append("'").append(val.toString()).append("'");
+ }
+ }
+ return sb.toString();
+ }
+
+ private static Pattern SELECT_WHERE_PATTERN = Pattern.compile(
+ "^\\s*(select\\b.*?\\b)(where).*", Pattern.CASE_INSENSITIVE);
+
+ public static final String QUERY = "query";
+
+ public static final String DELTA_QUERY = "deltaQuery";
+
+ public static final String DELTA_IMPORT_QUERY = "deltaImportQuery";
+
+ public static final String PARENT_DELTA_QUERY = "parentDeltaQuery";
+
+ public static final String DEL_PK_QUERY = "deletedPkQuery";
+
+ public static final Pattern DOT_PATTERN = Pattern.compile(".*?\\.(.*)$");
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SqlEscapingEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SqlEscapingEvaluator.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SqlEscapingEvaluator.java
new file mode 100644
index 0000000..7f9c26e
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SqlEscapingEvaluator.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+
+import java.util.List;
+
+/**
+ * <p> Escapes values in SQL queries. It escapes the value of the given expression
+ * by replacing all occurrences of single-quotes by two single-quotes and similarily
+ * for double-quotes </p>
+ */
+public class SqlEscapingEvaluator extends Evaluator {
+ @Override
+ public String evaluate(String expression, Context context) {
+ List<Object> l = parseParams(expression, context.getVariableResolver());
+ if (l.size() != 1) {
+ throw new DataImportHandlerException(SEVERE, "'escapeSql' must have at least one parameter ");
+ }
+ String s = l.get(0).toString();
+ // escape single quote with two single quotes, double quote
+ // with two doule quotes, and backslash with double backslash.
+ // See: http://dev.mysql.com/doc/refman/4.1/en/mysql-real-escape-string.html
+ return s.replaceAll("'", "''").replaceAll("\"", "\"\"").replaceAll("\\\\", "\\\\\\\\");
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/TemplateTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/TemplateTransformer.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/TemplateTransformer.java
new file mode 100644
index 0000000..f655edd
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/TemplateTransformer.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>
+ * A {@link Transformer} which can put values into a column by resolving an expression
+ * containing other columns
+ * </p>
+ * <p>
+ * For example:<br>
+ * <field column="name" template="${e.lastName}, ${e.firstName}
+ * ${e.middleName}" /> will produce the name by combining values from
+ * lastName, firstName and middleName fields as given in the template attribute.
+ * </p>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TemplateTransformer extends Transformer {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private Map<String ,List<String>> templateVsVars = new HashMap<>();
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Object transformRow(Map<String, Object> row, Context context) {
+
+
+ VariableResolver resolver = context.getVariableResolver();
+ // Add current row to the copy of resolver map
+
+ for (Map<String, String> map : context.getAllEntityFields()) {
+ map.entrySet();
+ String expr = map.get(TEMPLATE);
+ if (expr == null)
+ continue;
+
+ String column = map.get(DataImporter.COLUMN);
+
+ // Verify if all variables can be resolved or not
+ boolean resolvable = true;
+ List<String> variables = this.templateVsVars.get(expr);
+ if(variables == null){
+ variables = resolver.getVariables(expr);
+ this.templateVsVars.put(expr, variables);
+ }
+ for (String v : variables) {
+ if (resolver.resolve(v) == null) {
+ log.warn("Unable to resolve variable: " + v
+ + " while parsing expression: " + expr);
+ resolvable = false;
+ }
+ }
+
+ if (!resolvable)
+ continue;
+ if(variables.size() == 1 && expr.startsWith("${") && expr.endsWith("}")){
+ addToRow(column, row, resolver.resolve(variables.get(0)));
+ } else {
+ addToRow(column, row, resolver.replaceTokens(expr));
+ }
+ }
+
+ return row;
+ }
+
+ private void addToRow(String key, Map<String, Object> row, Object value) {
+ Object prevVal = row.get(key);
+ if (prevVal != null) {
+ if (prevVal instanceof List) {
+ ((List) prevVal).add(value);
+ } else {
+ ArrayList<Object> valList = new ArrayList<Object>();
+ valList.add(prevVal);
+ valList.add(value);
+ row.put(key, valList);
+ }
+ } else {
+ row.put(key, value);
+ }
+ }
+
+ public static final String TEMPLATE = "template";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Transformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Transformer.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Transformer.java
new file mode 100644
index 0000000..c7923e1
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Transformer.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.Map;
+
+/**
+ * <p>
+ * Use this API to implement a custom transformer for any given entity
+ * </p>
+ * <p>
+ * Implementations of this abstract class must provide a public no-args constructor.
+ * </p>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ *
+ * @since solr 1.3
+ */
+public abstract class Transformer {
+ /**
+ * The input is a row of data and the output has to be a new row.
+ *
+ * @param context The current context
+ * @param row A row of data
+ * @return The changed data. It must be a {@link Map}<{@link String}, {@link Object}> if it returns
+ * only one row or if there are multiple rows to be returned it must
+ * be a {@link java.util.List}<{@link Map}<{@link String}, {@link Object}>>
+ */
+ public abstract Object transformRow(Map<String, Object> row, Context context);
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/URLDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/URLDataSource.java b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/URLDataSource.java
new file mode 100644
index 0000000..145ffc4
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/URLDataSource.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.lang.invoke.MethodHandles;
+import java.net.URL;
+import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
+import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * <p> A data source implementation which can be used to read character files using HTTP. </p> <p> Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a> for more
+ * details. </p>
+ * <p>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ *
+ * @since solr 1.4
+ */
+public class URLDataSource extends DataSource<Reader> {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private String baseUrl;
+
+ private String encoding;
+
+ private int connectionTimeout = CONNECTION_TIMEOUT;
+
+ private int readTimeout = READ_TIMEOUT;
+
+ private Context context;
+
+ private Properties initProps;
+
+ public URLDataSource() {
+ }
+
+ @Override
+ public void init(Context context, Properties initProps) {
+ this.context = context;
+ this.initProps = initProps;
+
+ baseUrl = getInitPropWithReplacements(BASE_URL);
+ if (getInitPropWithReplacements(ENCODING) != null)
+ encoding = getInitPropWithReplacements(ENCODING);
+ String cTimeout = getInitPropWithReplacements(CONNECTION_TIMEOUT_FIELD_NAME);
+ String rTimeout = getInitPropWithReplacements(READ_TIMEOUT_FIELD_NAME);
+ if (cTimeout != null) {
+ try {
+ connectionTimeout = Integer.parseInt(cTimeout);
+ } catch (NumberFormatException e) {
+ log.warn("Invalid connection timeout: " + cTimeout);
+ }
+ }
+ if (rTimeout != null) {
+ try {
+ readTimeout = Integer.parseInt(rTimeout);
+ } catch (NumberFormatException e) {
+ log.warn("Invalid read timeout: " + rTimeout);
+ }
+ }
+ }
+
+ @Override
+ public Reader getData(String query) {
+ URL url = null;
+ try {
+ if (URIMETHOD.matcher(query).find()) url = new URL(query);
+ else url = new URL(baseUrl + query);
+
+ log.debug("Accessing URL: " + url.toString());
+
+ URLConnection conn = url.openConnection();
+ conn.setConnectTimeout(connectionTimeout);
+ conn.setReadTimeout(readTimeout);
+ InputStream in = conn.getInputStream();
+ String enc = encoding;
+ if (enc == null) {
+ String cType = conn.getContentType();
+ if (cType != null) {
+ Matcher m = CHARSET_PATTERN.matcher(cType);
+ if (m.find()) {
+ enc = m.group(1);
+ }
+ }
+ }
+ if (enc == null)
+ enc = UTF_8;
+ DataImporter.QUERY_COUNT.get().incrementAndGet();
+ return new InputStreamReader(in, enc);
+ } catch (Exception e) {
+ log.error("Exception thrown while getting data", e);
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Exception in invoking url " + url, e);
+ }
+ }
+
+ @Override
+ public void close() {
+ }
+
+ public String getBaseUrl() {
+ return baseUrl;
+ }
+
+ private String getInitPropWithReplacements(String propertyName) {
+ final String expr = initProps.getProperty(propertyName);
+ if (expr == null) {
+ return null;
+ }
+ return context.replaceTokens(expr);
+ }
+
+ static final Pattern URIMETHOD = Pattern.compile("\\w{3,}:/");
+
+ private static final Pattern CHARSET_PATTERN = Pattern.compile(".*?charset=(.*)$", Pattern.CASE_INSENSITIVE);
+
+ public static final String ENCODING = "encoding";
+
+ public static final String BASE_URL = "baseUrl";
+
+ public static final String UTF_8 = StandardCharsets.UTF_8.name();
+
+ public static final String CONNECTION_TIMEOUT_FIELD_NAME = "connectionTimeout";
+
+ public static final String READ_TIMEOUT_FIELD_NAME = "readTimeout";
+
+ public static final int CONNECTION_TIMEOUT = 5000;
+
+ public static final int READ_TIMEOUT = 10000;
+}
[24/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/main/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/main/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java b/solr/contrib/dataimporthandler-extras/src/main/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java
new file mode 100644
index 0000000..54f5984
--- /dev/null
+++ b/solr/contrib/dataimporthandler-extras/src/main/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java
@@ -0,0 +1,888 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import com.sun.mail.imap.IMAPMessage;
+
+import org.apache.solr.common.util.SuppressForbidden;
+import org.apache.solr.handler.dataimport.config.ConfigNameConstants;
+import org.apache.solr.util.RTimer;
+import org.apache.tika.Tika;
+import org.apache.tika.metadata.Metadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.mail.*;
+import javax.mail.internet.AddressException;
+import javax.mail.internet.ContentType;
+import javax.mail.internet.InternetAddress;
+import javax.mail.internet.MimeMessage;
+import javax.mail.search.*;
+
+import java.io.InputStream;
+import java.lang.invoke.MethodHandles;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.function.Supplier;
+
+import com.sun.mail.gimap.GmailFolder;
+import com.sun.mail.gimap.GmailRawSearchTerm;
+
+/**
+ * An EntityProcessor instance which can index emails along with their
+ * attachments from POP3 or IMAP sources. Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler"
+ * >http://wiki.apache.org/solr/DataImportHandler</a> for more details. <b>This
+ * API is experimental and subject to change</b>
+ *
+ * @since solr 1.4
+ */
+public class MailEntityProcessor extends EntityProcessorBase {
+
+ private static final SimpleDateFormat sinceDateParser =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT);
+ private static final SimpleDateFormat afterFmt =
+ new SimpleDateFormat("yyyy/MM/dd", Locale.ROOT);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static interface CustomFilter {
+ public SearchTerm getCustomSearch(Folder folder);
+ }
+
+ public void init(Context context) {
+ super.init(context);
+ // set attributes using XXX getXXXFromContext(attribute, defualtValue);
+ // applies variable resolver and return default if value is not found or null
+ // REQUIRED : connection and folder info
+ user = getStringFromContext("user", null);
+ password = getStringFromContext("password", null);
+ host = getStringFromContext("host", null);
+ protocol = getStringFromContext("protocol", null);
+ folderNames = getStringFromContext("folders", null);
+ // validate
+ if (host == null || protocol == null || user == null || password == null
+ || folderNames == null) throw new DataImportHandlerException(
+ DataImportHandlerException.SEVERE,
+ "'user|password|protocol|host|folders' are required attributes");
+
+ // OPTIONAL : have defaults and are optional
+ recurse = getBoolFromContext("recurse", true);
+
+ exclude.clear();
+ String excludes = getStringFromContext("exclude", "");
+ if (excludes != null && !excludes.trim().equals("")) {
+ exclude = Arrays.asList(excludes.split(","));
+ }
+
+ include.clear();
+ String includes = getStringFromContext("include", "");
+ if (includes != null && !includes.trim().equals("")) {
+ include = Arrays.asList(includes.split(","));
+ }
+ batchSize = getIntFromContext("batchSize", 20);
+ customFilter = getStringFromContext("customFilter", "");
+ if (filters != null) filters.clear();
+ folderIter = null;
+ msgIter = null;
+
+ String lastIndexTime = null;
+ String command =
+ String.valueOf(context.getRequestParameters().get("command"));
+ if (!DataImporter.FULL_IMPORT_CMD.equals(command))
+ throw new IllegalArgumentException(this.getClass().getSimpleName()+
+ " only supports "+DataImporter.FULL_IMPORT_CMD);
+
+ // Read the last_index_time out of the dataimport.properties if available
+ String cname = getStringFromContext("name", "mailimporter");
+ String varName = ConfigNameConstants.IMPORTER_NS_SHORT + "." + cname + "."
+ + DocBuilder.LAST_INDEX_TIME;
+ Object varValue = context.getVariableResolver().resolve(varName);
+ log.info(varName+"="+varValue);
+
+ if (varValue != null && !"".equals(varValue) &&
+ !"".equals(getStringFromContext("fetchMailsSince", ""))) {
+
+ // need to check if varValue is the epoch, which we'll take to mean the
+ // initial value, in which case means we should use fetchMailsSince instead
+ Date tmp = null;
+ try {
+ tmp = sinceDateParser.parse((String)varValue);
+ if (tmp.getTime() == 0) {
+ log.info("Ignoring initial value "+varValue+" for "+varName+
+ " in favor of fetchMailsSince config parameter");
+ tmp = null; // don't use this value
+ }
+ } catch (ParseException e) {
+ // probably ok to ignore this since we have other options below
+ // as we're just trying to figure out if the date is 0
+ log.warn("Failed to parse "+varValue+" from "+varName+" due to: "+e);
+ }
+
+ if (tmp == null) {
+ // favor fetchMailsSince in this case because the value from
+ // dataimport.properties is the default/init value
+ varValue = getStringFromContext("fetchMailsSince", "");
+ log.info("fetchMailsSince="+varValue);
+ }
+ }
+
+ if (varValue == null || "".equals(varValue)) {
+ varName = ConfigNameConstants.IMPORTER_NS_SHORT + "."
+ + DocBuilder.LAST_INDEX_TIME;
+ varValue = context.getVariableResolver().resolve(varName);
+ log.info(varName+"="+varValue);
+ }
+
+ if (varValue != null && varValue instanceof String) {
+ lastIndexTime = (String)varValue;
+ if (lastIndexTime != null && lastIndexTime.length() == 0)
+ lastIndexTime = null;
+ }
+
+ if (lastIndexTime == null)
+ lastIndexTime = getStringFromContext("fetchMailsSince", "");
+
+ log.info("Using lastIndexTime "+lastIndexTime+" for mail import");
+
+ this.fetchMailsSince = null;
+ if (lastIndexTime != null && lastIndexTime.length() > 0) {
+ try {
+ fetchMailsSince = sinceDateParser.parse(lastIndexTime);
+ log.info("Parsed fetchMailsSince=" + lastIndexTime);
+ } catch (ParseException e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Invalid value for fetchMailSince: " + lastIndexTime, e);
+ }
+ }
+
+ fetchSize = getIntFromContext("fetchSize", 32 * 1024);
+ cTimeout = getIntFromContext("connectTimeout", 30 * 1000);
+ rTimeout = getIntFromContext("readTimeout", 60 * 1000);
+
+ String tmp = context.getEntityAttribute("includeOtherUserFolders");
+ includeOtherUserFolders = (tmp != null && Boolean.valueOf(tmp.trim()));
+ tmp = context.getEntityAttribute("includeSharedFolders");
+ includeSharedFolders = (tmp != null && Boolean.valueOf(tmp.trim()));
+
+ setProcessAttachmentConfig();
+ includeContent = getBoolFromContext("includeContent", true);
+
+ logConfig();
+ }
+
+ private void setProcessAttachmentConfig() {
+ processAttachment = true;
+ String tbval = context.getEntityAttribute("processAttachments");
+ if (tbval == null) {
+ tbval = context.getEntityAttribute("processAttachement");
+ if (tbval != null) processAttachment = Boolean.valueOf(tbval);
+ } else processAttachment = Boolean.valueOf(tbval);
+ }
+
+ @Override
+ public Map<String,Object> nextRow() {
+ Message mail = null;
+ Map<String,Object> row = null;
+ do {
+ // try till there is a valid document or folders get exhausted.
+ // when mail == NULL, it means end of processing
+ mail = getNextMail();
+
+ if (mail != null)
+ row = getDocumentFromMail(mail);
+
+ if (row != null && row.get("folder") == null)
+ row.put("folder", mail.getFolder().getFullName());
+
+ } while (row == null && mail != null);
+ return row;
+ }
+
+ private Message getNextMail() {
+ if (!connected) {
+ // this is needed to load the activation mail stuff correctly
+ // otherwise, the JavaMail multipart support doesn't get configured
+ // correctly, which leads to a class cast exception when processing
+ // multipart messages: IMAPInputStream cannot be cast to
+ // javax.mail.Multipart
+ if (false == withContextClassLoader(getClass().getClassLoader(), this::connectToMailBox)) {
+ return null;
+ }
+ connected = true;
+ }
+ if (folderIter == null) {
+ createFilters();
+ folderIter = new FolderIterator(mailbox);
+ }
+ // get next message from the folder
+ // if folder is exhausted get next folder
+ // loop till a valid mail or all folders exhausted.
+ while (msgIter == null || !msgIter.hasNext()) {
+ Folder next = folderIter.hasNext() ? folderIter.next() : null;
+ if (next == null) return null;
+
+ msgIter = new MessageIterator(next, batchSize);
+ }
+ return msgIter.next();
+ }
+
+ private Map<String,Object> getDocumentFromMail(Message mail) {
+ Map<String,Object> row = new HashMap<>();
+ try {
+ addPartToDocument(mail, row, true);
+ return row;
+ } catch (Exception e) {
+ log.error("Failed to convert message [" + mail.toString()
+ + "] to document due to: " + e, e);
+ return null;
+ }
+ }
+
+ public void addPartToDocument(Part part, Map<String,Object> row, boolean outerMost) throws Exception {
+ if (part instanceof Message) {
+ addEnvelopeToDocument(part, row);
+ }
+
+ String ct = part.getContentType().toLowerCase(Locale.ROOT);
+ ContentType ctype = new ContentType(ct);
+ if (part.isMimeType("multipart/*")) {
+ Object content = part.getContent();
+ if (content != null && content instanceof Multipart) {
+ Multipart mp = (Multipart) part.getContent();
+ int count = mp.getCount();
+ if (part.isMimeType("multipart/alternative")) count = 1;
+ for (int i = 0; i < count; i++)
+ addPartToDocument(mp.getBodyPart(i), row, false);
+ } else {
+ log.warn("Multipart content is a not an instance of Multipart! Content is: "
+ + (content != null ? content.getClass().getName() : "null")
+ + ". Typically, this is due to the Java Activation JAR being loaded by the wrong classloader.");
+ }
+ } else if (part.isMimeType("message/rfc822")) {
+ addPartToDocument((Part) part.getContent(), row, false);
+ } else {
+ String disp = part.getDisposition();
+ if (includeContent
+ && !(disp != null && disp.equalsIgnoreCase(Part.ATTACHMENT))) {
+ InputStream is = part.getInputStream();
+ Metadata contentTypeHint = new Metadata();
+ contentTypeHint.set(Metadata.CONTENT_TYPE, ctype.getBaseType()
+ .toLowerCase(Locale.ENGLISH));
+ String content = (new Tika()).parseToString(is, contentTypeHint);
+ if (row.get(CONTENT) == null) row.put(CONTENT, new ArrayList<String>());
+ List<String> contents = (List<String>) row.get(CONTENT);
+ contents.add(content.trim());
+ row.put(CONTENT, contents);
+ }
+ if (!processAttachment || disp == null
+ || !disp.equalsIgnoreCase(Part.ATTACHMENT)) return;
+ InputStream is = part.getInputStream();
+ String fileName = part.getFileName();
+ Metadata contentTypeHint = new Metadata();
+ contentTypeHint.set(Metadata.CONTENT_TYPE, ctype.getBaseType()
+ .toLowerCase(Locale.ENGLISH));
+ String content = (new Tika()).parseToString(is, contentTypeHint);
+ if (content == null || content.trim().length() == 0) return;
+
+ if (row.get(ATTACHMENT) == null) row.put(ATTACHMENT,
+ new ArrayList<String>());
+ List<String> contents = (List<String>) row.get(ATTACHMENT);
+ contents.add(content.trim());
+ row.put(ATTACHMENT, contents);
+ if (row.get(ATTACHMENT_NAMES) == null) row.put(ATTACHMENT_NAMES,
+ new ArrayList<String>());
+ List<String> names = (List<String>) row.get(ATTACHMENT_NAMES);
+ names.add(fileName);
+ row.put(ATTACHMENT_NAMES, names);
+ }
+ }
+
+ private void addEnvelopeToDocument(Part part, Map<String,Object> row)
+ throws MessagingException {
+ MimeMessage mail = (MimeMessage) part;
+ Address[] adresses;
+ if ((adresses = mail.getFrom()) != null && adresses.length > 0) row.put(
+ FROM, adresses[0].toString());
+
+ List<String> to = new ArrayList<>();
+ if ((adresses = mail.getRecipients(Message.RecipientType.TO)) != null) addAddressToList(
+ adresses, to);
+ if ((adresses = mail.getRecipients(Message.RecipientType.CC)) != null) addAddressToList(
+ adresses, to);
+ if ((adresses = mail.getRecipients(Message.RecipientType.BCC)) != null) addAddressToList(
+ adresses, to);
+ if (to.size() > 0) row.put(TO_CC_BCC, to);
+
+ row.put(MESSAGE_ID, mail.getMessageID());
+ row.put(SUBJECT, mail.getSubject());
+
+ Date d = mail.getSentDate();
+ if (d != null) {
+ row.put(SENT_DATE, d);
+ }
+
+ List<String> flags = new ArrayList<>();
+ for (Flags.Flag flag : mail.getFlags().getSystemFlags()) {
+ if (flag == Flags.Flag.ANSWERED) flags.add(FLAG_ANSWERED);
+ else if (flag == Flags.Flag.DELETED) flags.add(FLAG_DELETED);
+ else if (flag == Flags.Flag.DRAFT) flags.add(FLAG_DRAFT);
+ else if (flag == Flags.Flag.FLAGGED) flags.add(FLAG_FLAGGED);
+ else if (flag == Flags.Flag.RECENT) flags.add(FLAG_RECENT);
+ else if (flag == Flags.Flag.SEEN) flags.add(FLAG_SEEN);
+ }
+ flags.addAll(Arrays.asList(mail.getFlags().getUserFlags()));
+ if (flags.size() == 0) flags.add(FLAG_NONE);
+ row.put(FLAGS, flags);
+
+ String[] hdrs = mail.getHeader("X-Mailer");
+ if (hdrs != null) row.put(XMAILER, hdrs[0]);
+ }
+
+ private void addAddressToList(Address[] adresses, List<String> to)
+ throws AddressException {
+ for (Address address : adresses) {
+ to.add(address.toString());
+ InternetAddress ia = (InternetAddress) address;
+ if (ia.isGroup()) {
+ InternetAddress[] group = ia.getGroup(false);
+ for (InternetAddress member : group)
+ to.add(member.toString());
+ }
+ }
+ }
+
+ private boolean connectToMailBox() {
+ try {
+ Properties props = new Properties();
+ if (System.getProperty("mail.debug") != null)
+ props.setProperty("mail.debug", System.getProperty("mail.debug"));
+
+ if (("imap".equals(protocol) || "imaps".equals(protocol))
+ && "imap.gmail.com".equals(host)) {
+ log.info("Consider using 'gimaps' protocol instead of '" + protocol
+ + "' for enabling GMail specific extensions for " + host);
+ }
+
+ props.setProperty("mail.store.protocol", protocol);
+
+ String imapPropPrefix = protocol.startsWith("gimap") ? "gimap" : "imap";
+ props.setProperty("mail." + imapPropPrefix + ".fetchsize", "" + fetchSize);
+ props.setProperty("mail." + imapPropPrefix + ".timeout", "" + rTimeout);
+ props.setProperty("mail." + imapPropPrefix + ".connectiontimeout", "" + cTimeout);
+
+ int port = -1;
+ int colonAt = host.indexOf(":");
+ if (colonAt != -1) {
+ port = Integer.parseInt(host.substring(colonAt + 1));
+ host = host.substring(0, colonAt);
+ }
+
+ Session session = Session.getDefaultInstance(props, null);
+ mailbox = session.getStore(protocol);
+ if (port != -1) {
+ mailbox.connect(host, port, user, password);
+ } else {
+ mailbox.connect(host, user, password);
+ }
+ log.info("Connected to " + user + "'s mailbox on " + host);
+
+ return true;
+ } catch (MessagingException e) {
+ String errMsg = String.format(Locale.ENGLISH,
+ "Failed to connect to %s server %s as user %s due to: %s", protocol,
+ host, user, e.toString());
+ log.error(errMsg, e);
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ errMsg, e);
+ }
+ }
+
+ private void createFilters() {
+ if (fetchMailsSince != null) {
+ filters.add(new MailsSinceLastCheckFilter(fetchMailsSince));
+ }
+ if (customFilter != null && !customFilter.equals("")) {
+ try {
+ Class cf = Class.forName(customFilter);
+ Object obj = cf.newInstance();
+ if (obj instanceof CustomFilter) {
+ filters.add((CustomFilter) obj);
+ }
+ } catch (Exception e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Custom filter could not be created", e);
+ }
+ }
+ }
+
+ private void logConfig() {
+ if (!log.isInfoEnabled()) return;
+
+ String lineSep = System.getProperty("line.separator");
+
+ StringBuffer config = new StringBuffer();
+ config.append("user : ").append(user).append(lineSep);
+ config
+ .append("pwd : ")
+ .append(
+ password != null && password.length() > 0 ? "<non-null>" : "<null>")
+ .append(lineSep);
+ config.append("protocol : ").append(protocol)
+ .append(lineSep);
+ config.append("host : ").append(host)
+ .append(lineSep);
+ config.append("folders : ").append(folderNames)
+ .append(lineSep);
+ config.append("recurse : ").append(recurse)
+ .append(lineSep);
+ config.append("exclude : ").append(exclude.toString())
+ .append(lineSep);
+ config.append("include : ").append(include.toString())
+ .append(lineSep);
+ config.append("batchSize : ").append(batchSize)
+ .append(lineSep);
+ config.append("fetchSize : ").append(fetchSize)
+ .append(lineSep);
+ config.append("read timeout : ").append(rTimeout)
+ .append(lineSep);
+ config.append("conection timeout : ").append(cTimeout)
+ .append(lineSep);
+ config.append("custom filter : ").append(customFilter)
+ .append(lineSep);
+ config.append("fetch mail since : ").append(fetchMailsSince)
+ .append(lineSep);
+ config.append("includeContent : ").append(includeContent)
+ .append(lineSep);
+ config.append("processAttachments : ").append(processAttachment)
+ .append(lineSep);
+ config.append("includeOtherUserFolders : ").append(includeOtherUserFolders)
+ .append(lineSep);
+ config.append("includeSharedFolders : ").append(includeSharedFolders)
+ .append(lineSep);
+ log.info(config.toString());
+ }
+
+ class FolderIterator implements Iterator<Folder> {
+ private Store mailbox;
+ private List<String> topLevelFolders;
+ private List<Folder> folders = null;
+ private Folder lastFolder = null;
+
+ public FolderIterator(Store mailBox) {
+ this.mailbox = mailBox;
+ folders = new ArrayList<>();
+ getTopLevelFolders(mailBox);
+ if (includeOtherUserFolders) getOtherUserFolders();
+ if (includeSharedFolders) getSharedFolders();
+ }
+
+ public boolean hasNext() {
+ return !folders.isEmpty();
+ }
+
+ public Folder next() {
+ try {
+ boolean hasMessages = false;
+ Folder next;
+ do {
+ if (lastFolder != null) {
+ lastFolder.close(false);
+ lastFolder = null;
+ }
+ if (folders.isEmpty()) {
+ mailbox.close();
+ return null;
+ }
+ next = folders.remove(0);
+ if (next != null) {
+ String fullName = next.getFullName();
+ if (!excludeFolder(fullName)) {
+ hasMessages = (next.getType() & Folder.HOLDS_MESSAGES) != 0;
+ next.open(Folder.READ_ONLY);
+ lastFolder = next;
+ log.info("Opened folder : " + fullName);
+ }
+ if (recurse && ((next.getType() & Folder.HOLDS_FOLDERS) != 0)) {
+ Folder[] children = next.list();
+ log.info("Added its children to list : ");
+ for (int i = children.length - 1; i >= 0; i--) {
+ folders.add(0, children[i]);
+ log.info("child name : " + children[i].getFullName());
+ }
+ if (children.length == 0) log.info("NO children : ");
+ }
+ }
+ } while (!hasMessages);
+ return next;
+ } catch (Exception e) {
+ log.warn("Failed to read folders due to: "+e);
+ // throw new
+ // DataImportHandlerException(DataImportHandlerException.SEVERE,
+ // "Folder open failed", e);
+ }
+ return null;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException("It's read only mode...");
+ }
+
+ private void getTopLevelFolders(Store mailBox) {
+ if (folderNames != null) topLevelFolders = Arrays.asList(folderNames
+ .split(","));
+ for (int i = 0; topLevelFolders != null && i < topLevelFolders.size(); i++) {
+ try {
+ folders.add(mailbox.getFolder(topLevelFolders.get(i)));
+ } catch (MessagingException e) {
+ // skip bad ones unless it's the last one and still no good folder
+ if (folders.size() == 0 && i == topLevelFolders.size() - 1) throw new DataImportHandlerException(
+ DataImportHandlerException.SEVERE, "Folder retreival failed");
+ }
+ }
+ if (topLevelFolders == null || topLevelFolders.size() == 0) {
+ try {
+ folders.add(mailBox.getDefaultFolder());
+ } catch (MessagingException e) {
+ throw new DataImportHandlerException(
+ DataImportHandlerException.SEVERE, "Folder retreival failed");
+ }
+ }
+ }
+
+ private void getOtherUserFolders() {
+ try {
+ Folder[] ufldrs = mailbox.getUserNamespaces(null);
+ if (ufldrs != null) {
+ log.info("Found " + ufldrs.length + " user namespace folders");
+ for (Folder ufldr : ufldrs)
+ folders.add(ufldr);
+ }
+ } catch (MessagingException me) {
+ log.warn("Messaging exception retrieving user namespaces: "
+ + me.getMessage());
+ }
+ }
+
+ private void getSharedFolders() {
+ try {
+ Folder[] sfldrs = mailbox.getSharedNamespaces();
+ if (sfldrs != null) {
+ log.info("Found " + sfldrs.length + " shared namespace folders");
+ for (Folder sfldr : sfldrs)
+ folders.add(sfldr);
+ }
+ } catch (MessagingException me) {
+ log.warn("Messaging exception retrieving shared namespaces: "
+ + me.getMessage());
+ }
+ }
+
+ private boolean excludeFolder(String name) {
+ for (String s : exclude) {
+ if (name.matches(s)) return true;
+ }
+ for (String s : include) {
+ if (name.matches(s)) return false;
+ }
+ return include.size() > 0;
+ }
+ }
+
+ class MessageIterator extends SearchTerm implements Iterator<Message> {
+ private Folder folder;
+ private Message[] messagesInCurBatch = null;
+ private int current = 0;
+ private int currentBatch = 0;
+ private int batchSize = 0;
+ private int totalInFolder = 0;
+ private boolean doBatching = true;
+
+ public MessageIterator(Folder folder, int batchSize) {
+ super();
+
+ try {
+ this.folder = folder;
+ this.batchSize = batchSize;
+ SearchTerm st = getSearchTerm();
+
+ log.info("SearchTerm=" + st);
+
+ if (st != null || folder instanceof GmailFolder) {
+ doBatching = false;
+ // Searching can still take a while even though we're only pulling
+ // envelopes; unless you're using gmail server-side filter, which is
+ // fast
+ log.info("Searching folder " + folder.getName() + " for messages");
+ final RTimer searchTimer = new RTimer();
+
+ // If using GMail, speed up the envelope processing by doing a
+ // server-side
+ // search for messages occurring on or after the fetch date (at
+ // midnight),
+ // which reduces the number of envelopes we need to pull from the
+ // server
+ // to apply the precise DateTerm filter; GMail server-side search has
+ // date
+ // granularity only but the local filters are also applied
+
+ if (folder instanceof GmailFolder && fetchMailsSince != null) {
+ String afterCrit = "after:" + afterFmt.format(fetchMailsSince);
+ log.info("Added server-side gmail filter: " + afterCrit);
+ Message[] afterMessages = folder.search(new GmailRawSearchTerm(
+ afterCrit));
+
+ log.info("GMail server-side filter found " + afterMessages.length
+ + " messages received " + afterCrit + " in folder " + folder.getName());
+
+ // now pass in the server-side filtered messages to the local filter
+ messagesInCurBatch = folder.search((st != null ? st : this), afterMessages);
+ } else {
+ messagesInCurBatch = folder.search(st);
+ }
+ totalInFolder = messagesInCurBatch.length;
+ folder.fetch(messagesInCurBatch, fp);
+ current = 0;
+ log.info("Total messages : " + totalInFolder);
+ log.info("Search criteria applied. Batching disabled. Took {} (ms)", searchTimer.getTime());
+ } else {
+ totalInFolder = folder.getMessageCount();
+ log.info("Total messages : " + totalInFolder);
+ getNextBatch(batchSize, folder);
+ }
+ } catch (MessagingException e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Message retreival failed", e);
+ }
+ }
+
+ private void getNextBatch(int batchSize, Folder folder)
+ throws MessagingException {
+ // after each batch invalidate cache
+ if (messagesInCurBatch != null) {
+ for (Message m : messagesInCurBatch) {
+ if (m instanceof IMAPMessage) ((IMAPMessage) m).invalidateHeaders();
+ }
+ }
+ int lastMsg = (currentBatch + 1) * batchSize;
+ lastMsg = lastMsg > totalInFolder ? totalInFolder : lastMsg;
+ messagesInCurBatch = folder.getMessages(currentBatch * batchSize + 1,
+ lastMsg);
+ folder.fetch(messagesInCurBatch, fp);
+ current = 0;
+ currentBatch++;
+ log.info("Current Batch : " + currentBatch);
+ log.info("Messages in this batch : " + messagesInCurBatch.length);
+ }
+
+ public boolean hasNext() {
+ boolean hasMore = current < messagesInCurBatch.length;
+ if (!hasMore && doBatching && currentBatch * batchSize < totalInFolder) {
+ // try next batch
+ try {
+ getNextBatch(batchSize, folder);
+ hasMore = current < messagesInCurBatch.length;
+ } catch (MessagingException e) {
+ throw new DataImportHandlerException(
+ DataImportHandlerException.SEVERE, "Message retreival failed", e);
+ }
+ }
+ return hasMore;
+ }
+
+ public Message next() {
+ return hasNext() ? messagesInCurBatch[current++] : null;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException("It's read only mode...");
+ }
+
+ private SearchTerm getSearchTerm() {
+ if (filters.size() == 0) return null;
+ if (filters.size() == 1) return filters.get(0).getCustomSearch(folder);
+ SearchTerm last = filters.get(0).getCustomSearch(folder);
+ for (int i = 1; i < filters.size(); i++) {
+ CustomFilter filter = filters.get(i);
+ SearchTerm st = filter.getCustomSearch(folder);
+ if (st != null) {
+ last = new AndTerm(last, st);
+ }
+ }
+ return last;
+ }
+
+ public boolean match(Message message) {
+ return true;
+ }
+ }
+
+ static class MailsSinceLastCheckFilter implements CustomFilter {
+
+ private Date since;
+
+ public MailsSinceLastCheckFilter(Date date) {
+ since = date;
+ }
+
+ @SuppressWarnings("serial")
+ public SearchTerm getCustomSearch(final Folder folder) {
+ log.info("Building mail filter for messages in " + folder.getName()
+ + " that occur after " + sinceDateParser.format(since));
+ return new DateTerm(ComparisonTerm.GE, since) {
+ private int matched = 0;
+ private int seen = 0;
+
+ @Override
+ public boolean match(Message msg) {
+ boolean isMatch = false;
+ ++seen;
+ try {
+ Date msgDate = msg.getReceivedDate();
+ if (msgDate == null) msgDate = msg.getSentDate();
+
+ if (msgDate != null && msgDate.getTime() >= since.getTime()) {
+ ++matched;
+ isMatch = true;
+ } else {
+ String msgDateStr = (msgDate != null) ? sinceDateParser.format(msgDate) : "null";
+ String sinceDateStr = (since != null) ? sinceDateParser.format(since) : "null";
+ log.debug("Message " + msg.getSubject() + " was received at [" + msgDateStr
+ + "], since filter is [" + sinceDateStr + "]");
+ }
+ } catch (MessagingException e) {
+ log.warn("Failed to process message due to: "+e, e);
+ }
+
+ if (seen % 100 == 0) {
+ log.info("Matched " + matched + " of " + seen + " messages since: "
+ + sinceDateParser.format(since));
+ }
+
+ return isMatch;
+ }
+ };
+ }
+ }
+
+ // user settings stored in member variables
+ private String user;
+ private String password;
+ private String host;
+ private String protocol;
+
+ private String folderNames;
+ private List<String> exclude = new ArrayList<>();
+ private List<String> include = new ArrayList<>();
+ private boolean recurse;
+
+ private int batchSize;
+ private int fetchSize;
+ private int cTimeout;
+ private int rTimeout;
+
+ private Date fetchMailsSince;
+ private String customFilter;
+
+ private boolean processAttachment = true;
+ private boolean includeContent = true;
+ private boolean includeOtherUserFolders = false;
+ private boolean includeSharedFolders = false;
+
+ // holds the current state
+ private Store mailbox;
+ private boolean connected = false;
+ private FolderIterator folderIter;
+ private MessageIterator msgIter;
+ private List<CustomFilter> filters = new ArrayList<>();
+ private static FetchProfile fp = new FetchProfile();
+
+ static {
+ fp.add(FetchProfile.Item.ENVELOPE);
+ fp.add(FetchProfile.Item.FLAGS);
+ fp.add("X-Mailer");
+ }
+
+ // Fields To Index
+ // single valued
+ private static final String MESSAGE_ID = "messageId";
+ private static final String SUBJECT = "subject";
+ private static final String FROM = "from";
+ private static final String SENT_DATE = "sentDate";
+ private static final String XMAILER = "xMailer";
+ // multi valued
+ private static final String TO_CC_BCC = "allTo";
+ private static final String FLAGS = "flags";
+ private static final String CONTENT = "content";
+ private static final String ATTACHMENT = "attachment";
+ private static final String ATTACHMENT_NAMES = "attachmentNames";
+ // flag values
+ private static final String FLAG_NONE = "none";
+ private static final String FLAG_ANSWERED = "answered";
+ private static final String FLAG_DELETED = "deleted";
+ private static final String FLAG_DRAFT = "draft";
+ private static final String FLAG_FLAGGED = "flagged";
+ private static final String FLAG_RECENT = "recent";
+ private static final String FLAG_SEEN = "seen";
+
+ private int getIntFromContext(String prop, int ifNull) {
+ int v = ifNull;
+ try {
+ String val = context.getEntityAttribute(prop);
+ if (val != null) {
+ val = context.replaceTokens(val);
+ v = Integer.parseInt(val);
+ }
+ } catch (NumberFormatException e) {
+ // do nothing
+ }
+ return v;
+ }
+
+ private boolean getBoolFromContext(String prop, boolean ifNull) {
+ boolean v = ifNull;
+ String val = context.getEntityAttribute(prop);
+ if (val != null) {
+ val = context.replaceTokens(val);
+ v = Boolean.valueOf(val);
+ }
+ return v;
+ }
+
+ private String getStringFromContext(String prop, String ifNull) {
+ String v = ifNull;
+ String val = context.getEntityAttribute(prop);
+ if (val != null) {
+ val = context.replaceTokens(val);
+ v = val;
+ }
+ return v;
+ }
+
+ @SuppressForbidden(reason = "Uses context class loader as a workaround to inject correct classloader to 3rd party libs")
+ private static <T> T withContextClassLoader(ClassLoader loader, Supplier<T> action) {
+ Thread ct = Thread.currentThread();
+ ClassLoader prev = ct.getContextClassLoader();
+ try {
+ ct.setContextClassLoader(loader);
+ return action.get();
+ } finally {
+ ct.setContextClassLoader(prev);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/main/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/main/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java b/solr/contrib/dataimporthandler-extras/src/main/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java
new file mode 100644
index 0000000..875032b
--- /dev/null
+++ b/solr/contrib/dataimporthandler-extras/src/main/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.EmptyParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.html.HtmlMapper;
+import org.apache.tika.parser.html.IdentityHtmlMapper;
+import org.apache.tika.sax.BodyContentHandler;
+import org.apache.tika.sax.ContentHandlerDecorator;
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.Attributes;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.sax.SAXTransformerFactory;
+import javax.xml.transform.sax.TransformerHandler;
+import javax.xml.transform.stream.StreamResult;
+import java.io.File;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+import static org.apache.solr.handler.dataimport.DataImporter.COLUMN;
+import static org.apache.solr.handler.dataimport.XPathEntityProcessor.URL;
+/**
+ * <p>An implementation of {@link EntityProcessor} which reads data from rich docs
+ * using <a href="http://tika.apache.org/">Apache Tika</a>
+ *
+ * <p>To index latitude/longitude data that might
+ * be extracted from a file's metadata, identify
+ * the geo field for this information with this attribute:
+ * <code>spatialMetadataField</code>
+ *
+ * @since solr 3.1
+ */
+public class TikaEntityProcessor extends EntityProcessorBase {
+ private static Parser EMPTY_PARSER = new EmptyParser();
+ private TikaConfig tikaConfig;
+ private String format = "text";
+ private boolean done = false;
+ private boolean extractEmbedded = false;
+ private String parser;
+ static final String AUTO_PARSER = "org.apache.tika.parser.AutoDetectParser";
+ private String htmlMapper;
+ private String spatialMetadataField;
+
+ @Override
+ public void init(Context context) {
+ super.init(context);
+ done = false;
+ }
+
+ @Override
+ protected void firstInit(Context context) {
+ super.firstInit(context);
+ try {
+ String tikaConfigFile = context.getResolvedEntityAttribute("tikaConfig");
+ if (tikaConfigFile == null) {
+ ClassLoader classLoader = context.getSolrCore().getResourceLoader().getClassLoader();
+ try (InputStream is = classLoader.getResourceAsStream("solr-default-tika-config.xml")) {
+ tikaConfig = new TikaConfig(is);
+ }
+ } else {
+ File configFile = new File(tikaConfigFile);
+ if (!configFile.isAbsolute()) {
+ configFile = new File(context.getSolrCore().getResourceLoader().getConfigDir(), tikaConfigFile);
+ }
+ tikaConfig = new TikaConfig(configFile);
+ }
+ } catch (Exception e) {
+ wrapAndThrow (SEVERE, e,"Unable to load Tika Config");
+ }
+
+ String extractEmbeddedString = context.getResolvedEntityAttribute("extractEmbedded");
+ if ("true".equals(extractEmbeddedString)) {
+ extractEmbedded = true;
+ }
+ format = context.getResolvedEntityAttribute("format");
+ if(format == null)
+ format = "text";
+ if (!"html".equals(format) && !"xml".equals(format) && !"text".equals(format)&& !"none".equals(format) )
+ throw new DataImportHandlerException(SEVERE, "'format' can be one of text|html|xml|none");
+
+ htmlMapper = context.getResolvedEntityAttribute("htmlMapper");
+ if (htmlMapper == null)
+ htmlMapper = "default";
+ if (!"default".equals(htmlMapper) && !"identity".equals(htmlMapper))
+ throw new DataImportHandlerException(SEVERE, "'htmlMapper', if present, must be 'default' or 'identity'");
+
+ parser = context.getResolvedEntityAttribute("parser");
+ if(parser == null) {
+ parser = AUTO_PARSER;
+ }
+
+ spatialMetadataField = context.getResolvedEntityAttribute("spatialMetadataField");
+ }
+
+ @Override
+ public Map<String, Object> nextRow() {
+ if(done) return null;
+ Map<String, Object> row = new HashMap<>();
+ DataSource<InputStream> dataSource = context.getDataSource();
+ InputStream is = dataSource.getData(context.getResolvedEntityAttribute(URL));
+ ContentHandler contentHandler = null;
+ Metadata metadata = new Metadata();
+ StringWriter sw = new StringWriter();
+ try {
+ if ("html".equals(format)) {
+ contentHandler = getHtmlHandler(sw);
+ } else if ("xml".equals(format)) {
+ contentHandler = getXmlContentHandler(sw);
+ } else if ("text".equals(format)) {
+ contentHandler = getTextContentHandler(sw);
+ } else if("none".equals(format)){
+ contentHandler = new DefaultHandler();
+ }
+ } catch (TransformerConfigurationException e) {
+ wrapAndThrow(SEVERE, e, "Unable to create content handler");
+ }
+ Parser tikaParser = null;
+ if(parser.equals(AUTO_PARSER)){
+ tikaParser = new AutoDetectParser(tikaConfig);
+ } else {
+ tikaParser = context.getSolrCore().getResourceLoader().newInstance(parser, Parser.class);
+ }
+ try {
+ ParseContext context = new ParseContext();
+ if ("identity".equals(htmlMapper)){
+ context.set(HtmlMapper.class, IdentityHtmlMapper.INSTANCE);
+ }
+ if (extractEmbedded) {
+ context.set(Parser.class, tikaParser);
+ } else {
+ context.set(Parser.class, EMPTY_PARSER);
+ }
+ tikaParser.parse(is, contentHandler, metadata , context);
+ } catch (Exception e) {
+ if(SKIP.equals(onError)) {
+ throw new DataImportHandlerException(DataImportHandlerException.SKIP_ROW,
+ "Document skipped :" + e.getMessage());
+ }
+ wrapAndThrow(SEVERE, e, "Unable to read content");
+ }
+ IOUtils.closeQuietly(is);
+ for (Map<String, String> field : context.getAllEntityFields()) {
+ if (!"true".equals(field.get("meta"))) continue;
+ String col = field.get(COLUMN);
+ String s = metadata.get(col);
+ if (s != null) row.put(col, s);
+ }
+ if(!"none".equals(format) ) row.put("text", sw.toString());
+ tryToAddLatLon(metadata, row);
+ done = true;
+ return row;
+ }
+
+ private void tryToAddLatLon(Metadata metadata, Map<String, Object> row) {
+ if (spatialMetadataField == null) return;
+ String latString = metadata.get(Metadata.LATITUDE);
+ String lonString = metadata.get(Metadata.LONGITUDE);
+ if (latString != null && lonString != null) {
+ row.put(spatialMetadataField, String.format(Locale.ROOT, "%s,%s", latString, lonString));
+ }
+ }
+
+ private static ContentHandler getHtmlHandler(Writer writer)
+ throws TransformerConfigurationException {
+ SAXTransformerFactory factory = (SAXTransformerFactory)
+ TransformerFactory.newInstance();
+ TransformerHandler handler = factory.newTransformerHandler();
+ handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "html");
+ handler.setResult(new StreamResult(writer));
+ return new ContentHandlerDecorator(handler) {
+ @Override
+ public void startElement(
+ String uri, String localName, String name, Attributes atts)
+ throws SAXException {
+ if (XHTMLContentHandler.XHTML.equals(uri)) {
+ uri = null;
+ }
+ if (!"head".equals(localName)) {
+ super.startElement(uri, localName, name, atts);
+ }
+ }
+
+ @Override
+ public void endElement(String uri, String localName, String name)
+ throws SAXException {
+ if (XHTMLContentHandler.XHTML.equals(uri)) {
+ uri = null;
+ }
+ if (!"head".equals(localName)) {
+ super.endElement(uri, localName, name);
+ }
+ }
+
+ @Override
+ public void startPrefixMapping(String prefix, String uri) {/*no op*/ }
+
+ @Override
+ public void endPrefixMapping(String prefix) {/*no op*/ }
+ };
+ }
+
+ private static ContentHandler getTextContentHandler(Writer writer) {
+ return new BodyContentHandler(writer);
+ }
+
+ private static ContentHandler getXmlContentHandler(Writer writer)
+ throws TransformerConfigurationException {
+ SAXTransformerFactory factory = (SAXTransformerFactory)
+ TransformerFactory.newInstance();
+ TransformerHandler handler = factory.newTransformerHandler();
+ handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "xml");
+ handler.setResult(new StreamResult(writer));
+ return handler;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/main/java/org/apache/solr/handler/dataimport/package.html
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/main/java/org/apache/solr/handler/dataimport/package.html b/solr/contrib/dataimporthandler-extras/src/main/java/org/apache/solr/handler/dataimport/package.html
new file mode 100644
index 0000000..9a7f6f2
--- /dev/null
+++ b/solr/contrib/dataimporthandler-extras/src/main/java/org/apache/solr/handler/dataimport/package.html
@@ -0,0 +1,23 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- not a package-info.java, because we already defined this package in core/ -->
+<html>
+<body>
+Plugins for <code>DataImportHandler</code> that have additional dependencies.
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/main/java/overview.html
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/main/java/overview.html b/solr/contrib/dataimporthandler-extras/src/main/java/overview.html
new file mode 100644
index 0000000..a60a25e
--- /dev/null
+++ b/solr/contrib/dataimporthandler-extras/src/main/java/overview.html
@@ -0,0 +1,21 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<body>
+Apache Solr Search Server: DataImportHandler Extras contrib
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/main/resources/solr-default-tika-config.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/main/resources/solr-default-tika-config.xml b/solr/contrib/dataimporthandler-extras/src/main/resources/solr-default-tika-config.xml
new file mode 100644
index 0000000..b598d9e
--- /dev/null
+++ b/solr/contrib/dataimporthandler-extras/src/main/resources/solr-default-tika-config.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<properties>
+ <service-loader initializableProblemHandler="ignore"/>
+</properties>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/resources/solr-default-tika-config.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/resources/solr-default-tika-config.xml b/solr/contrib/dataimporthandler-extras/src/resources/solr-default-tika-config.xml
deleted file mode 100644
index b598d9e..0000000
--- a/solr/contrib/dataimporthandler-extras/src/resources/solr-default-tika-config.xml
+++ /dev/null
@@ -1,20 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-<properties>
- <service-loader initializableProblemHandler="ignore"/>
-</properties>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/bad.doc
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/bad.doc b/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/bad.doc
deleted file mode 100644
index 5944c24..0000000
Binary files a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/bad.doc and /dev/null differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr-word.pdf
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr-word.pdf b/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr-word.pdf
deleted file mode 100644
index bd8b865..0000000
Binary files a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr-word.pdf and /dev/null differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-schema-no-unique-key.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-schema-no-unique-key.xml b/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-schema-no-unique-key.xml
deleted file mode 100644
index 793482a..0000000
--- a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-schema-no-unique-key.xml
+++ /dev/null
@@ -1,205 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!--
- This is the Solr schema file. This file should be named "schema.xml" and
- should be in the conf directory under the solr home
- (i.e. ./solr/conf/schema.xml by default)
- or located where the classloader for the Solr webapp can find it.
-
- This example schema is the recommended starting point for users.
- It should be kept correct and concise, usable out-of-the-box.
-
- For more information, on how to customize this file, please see
- http://wiki.apache.org/solr/SchemaXml
--->
-
-<schema name="test" version="1.2">
- <!-- attribute "name" is the name of this schema and is only used for display purposes.
- Applications should change this to reflect the nature of the search collection.
- version="1.1" is Solr's version number for the schema syntax and semantics. It should
- not normally be changed by applications.
- 1.0: multiValued attribute did not exist, all fields are multiValued by nature
- 1.1: multiValued attribute introduced, false by default -->
-
-
- <!-- field type definitions. The "name" attribute is
- just a label to be used by field definitions. The "class"
- attribute and any other attributes determine the real
- behavior of the fieldType.
- Class names starting with "solr" refer to java classes in the
- org.apache.solr.analysis package.
- -->
-
- <!-- The StrField type is not analyzed, but indexed/stored verbatim.
- - StrField and TextField support an optional compressThreshold which
- limits compression (if enabled in the derived fields) to values which
- exceed a certain size (in characters).
- -->
- <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
-
- <!-- boolean type: "true" or "false" -->
- <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
-
- <!-- The optional sortMissingLast and sortMissingFirst attributes are
- currently supported on types that are sorted internally as strings.
- - If sortMissingLast="true", then a sort on this field will cause documents
- without the field to come after documents with the field,
- regardless of the requested sort order (asc or desc).
- - If sortMissingFirst="true", then a sort on this field will cause documents
- without the field to come before documents with the field,
- regardless of the requested sort order.
- - If sortMissingLast="false" and sortMissingFirst="false" (the default),
- then default lucene sorting will be used which places docs without the
- field first in an ascending sort and last in a descending sort.
- -->
-
-
- <!--
- Default numeric field types. For faster range queries, consider the tint/tfloat/tlong/tdouble types.
- -->
- <fieldType name="int" class="${solr.tests.IntegerFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="float" class="${solr.tests.FloatFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="long" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="double" class="${solr.tests.DoubleFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="latLon" class="solr.LatLonType" subFieldType="double"/>
-
-
- <!--
- Numeric field types that index each value at various levels of precision
- to accelerate range queries when the number of values between the range
- endpoints is large. See the javadoc for NumericRangeQuery for internal
- implementation details.
-
- Smaller precisionStep values (specified in bits) will lead to more tokens
- indexed per value, slightly larger index size, and faster range queries.
- A precisionStep of 0 disables indexing at different precision levels.
- -->
- <fieldType name="tint" class="${solr.tests.IntegerFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tfloat" class="${solr.tests.FloatFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tlong" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tdouble" class="${solr.tests.DoubleFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
-
-
- <!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
- is a more restricted form of the canonical representation of dateTime
- http://www.w3.org/TR/xmlschema-2/#dateTime
- The trailing "Z" designates UTC time and is mandatory.
- Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
- All other components are mandatory.
-
- Expressions can also be used to denote calculations that should be
- performed relative to "NOW" to determine the value, ie...
-
- NOW/HOUR
- ... Round to the start of the current hour
- NOW-1DAY
- ... Exactly 1 day prior to now
- NOW/DAY+6MONTHS+3DAYS
- ... 6 months and 3 days in the future from the start of
- the current day
-
- Consult the TrieDateField javadocs for more information.
- -->
- <fieldType name="date" class="${solr.tests.DateFieldType}" docValues="${solr.tests.numeric.dv}" sortMissingLast="true" omitNorms="true"/>
-
-
- <!-- The "RandomSortField" is not used to store or search any
- data. You can declare fields of this type it in your schema
- to generate psuedo-random orderings of your docs for sorting
- purposes. The ordering is generated based on the field name
- and the version of the index, As long as the index version
- remains unchanged, and the same field name is reused,
- the ordering of the docs will be consistent.
- If you want differend psuedo-random orderings of documents,
- for the same version of the index, use a dynamicField and
- change the name
- -->
- <fieldType name="random" class="solr.RandomSortField" indexed="true"/>
-
- <!-- solr.TextField allows the specification of custom text analyzers
- specified as a tokenizer and a list of token filters. Different
- analyzers may be specified for indexing and querying.
-
- The optional positionIncrementGap puts space between multiple fields of
- this type on the same document, with the purpose of preventing false phrase
- matching across fields.
-
- For more info on customizing your analyzer chain, please see
- http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters
- -->
-
- <!-- One can also specify an existing Analyzer class that has a
- default constructor via the class attribute on the analyzer element
- <fieldType name="text_greek" class="solr.TextField">
- <analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/>
- </fieldType>
- -->
-
- <!-- A text field that only splits on whitespace for exact matching of words -->
- <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
- <analyzer>
- <tokenizer class="solr.MockTokenizerFactory"/>
- </analyzer>
- </fieldType>
-
- <!-- A text field that uses WordDelimiterGraphFilter to enable splitting and matching of
- words on case-change, alpha numeric boundaries, and non-alphanumeric chars,
- so that a query of "wifi" or "wi fi" could match a document containing "Wi-Fi".
- Synonyms and stopwords are customized by external files, and stemming is enabled.
- Duplicate tokens at the same position (which may result from Stemmed Synonyms or
- WordDelim parts) are removed.
- -->
- <fieldType name="text" class="solr.TextField" positionIncrementGap="100">
- <analyzer type="index">
- <tokenizer class="solr.MockTokenizerFactory"/>
- <!-- in this example, we will only use synonyms at query time
- <filter class="solr.SynonymGraphFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
- -->
- <!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
- <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1"
- catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
- <filter class="solr.LowerCaseFilterFactory"/>
- <!--<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
- <filter class="solr.PorterStemFilterFactory"/>-->
- <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
- <filter class="solr.FlattenGraphFilterFactory"/>
- </analyzer>
- <analyzer type="query">
- <tokenizer class="solr.MockTokenizerFactory"/>
- <!--<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>-->
- <!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
- <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0"
- catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
- <filter class="solr.LowerCaseFilterFactory"/>
- <!--<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
- <filter class="solr.PorterStemFilterFactory"/>-->
- <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
- </analyzer>
- </fieldType>
- <!-- since fields of this type are by default not stored or indexed, any data added to
- them will be ignored outright
- -->
- <fieldType name="ignored" stored="false" indexed="false" class="solr.StrField"/>
-
- <field name="title" type="string" indexed="true" stored="true"/>
- <field name="author" type="string" indexed="true" stored="true"/>
- <field name="text" type="text" indexed="true" stored="true"/>
- <field name="foo_i" type="int" indexed="true" stored="false"/>
- <field name="home" type="latLon" indexed="true" stored="true"/>
-</schema>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-solrconfig.xml b/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-solrconfig.xml
deleted file mode 100644
index d426320..0000000
--- a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-solrconfig.xml
+++ /dev/null
@@ -1,283 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
- <indexConfig>
- <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
- </indexConfig>
-
- <!-- Used to specify an alternate directory to hold all index data
- other than the default ./data under the Solr home.
- If replication is in use, this should match the replication configuration. -->
- <dataDir>${solr.data.dir:}</dataDir>
-
- <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
- <schemaFactory class="ClassicIndexSchemaFactory"/>
-
- <!-- the default high-performance update handler -->
- <updateHandler class="solr.DirectUpdateHandler2">
-
- <!-- A prefix of "solr." for class names is an alias that
- causes solr to search appropriate packages, including
- org.apache.solr.(search|update|request|core|analysis)
- -->
-
- <!-- Limit the number of deletions Solr will buffer during doc updating.
-
- Setting this lower can help bound memory use during indexing.
- -->
- <maxPendingDeletes>100000</maxPendingDeletes>
-
- </updateHandler>
-
-
- <query>
- <!-- Maximum number of clauses in a boolean query... can affect
- range or prefix queries that expand to big boolean
- queries. An exception is thrown if exceeded. -->
- <maxBooleanClauses>1024</maxBooleanClauses>
-
-
- <!-- Cache used by SolrIndexSearcher for filters (DocSets),
- unordered sets of *all* documents that match a query.
- When a new searcher is opened, its caches may be prepopulated
- or "autowarmed" using data from caches in the old searcher.
- autowarmCount is the number of items to prepopulate. For LRUCache,
- the autowarmed items will be the most recently accessed items.
- Parameters:
- class - the SolrCache implementation (currently only LRUCache)
- size - the maximum number of entries in the cache
- initialSize - the initial capacity (number of entries) of
- the cache. (seel java.util.HashMap)
- autowarmCount - the number of entries to prepopulate from
- and old cache.
- -->
- <filterCache
- class="solr.LRUCache"
- size="512"
- initialSize="512"
- autowarmCount="256"/>
-
- <!-- queryResultCache caches results of searches - ordered lists of
- document ids (DocList) based on a query, a sort, and the range
- of documents requested. -->
- <queryResultCache
- class="solr.LRUCache"
- size="512"
- initialSize="512"
- autowarmCount="256"/>
-
- <!-- documentCache caches Lucene Document objects (the stored fields for each document).
- Since Lucene internal document ids are transient, this cache will not be autowarmed. -->
- <documentCache
- class="solr.LRUCache"
- size="512"
- initialSize="512"
- autowarmCount="0"/>
-
- <!-- If true, stored fields that are not requested will be loaded lazily.
-
- This can result in a significant speed improvement if the usual case is to
- not load all stored fields, especially if the skipped fields are large compressed
- text fields.
- -->
- <enableLazyFieldLoading>true</enableLazyFieldLoading>
-
- <!-- Example of a generic cache. These caches may be accessed by name
- through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert().
- The purpose is to enable easy caching of user/application level data.
- The regenerator argument should be specified as an implementation
- of solr.search.CacheRegenerator if autowarming is desired. -->
- <!--
- <cache name="myUserCache"
- class="solr.LRUCache"
- size="4096"
- initialSize="1024"
- autowarmCount="1024"
- regenerator="org.mycompany.mypackage.MyRegenerator"
- />
- -->
-
- <!-- An optimization that attempts to use a filter to satisfy a search.
- If the requested sort does not include score, then the filterCache
- will be checked for a filter matching the query. If found, the filter
- will be used as the source of document ids, and then the sort will be
- applied to that.
- <useFilterForSortedQuery>true</useFilterForSortedQuery>
- -->
-
- <!-- An optimization for use with the queryResultCache. When a search
- is requested, a superset of the requested number of document ids
- are collected. For example, if a search for a particular query
- requests matching documents 10 through 19, and queryWindowSize is 50,
- then documents 0 through 49 will be collected and cached. Any further
- requests in that range can be satisfied via the cache. -->
- <queryResultWindowSize>50</queryResultWindowSize>
-
- <!-- Maximum number of documents to cache for any entry in the
- queryResultCache. -->
- <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
-
- <!-- This entry enables an int hash representation for filters (DocSets)
- when the number of items in the set is less than maxSize. For smaller
- sets, this representation is more memory efficient, more efficient to
- iterate over, and faster to take intersections. -->
- <HashDocSet maxSize="3000" loadFactor="0.75"/>
-
- <!-- a newSearcher event is fired whenever a new searcher is being prepared
- and there is a current searcher handling requests (aka registered). -->
- <!-- QuerySenderListener takes an array of NamedList and executes a
- local query request for each NamedList in sequence. -->
- <!--<listener event="newSearcher" class="solr.QuerySenderListener">-->
- <!--<arr name="queries">-->
- <!--<lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
- <!--<lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
- <!--<lst><str name="q">static newSearcher warming query from solrconfig.xml</str></lst>-->
- <!--</arr>-->
- <!--</listener>-->
-
- <!-- a firstSearcher event is fired whenever a new searcher is being
- prepared but there is no current registered searcher to handle
- requests or to gain autowarming data from. -->
- <!--<listener event="firstSearcher" class="solr.QuerySenderListener">-->
- <!--<arr name="queries">-->
- <!--</arr>-->
- <!--</listener>-->
-
- <!-- If a search request comes in and there is no current registered searcher,
- then immediately register the still warming searcher and use it. If
- "false" then all requests will block until the first searcher is done
- warming. -->
- <useColdSearcher>false</useColdSearcher>
-
- <!-- Maximum number of searchers that may be warming in the background
- concurrently. An error is returned if this limit is exceeded. Recommend
- 1-2 for read-only slaves, higher for masters w/o cache warming. -->
- <maxWarmingSearchers>4</maxWarmingSearchers>
-
- </query>
-
- <requestDispatcher>
- <!--Make sure your system has some authentication before enabling remote streaming!
- <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="-1" />
- -->
-
- <!-- Set HTTP caching related parameters (for proxy caches and clients).
-
- To get the behaviour of Solr 1.2 (ie: no caching related headers)
- use the never304="true" option and do not specify a value for
- <cacheControl>
- -->
- <httpCaching never304="true">
- <!--httpCaching lastModifiedFrom="openTime"
- etagSeed="Solr"-->
- <!-- lastModFrom="openTime" is the default, the Last-Modified value
- (and validation against If-Modified-Since requests) will all be
- relative to when the current Searcher was opened.
- You can change it to lastModFrom="dirLastMod" if you want the
- value to exactly corrispond to when the physical index was last
- modified.
-
- etagSeed="..." is an option you can change to force the ETag
- header (and validation against If-None-Match requests) to be
- differnet even if the index has not changed (ie: when making
- significant changes to your config file)
-
- lastModifiedFrom and etagSeed are both ignored if you use the
- never304="true" option.
- -->
- <!-- If you include a <cacheControl> directive, it will be used to
- generate a Cache-Control header, as well as an Expires header
- if the value contains "max-age="
-
- By default, no Cache-Control header is generated.
-
- You can use the <cacheControl> option even if you have set
- never304="true"
- -->
- <!-- <cacheControl>max-age=30, public</cacheControl> -->
- </httpCaching>
- </requestDispatcher>
-
- <requestHandler name="/select" class="solr.SearchHandler">
- <!-- default values for query parameters -->
- <lst name="defaults">
- <str name="echoParams">explicit</str>
- <!--
- <int name="rows">10</int>
- <str name="fl">*</str>
- <str name="version">2.1</str>
- -->
- </lst>
- </requestHandler>
-
- <requestHandler name="/dataimport" class="org.apache.solr.handler.dataimport.DataImportHandler">
- </requestHandler>
-
- <!--
-
- Search components are registered to SolrCore and used by Search Handlers
-
- By default, the following components are avaliable:
-
- <searchComponent name="query" class="org.apache.solr.handler.component.QueryComponent" />
- <searchComponent name="facet" class="org.apache.solr.handler.component.FacetComponent" />
- <searchComponent name="mlt" class="org.apache.solr.handler.component.MoreLikeThisComponent" />
- <searchComponent name="highlight" class="org.apache.solr.handler.component.HighlightComponent" />
- <searchComponent name="debug" class="org.apache.solr.handler.component.DebugComponent" />
-
- If you register a searchComponent to one of the standard names, that will be used instead.
-
- -->
-
- <requestHandler name="/search" class="org.apache.solr.handler.component.SearchHandler">
- <lst name="defaults">
- <str name="echoParams">explicit</str>
- </lst>
- <!--
- By default, this will register the following components:
-
- <arr name="components">
- <str>query</str>
- <str>facet</str>
- <str>mlt</str>
- <str>highlight</str>
- <str>debug</str>
- </arr>
-
- To insert handlers before or after the 'standard' components, use:
-
- <arr name="first-components">
- <str>first</str>
- </arr>
-
- <arr name="last-components">
- <str>last</str>
- </arr>
-
- -->
- </requestHandler>
-
- <!-- config for the admin interface -->
- <admin>
- <defaultQuery>*:*</defaultQuery>
- </admin>
-
-</config>
-
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/structured.html
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/structured.html b/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/structured.html
deleted file mode 100644
index 1037481..0000000
--- a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/structured.html
+++ /dev/null
@@ -1,29 +0,0 @@
-<!DOCTYPE html>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<html>
-<head>
- <title>Title in the header</title>
-</head>
-<body>
-<h1>H1 Header</h1>
-<div>Basic div</div>
-<div class="classAttribute">Div with attribute</div>
-</body>
-</html>
-
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_jpeg.jpg
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_jpeg.jpg b/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_jpeg.jpg
deleted file mode 100644
index 10d1ebb..0000000
Binary files a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_jpeg.jpg and /dev/null differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_recursive_embedded.docx
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_recursive_embedded.docx b/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_recursive_embedded.docx
deleted file mode 100644
index cd562cb..0000000
Binary files a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_recursive_embedded.docx and /dev/null differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_vsdx.vsdx
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_vsdx.vsdx b/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_vsdx.vsdx
deleted file mode 100644
index 659ecdd..0000000
Binary files a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_vsdx.vsdx and /dev/null differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test/java/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test/java/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java b/solr/contrib/dataimporthandler-extras/src/test/java/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java
new file mode 100644
index 0000000..0b00be6
--- /dev/null
+++ b/solr/contrib/dataimporthandler-extras/src/test/java/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.handler.dataimport.config.Entity;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+// Test mailbox is like this: foldername(mailcount)
+// top1(2) -> child11(6)
+// -> child12(0)
+// top2(2) -> child21(1)
+// -> grandchild211(2)
+// -> grandchild212(1)
+// -> child22(2)
+
+/**
+ * Test for MailEntityProcessor. The tests are marked as ignored because we'd need a mail server (real or mocked) for
+ * these to work.
+ *
+ * TODO: Find a way to make the tests actually test code
+ *
+ *
+ * @see org.apache.solr.handler.dataimport.MailEntityProcessor
+ * @since solr 1.4
+ */
+@Ignore("Needs a Mock Mail Server to work")
+public class TestMailEntityProcessor extends AbstractDataImportHandlerTestCase {
+
+ // Credentials
+ private static final String user = "user";
+ private static final String password = "password";
+ private static final String host = "host";
+ private static final String protocol = "imaps";
+
+ private static Map<String, String> paramMap = new HashMap<>();
+
+ @Test
+ @Ignore("Needs a Mock Mail Server to work")
+ public void testConnection() {
+ // also tests recurse = false and default settings
+ paramMap.put("folders", "top2");
+ paramMap.put("recurse", "false");
+ paramMap.put("processAttachement", "false");
+ DataImporter di = new DataImporter();
+ di.loadAndInit(getConfigFromMap(paramMap));
+ Entity ent = di.getConfig().getEntities().get(0);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals("top1 did not return 2 messages", swi.docs.size(), 2);
+ }
+
+ @Test
+ @Ignore("Needs a Mock Mail Server to work")
+ public void testRecursion() {
+ paramMap.put("folders", "top2");
+ paramMap.put("recurse", "true");
+ paramMap.put("processAttachement", "false");
+ DataImporter di = new DataImporter();
+ di.loadAndInit(getConfigFromMap(paramMap));
+ Entity ent = di.getConfig().getEntities().get(0);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals("top2 and its children did not return 8 messages", swi.docs.size(), 8);
+ }
+
+ @Test
+ @Ignore("Needs a Mock Mail Server to work")
+ public void testExclude() {
+ paramMap.put("folders", "top2");
+ paramMap.put("recurse", "true");
+ paramMap.put("processAttachement", "false");
+ paramMap.put("exclude", ".*grandchild.*");
+ DataImporter di = new DataImporter();
+ di.loadAndInit(getConfigFromMap(paramMap));
+ Entity ent = di.getConfig().getEntities().get(0);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals("top2 and its direct children did not return 5 messages", swi.docs.size(), 5);
+ }
+
+ @Test
+ @Ignore("Needs a Mock Mail Server to work")
+ public void testInclude() {
+ paramMap.put("folders", "top2");
+ paramMap.put("recurse", "true");
+ paramMap.put("processAttachement", "false");
+ paramMap.put("include", ".*grandchild.*");
+ DataImporter di = new DataImporter();
+ di.loadAndInit(getConfigFromMap(paramMap));
+ Entity ent = di.getConfig().getEntities().get(0);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals("top2 and its direct children did not return 3 messages", swi.docs.size(), 3);
+ }
+
+ @Test
+ @Ignore("Needs a Mock Mail Server to work")
+ public void testIncludeAndExclude() {
+ paramMap.put("folders", "top1,top2");
+ paramMap.put("recurse", "true");
+ paramMap.put("processAttachement", "false");
+ paramMap.put("exclude", ".*top1.*");
+ paramMap.put("include", ".*grandchild.*");
+ DataImporter di = new DataImporter();
+ di.loadAndInit(getConfigFromMap(paramMap));
+ Entity ent = di.getConfig().getEntities().get(0);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals("top2 and its direct children did not return 3 messages", swi.docs.size(), 3);
+ }
+
+ @Test
+ @Ignore("Needs a Mock Mail Server to work")
+ public void testFetchTimeSince() throws ParseException {
+ paramMap.put("folders", "top1/child11");
+ paramMap.put("recurse", "true");
+ paramMap.put("processAttachement", "false");
+ paramMap.put("fetchMailsSince", "2008-12-26 00:00:00");
+ DataImporter di = new DataImporter();
+ di.loadAndInit(getConfigFromMap(paramMap));
+ Entity ent = di.getConfig().getEntities().get(0);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals("top2 and its direct children did not return 3 messages", swi.docs.size(), 3);
+ }
+
+ private String getConfigFromMap(Map<String, String> params) {
+ String conf =
+ "<dataConfig>" +
+ "<document>" +
+ "<entity processor=\"org.apache.solr.handler.dataimport.MailEntityProcessor\" " +
+ "someconfig" +
+ "/>" +
+ "</document>" +
+ "</dataConfig>";
+ params.put("user", user);
+ params.put("password", password);
+ params.put("host", host);
+ params.put("protocol", protocol);
+ StringBuilder attribs = new StringBuilder("");
+ for (String key : params.keySet())
+ attribs.append(" ").append(key).append("=" + "\"").append(params.get(key)).append("\"");
+ attribs.append(" ");
+ return conf.replace("someconfig", attribs.toString());
+ }
+
+ static class SolrWriterImpl extends SolrWriter {
+ List<SolrInputDocument> docs = new ArrayList<>();
+ Boolean deleteAllCalled;
+ Boolean commitCalled;
+
+ public SolrWriterImpl() {
+ super(null, null);
+ }
+
+ @Override
+ public boolean upload(SolrInputDocument doc) {
+ return docs.add(doc);
+ }
+
+
+ @Override
+ public void doDeleteAll() {
+ deleteAllCalled = Boolean.TRUE;
+ }
+
+ @Override
+ public void commit(boolean b) {
+ commitCalled = Boolean.TRUE;
+ }
+ }
+}
[23/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test/java/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test/java/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java b/solr/contrib/dataimporthandler-extras/src/test/java/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java
new file mode 100644
index 0000000..05acfca
--- /dev/null
+++ b/solr/contrib/dataimporthandler-extras/src/test/java/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.Locale;
+
+/**Testcase for TikaEntityProcessor
+ *
+ * @since solr 3.1
+ */
+public class TestTikaEntityProcessor extends AbstractDataImportHandlerTestCase {
+ private String conf =
+ "<dataConfig>" +
+ " <dataSource type=\"BinFileDataSource\"/>" +
+ " <document>" +
+ " <entity name=\"Tika\" processor=\"TikaEntityProcessor\" url=\"" + getFile("dihextras/solr-word.pdf").getAbsolutePath() + "\" >" +
+ " <field column=\"Author\" meta=\"true\" name=\"author\"/>" +
+ " <field column=\"title\" meta=\"true\" name=\"title\"/>" +
+ " <field column=\"text\"/>" +
+ " </entity>" +
+ " </document>" +
+ "</dataConfig>";
+
+ private String skipOnErrConf =
+ "<dataConfig>" +
+ " <dataSource type=\"BinFileDataSource\"/>" +
+ " <document>" +
+ " <entity name=\"Tika\" onError=\"skip\" processor=\"TikaEntityProcessor\" url=\"" + getFile("dihextras/bad.doc").getAbsolutePath() + "\" >" +
+ "<field column=\"content\" name=\"text\"/>" +
+ " </entity>" +
+ " <entity name=\"Tika\" processor=\"TikaEntityProcessor\" url=\"" + getFile("dihextras/solr-word.pdf").getAbsolutePath() + "\" >" +
+ " <field column=\"text\"/>" +
+ "</entity>" +
+ " </document>" +
+ "</dataConfig>";
+
+ private String spatialConf =
+ "<dataConfig>" +
+ " <dataSource type=\"BinFileDataSource\"/>" +
+ " <document>" +
+ " <entity name=\"Tika\" processor=\"TikaEntityProcessor\" url=\"" +
+ getFile("dihextras/test_jpeg.jpg").getAbsolutePath() + "\" spatialMetadataField=\"home\">" +
+ " <field column=\"text\"/>" +
+ " </entity>" +
+ " </document>" +
+ "</dataConfig>";
+
+ private String vsdxConf =
+ "<dataConfig>" +
+ " <dataSource type=\"BinFileDataSource\"/>" +
+ " <document>" +
+ " <entity name=\"Tika\" processor=\"TikaEntityProcessor\" url=\"" + getFile("dihextras/test_vsdx.vsdx").getAbsolutePath() + "\" >" +
+ " <field column=\"text\"/>" +
+ " </entity>" +
+ " </document>" +
+ "</dataConfig>";
+
+ private String[] tests = {
+ "//*[@numFound='1']"
+ ,"//str[@name='author'][.='Grant Ingersoll']"
+ ,"//str[@name='title'][.='solr-word']"
+ ,"//str[@name='text']"
+ };
+
+ private String[] testsHTMLDefault = {
+ "//*[@numFound='1']"
+ , "//str[@name='text'][contains(.,'Basic div')]"
+ , "//str[@name='text'][contains(.,'<h1>')]"
+ , "//str[@name='text'][not(contains(.,'<div>'))]" //default mapper lower-cases elements as it maps
+ , "//str[@name='text'][not(contains(.,'<DIV>'))]"
+ };
+
+ private String[] testsHTMLIdentity = {
+ "//*[@numFound='1']"
+ , "//str[@name='text'][contains(.,'Basic div')]"
+ , "//str[@name='text'][contains(.,'<h1>')]"
+ , "//str[@name='text'][contains(.,'<div>')]"
+ , "//str[@name='text'][contains(.,'class=\"classAttribute\"')]" //attributes are lower-cased
+ };
+
+ private String[] testsSpatial = {
+ "//*[@numFound='1']"
+ };
+
+ private String[] testsEmbedded = {
+ "//*[@numFound='1']",
+ "//str[@name='text'][contains(.,'When in the Course')]"
+ };
+
+ private String[] testsIgnoreEmbedded = {
+ "//*[@numFound='1']",
+ "//str[@name='text'][not(contains(.,'When in the Course'))]"
+ };
+
+ private String[] testsVSDX = {
+ "//*[@numFound='1']",
+ "//str[@name='text'][contains(.,'Arrears')]"
+ };
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ assumeFalse("This test fails on UNIX with Turkish default locale (https://issues.apache.org/jira/browse/SOLR-6387)",
+ new Locale("tr").getLanguage().equals(Locale.getDefault().getLanguage()));
+ initCore("dataimport-solrconfig.xml", "dataimport-schema-no-unique-key.xml", getFile("dihextras/solr").getAbsolutePath());
+ }
+
+ @Test
+ public void testIndexingWithTikaEntityProcessor() throws Exception {
+ runFullImport(conf);
+ assertQ(req("*:*"), tests );
+ }
+
+ @Test
+ public void testSkip() throws Exception {
+ runFullImport(skipOnErrConf);
+ assertQ(req("*:*"), "//*[@numFound='1']");
+ }
+
+ @Test
+ public void testVSDX() throws Exception {
+ //this ensures that we've included the curvesapi dependency
+ //and that the ConnectsType class is bundled with poi-ooxml-schemas.
+ runFullImport(vsdxConf);
+ assertQ(req("*:*"), testsVSDX);
+ }
+
+ @Test
+ public void testTikaHTMLMapperEmpty() throws Exception {
+ runFullImport(getConfigHTML(null));
+ assertQ(req("*:*"), testsHTMLDefault);
+ }
+
+ @Test
+ public void testTikaHTMLMapperDefault() throws Exception {
+ runFullImport(getConfigHTML("default"));
+ assertQ(req("*:*"), testsHTMLDefault);
+ }
+
+ @Test
+ public void testTikaHTMLMapperIdentity() throws Exception {
+ runFullImport(getConfigHTML("identity"));
+ assertQ(req("*:*"), testsHTMLIdentity);
+ }
+
+ @Test
+ public void testTikaGeoMetadata() throws Exception {
+ runFullImport(spatialConf);
+ String pt = "38.97,-77.018";
+ Double distance = 5.0d;
+ assertQ(req("q", "*:* OR foo_i:" + random().nextInt(100), "fq",
+ "{!geofilt sfield=\"home\"}\"",
+ "pt", pt, "d", String.valueOf(distance)), testsSpatial);
+ }
+
+ private String getConfigHTML(String htmlMapper) {
+ return
+ "<dataConfig>" +
+ " <dataSource type='BinFileDataSource'/>" +
+ " <document>" +
+ " <entity name='Tika' format='xml' processor='TikaEntityProcessor' " +
+ " url='" + getFile("dihextras/structured.html").getAbsolutePath() + "' " +
+ ((htmlMapper == null) ? "" : (" htmlMapper='" + htmlMapper + "'")) + ">" +
+ " <field column='text'/>" +
+ " </entity>" +
+ " </document>" +
+ "</dataConfig>";
+
+ }
+
+ @Test
+ public void testEmbeddedDocsLegacy() throws Exception {
+ //test legacy behavior: ignore embedded docs
+ runFullImport(conf);
+ assertQ(req("*:*"), testsIgnoreEmbedded);
+ }
+
+ @Test
+ public void testEmbeddedDocsTrue() throws Exception {
+ runFullImport(getConfigEmbedded(true));
+ assertQ(req("*:*"), testsEmbedded);
+ }
+
+ @Test
+ public void testEmbeddedDocsFalse() throws Exception {
+ runFullImport(getConfigEmbedded(false));
+ assertQ(req("*:*"), testsIgnoreEmbedded);
+ }
+
+ private String getConfigEmbedded(boolean extractEmbedded) {
+ return
+ "<dataConfig>" +
+ " <dataSource type=\"BinFileDataSource\"/>" +
+ " <document>" +
+ " <entity name=\"Tika\" processor=\"TikaEntityProcessor\" url=\"" +
+ getFile("dihextras/test_recursive_embedded.docx").getAbsolutePath() + "\" " +
+ " extractEmbedded=\""+extractEmbedded+"\">" +
+ " <field column=\"Author\" meta=\"true\" name=\"author\"/>" +
+ " <field column=\"title\" meta=\"true\" name=\"title\"/>" +
+ " <field column=\"text\"/>" +
+ " </entity>" +
+ " </document>" +
+ "</dataConfig>";
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java b/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java
deleted file mode 100644
index 0b00be6..0000000
--- a/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.handler.dataimport.config.Entity;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.text.ParseException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-// Test mailbox is like this: foldername(mailcount)
-// top1(2) -> child11(6)
-// -> child12(0)
-// top2(2) -> child21(1)
-// -> grandchild211(2)
-// -> grandchild212(1)
-// -> child22(2)
-
-/**
- * Test for MailEntityProcessor. The tests are marked as ignored because we'd need a mail server (real or mocked) for
- * these to work.
- *
- * TODO: Find a way to make the tests actually test code
- *
- *
- * @see org.apache.solr.handler.dataimport.MailEntityProcessor
- * @since solr 1.4
- */
-@Ignore("Needs a Mock Mail Server to work")
-public class TestMailEntityProcessor extends AbstractDataImportHandlerTestCase {
-
- // Credentials
- private static final String user = "user";
- private static final String password = "password";
- private static final String host = "host";
- private static final String protocol = "imaps";
-
- private static Map<String, String> paramMap = new HashMap<>();
-
- @Test
- @Ignore("Needs a Mock Mail Server to work")
- public void testConnection() {
- // also tests recurse = false and default settings
- paramMap.put("folders", "top2");
- paramMap.put("recurse", "false");
- paramMap.put("processAttachement", "false");
- DataImporter di = new DataImporter();
- di.loadAndInit(getConfigFromMap(paramMap));
- Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals("top1 did not return 2 messages", swi.docs.size(), 2);
- }
-
- @Test
- @Ignore("Needs a Mock Mail Server to work")
- public void testRecursion() {
- paramMap.put("folders", "top2");
- paramMap.put("recurse", "true");
- paramMap.put("processAttachement", "false");
- DataImporter di = new DataImporter();
- di.loadAndInit(getConfigFromMap(paramMap));
- Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals("top2 and its children did not return 8 messages", swi.docs.size(), 8);
- }
-
- @Test
- @Ignore("Needs a Mock Mail Server to work")
- public void testExclude() {
- paramMap.put("folders", "top2");
- paramMap.put("recurse", "true");
- paramMap.put("processAttachement", "false");
- paramMap.put("exclude", ".*grandchild.*");
- DataImporter di = new DataImporter();
- di.loadAndInit(getConfigFromMap(paramMap));
- Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals("top2 and its direct children did not return 5 messages", swi.docs.size(), 5);
- }
-
- @Test
- @Ignore("Needs a Mock Mail Server to work")
- public void testInclude() {
- paramMap.put("folders", "top2");
- paramMap.put("recurse", "true");
- paramMap.put("processAttachement", "false");
- paramMap.put("include", ".*grandchild.*");
- DataImporter di = new DataImporter();
- di.loadAndInit(getConfigFromMap(paramMap));
- Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals("top2 and its direct children did not return 3 messages", swi.docs.size(), 3);
- }
-
- @Test
- @Ignore("Needs a Mock Mail Server to work")
- public void testIncludeAndExclude() {
- paramMap.put("folders", "top1,top2");
- paramMap.put("recurse", "true");
- paramMap.put("processAttachement", "false");
- paramMap.put("exclude", ".*top1.*");
- paramMap.put("include", ".*grandchild.*");
- DataImporter di = new DataImporter();
- di.loadAndInit(getConfigFromMap(paramMap));
- Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals("top2 and its direct children did not return 3 messages", swi.docs.size(), 3);
- }
-
- @Test
- @Ignore("Needs a Mock Mail Server to work")
- public void testFetchTimeSince() throws ParseException {
- paramMap.put("folders", "top1/child11");
- paramMap.put("recurse", "true");
- paramMap.put("processAttachement", "false");
- paramMap.put("fetchMailsSince", "2008-12-26 00:00:00");
- DataImporter di = new DataImporter();
- di.loadAndInit(getConfigFromMap(paramMap));
- Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals("top2 and its direct children did not return 3 messages", swi.docs.size(), 3);
- }
-
- private String getConfigFromMap(Map<String, String> params) {
- String conf =
- "<dataConfig>" +
- "<document>" +
- "<entity processor=\"org.apache.solr.handler.dataimport.MailEntityProcessor\" " +
- "someconfig" +
- "/>" +
- "</document>" +
- "</dataConfig>";
- params.put("user", user);
- params.put("password", password);
- params.put("host", host);
- params.put("protocol", protocol);
- StringBuilder attribs = new StringBuilder("");
- for (String key : params.keySet())
- attribs.append(" ").append(key).append("=" + "\"").append(params.get(key)).append("\"");
- attribs.append(" ");
- return conf.replace("someconfig", attribs.toString());
- }
-
- static class SolrWriterImpl extends SolrWriter {
- List<SolrInputDocument> docs = new ArrayList<>();
- Boolean deleteAllCalled;
- Boolean commitCalled;
-
- public SolrWriterImpl() {
- super(null, null);
- }
-
- @Override
- public boolean upload(SolrInputDocument doc) {
- return docs.add(doc);
- }
-
-
- @Override
- public void doDeleteAll() {
- deleteAllCalled = Boolean.TRUE;
- }
-
- @Override
- public void commit(boolean b) {
- commitCalled = Boolean.TRUE;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java b/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java
deleted file mode 100644
index 05acfca..0000000
--- a/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.util.Locale;
-
-/**Testcase for TikaEntityProcessor
- *
- * @since solr 3.1
- */
-public class TestTikaEntityProcessor extends AbstractDataImportHandlerTestCase {
- private String conf =
- "<dataConfig>" +
- " <dataSource type=\"BinFileDataSource\"/>" +
- " <document>" +
- " <entity name=\"Tika\" processor=\"TikaEntityProcessor\" url=\"" + getFile("dihextras/solr-word.pdf").getAbsolutePath() + "\" >" +
- " <field column=\"Author\" meta=\"true\" name=\"author\"/>" +
- " <field column=\"title\" meta=\"true\" name=\"title\"/>" +
- " <field column=\"text\"/>" +
- " </entity>" +
- " </document>" +
- "</dataConfig>";
-
- private String skipOnErrConf =
- "<dataConfig>" +
- " <dataSource type=\"BinFileDataSource\"/>" +
- " <document>" +
- " <entity name=\"Tika\" onError=\"skip\" processor=\"TikaEntityProcessor\" url=\"" + getFile("dihextras/bad.doc").getAbsolutePath() + "\" >" +
- "<field column=\"content\" name=\"text\"/>" +
- " </entity>" +
- " <entity name=\"Tika\" processor=\"TikaEntityProcessor\" url=\"" + getFile("dihextras/solr-word.pdf").getAbsolutePath() + "\" >" +
- " <field column=\"text\"/>" +
- "</entity>" +
- " </document>" +
- "</dataConfig>";
-
- private String spatialConf =
- "<dataConfig>" +
- " <dataSource type=\"BinFileDataSource\"/>" +
- " <document>" +
- " <entity name=\"Tika\" processor=\"TikaEntityProcessor\" url=\"" +
- getFile("dihextras/test_jpeg.jpg").getAbsolutePath() + "\" spatialMetadataField=\"home\">" +
- " <field column=\"text\"/>" +
- " </entity>" +
- " </document>" +
- "</dataConfig>";
-
- private String vsdxConf =
- "<dataConfig>" +
- " <dataSource type=\"BinFileDataSource\"/>" +
- " <document>" +
- " <entity name=\"Tika\" processor=\"TikaEntityProcessor\" url=\"" + getFile("dihextras/test_vsdx.vsdx").getAbsolutePath() + "\" >" +
- " <field column=\"text\"/>" +
- " </entity>" +
- " </document>" +
- "</dataConfig>";
-
- private String[] tests = {
- "//*[@numFound='1']"
- ,"//str[@name='author'][.='Grant Ingersoll']"
- ,"//str[@name='title'][.='solr-word']"
- ,"//str[@name='text']"
- };
-
- private String[] testsHTMLDefault = {
- "//*[@numFound='1']"
- , "//str[@name='text'][contains(.,'Basic div')]"
- , "//str[@name='text'][contains(.,'<h1>')]"
- , "//str[@name='text'][not(contains(.,'<div>'))]" //default mapper lower-cases elements as it maps
- , "//str[@name='text'][not(contains(.,'<DIV>'))]"
- };
-
- private String[] testsHTMLIdentity = {
- "//*[@numFound='1']"
- , "//str[@name='text'][contains(.,'Basic div')]"
- , "//str[@name='text'][contains(.,'<h1>')]"
- , "//str[@name='text'][contains(.,'<div>')]"
- , "//str[@name='text'][contains(.,'class=\"classAttribute\"')]" //attributes are lower-cased
- };
-
- private String[] testsSpatial = {
- "//*[@numFound='1']"
- };
-
- private String[] testsEmbedded = {
- "//*[@numFound='1']",
- "//str[@name='text'][contains(.,'When in the Course')]"
- };
-
- private String[] testsIgnoreEmbedded = {
- "//*[@numFound='1']",
- "//str[@name='text'][not(contains(.,'When in the Course'))]"
- };
-
- private String[] testsVSDX = {
- "//*[@numFound='1']",
- "//str[@name='text'][contains(.,'Arrears')]"
- };
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- assumeFalse("This test fails on UNIX with Turkish default locale (https://issues.apache.org/jira/browse/SOLR-6387)",
- new Locale("tr").getLanguage().equals(Locale.getDefault().getLanguage()));
- initCore("dataimport-solrconfig.xml", "dataimport-schema-no-unique-key.xml", getFile("dihextras/solr").getAbsolutePath());
- }
-
- @Test
- public void testIndexingWithTikaEntityProcessor() throws Exception {
- runFullImport(conf);
- assertQ(req("*:*"), tests );
- }
-
- @Test
- public void testSkip() throws Exception {
- runFullImport(skipOnErrConf);
- assertQ(req("*:*"), "//*[@numFound='1']");
- }
-
- @Test
- public void testVSDX() throws Exception {
- //this ensures that we've included the curvesapi dependency
- //and that the ConnectsType class is bundled with poi-ooxml-schemas.
- runFullImport(vsdxConf);
- assertQ(req("*:*"), testsVSDX);
- }
-
- @Test
- public void testTikaHTMLMapperEmpty() throws Exception {
- runFullImport(getConfigHTML(null));
- assertQ(req("*:*"), testsHTMLDefault);
- }
-
- @Test
- public void testTikaHTMLMapperDefault() throws Exception {
- runFullImport(getConfigHTML("default"));
- assertQ(req("*:*"), testsHTMLDefault);
- }
-
- @Test
- public void testTikaHTMLMapperIdentity() throws Exception {
- runFullImport(getConfigHTML("identity"));
- assertQ(req("*:*"), testsHTMLIdentity);
- }
-
- @Test
- public void testTikaGeoMetadata() throws Exception {
- runFullImport(spatialConf);
- String pt = "38.97,-77.018";
- Double distance = 5.0d;
- assertQ(req("q", "*:* OR foo_i:" + random().nextInt(100), "fq",
- "{!geofilt sfield=\"home\"}\"",
- "pt", pt, "d", String.valueOf(distance)), testsSpatial);
- }
-
- private String getConfigHTML(String htmlMapper) {
- return
- "<dataConfig>" +
- " <dataSource type='BinFileDataSource'/>" +
- " <document>" +
- " <entity name='Tika' format='xml' processor='TikaEntityProcessor' " +
- " url='" + getFile("dihextras/structured.html").getAbsolutePath() + "' " +
- ((htmlMapper == null) ? "" : (" htmlMapper='" + htmlMapper + "'")) + ">" +
- " <field column='text'/>" +
- " </entity>" +
- " </document>" +
- "</dataConfig>";
-
- }
-
- @Test
- public void testEmbeddedDocsLegacy() throws Exception {
- //test legacy behavior: ignore embedded docs
- runFullImport(conf);
- assertQ(req("*:*"), testsIgnoreEmbedded);
- }
-
- @Test
- public void testEmbeddedDocsTrue() throws Exception {
- runFullImport(getConfigEmbedded(true));
- assertQ(req("*:*"), testsEmbedded);
- }
-
- @Test
- public void testEmbeddedDocsFalse() throws Exception {
- runFullImport(getConfigEmbedded(false));
- assertQ(req("*:*"), testsIgnoreEmbedded);
- }
-
- private String getConfigEmbedded(boolean extractEmbedded) {
- return
- "<dataConfig>" +
- " <dataSource type=\"BinFileDataSource\"/>" +
- " <document>" +
- " <entity name=\"Tika\" processor=\"TikaEntityProcessor\" url=\"" +
- getFile("dihextras/test_recursive_embedded.docx").getAbsolutePath() + "\" " +
- " extractEmbedded=\""+extractEmbedded+"\">" +
- " <field column=\"Author\" meta=\"true\" name=\"author\"/>" +
- " <field column=\"title\" meta=\"true\" name=\"title\"/>" +
- " <field column=\"text\"/>" +
- " </entity>" +
- " </document>" +
- "</dataConfig>";
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/bad.doc
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/bad.doc b/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/bad.doc
new file mode 100644
index 0000000..5944c24
Binary files /dev/null and b/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/bad.doc differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/solr-word.pdf
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/solr-word.pdf b/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/solr-word.pdf
new file mode 100644
index 0000000..bd8b865
Binary files /dev/null and b/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/solr-word.pdf differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/solr/collection1/conf/dataimport-schema-no-unique-key.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/solr/collection1/conf/dataimport-schema-no-unique-key.xml b/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/solr/collection1/conf/dataimport-schema-no-unique-key.xml
new file mode 100644
index 0000000..793482a
--- /dev/null
+++ b/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/solr/collection1/conf/dataimport-schema-no-unique-key.xml
@@ -0,0 +1,205 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ This is the Solr schema file. This file should be named "schema.xml" and
+ should be in the conf directory under the solr home
+ (i.e. ./solr/conf/schema.xml by default)
+ or located where the classloader for the Solr webapp can find it.
+
+ This example schema is the recommended starting point for users.
+ It should be kept correct and concise, usable out-of-the-box.
+
+ For more information, on how to customize this file, please see
+ http://wiki.apache.org/solr/SchemaXml
+-->
+
+<schema name="test" version="1.2">
+ <!-- attribute "name" is the name of this schema and is only used for display purposes.
+ Applications should change this to reflect the nature of the search collection.
+ version="1.1" is Solr's version number for the schema syntax and semantics. It should
+ not normally be changed by applications.
+ 1.0: multiValued attribute did not exist, all fields are multiValued by nature
+ 1.1: multiValued attribute introduced, false by default -->
+
+
+ <!-- field type definitions. The "name" attribute is
+ just a label to be used by field definitions. The "class"
+ attribute and any other attributes determine the real
+ behavior of the fieldType.
+ Class names starting with "solr" refer to java classes in the
+ org.apache.solr.analysis package.
+ -->
+
+ <!-- The StrField type is not analyzed, but indexed/stored verbatim.
+ - StrField and TextField support an optional compressThreshold which
+ limits compression (if enabled in the derived fields) to values which
+ exceed a certain size (in characters).
+ -->
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
+
+ <!-- boolean type: "true" or "false" -->
+ <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
+
+ <!-- The optional sortMissingLast and sortMissingFirst attributes are
+ currently supported on types that are sorted internally as strings.
+ - If sortMissingLast="true", then a sort on this field will cause documents
+ without the field to come after documents with the field,
+ regardless of the requested sort order (asc or desc).
+ - If sortMissingFirst="true", then a sort on this field will cause documents
+ without the field to come before documents with the field,
+ regardless of the requested sort order.
+ - If sortMissingLast="false" and sortMissingFirst="false" (the default),
+ then default lucene sorting will be used which places docs without the
+ field first in an ascending sort and last in a descending sort.
+ -->
+
+
+ <!--
+ Default numeric field types. For faster range queries, consider the tint/tfloat/tlong/tdouble types.
+ -->
+ <fieldType name="int" class="${solr.tests.IntegerFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="float" class="${solr.tests.FloatFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="long" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="double" class="${solr.tests.DoubleFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="latLon" class="solr.LatLonType" subFieldType="double"/>
+
+
+ <!--
+ Numeric field types that index each value at various levels of precision
+ to accelerate range queries when the number of values between the range
+ endpoints is large. See the javadoc for NumericRangeQuery for internal
+ implementation details.
+
+ Smaller precisionStep values (specified in bits) will lead to more tokens
+ indexed per value, slightly larger index size, and faster range queries.
+ A precisionStep of 0 disables indexing at different precision levels.
+ -->
+ <fieldType name="tint" class="${solr.tests.IntegerFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tfloat" class="${solr.tests.FloatFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tlong" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tdouble" class="${solr.tests.DoubleFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" positionIncrementGap="0"/>
+
+
+ <!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
+ is a more restricted form of the canonical representation of dateTime
+ http://www.w3.org/TR/xmlschema-2/#dateTime
+ The trailing "Z" designates UTC time and is mandatory.
+ Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
+ All other components are mandatory.
+
+ Expressions can also be used to denote calculations that should be
+ performed relative to "NOW" to determine the value, ie...
+
+ NOW/HOUR
+ ... Round to the start of the current hour
+ NOW-1DAY
+ ... Exactly 1 day prior to now
+ NOW/DAY+6MONTHS+3DAYS
+ ... 6 months and 3 days in the future from the start of
+ the current day
+
+ Consult the TrieDateField javadocs for more information.
+ -->
+ <fieldType name="date" class="${solr.tests.DateFieldType}" docValues="${solr.tests.numeric.dv}" sortMissingLast="true" omitNorms="true"/>
+
+
+ <!-- The "RandomSortField" is not used to store or search any
+ data. You can declare fields of this type it in your schema
+ to generate psuedo-random orderings of your docs for sorting
+ purposes. The ordering is generated based on the field name
+ and the version of the index, As long as the index version
+ remains unchanged, and the same field name is reused,
+ the ordering of the docs will be consistent.
+ If you want differend psuedo-random orderings of documents,
+ for the same version of the index, use a dynamicField and
+ change the name
+ -->
+ <fieldType name="random" class="solr.RandomSortField" indexed="true"/>
+
+ <!-- solr.TextField allows the specification of custom text analyzers
+ specified as a tokenizer and a list of token filters. Different
+ analyzers may be specified for indexing and querying.
+
+ The optional positionIncrementGap puts space between multiple fields of
+ this type on the same document, with the purpose of preventing false phrase
+ matching across fields.
+
+ For more info on customizing your analyzer chain, please see
+ http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters
+ -->
+
+ <!-- One can also specify an existing Analyzer class that has a
+ default constructor via the class attribute on the analyzer element
+ <fieldType name="text_greek" class="solr.TextField">
+ <analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/>
+ </fieldType>
+ -->
+
+ <!-- A text field that only splits on whitespace for exact matching of words -->
+ <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.MockTokenizerFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- A text field that uses WordDelimiterGraphFilter to enable splitting and matching of
+ words on case-change, alpha numeric boundaries, and non-alphanumeric chars,
+ so that a query of "wifi" or "wi fi" could match a document containing "Wi-Fi".
+ Synonyms and stopwords are customized by external files, and stemming is enabled.
+ Duplicate tokens at the same position (which may result from Stemmed Synonyms or
+ WordDelim parts) are removed.
+ -->
+ <fieldType name="text" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.MockTokenizerFactory"/>
+ <!-- in this example, we will only use synonyms at query time
+ <filter class="solr.SynonymGraphFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
+ -->
+ <!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
+ <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1"
+ catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <!--<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+ <filter class="solr.PorterStemFilterFactory"/>-->
+ <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ <filter class="solr.FlattenGraphFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.MockTokenizerFactory"/>
+ <!--<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>-->
+ <!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
+ <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0"
+ catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <!--<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+ <filter class="solr.PorterStemFilterFactory"/>-->
+ <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ <!-- since fields of this type are by default not stored or indexed, any data added to
+ them will be ignored outright
+ -->
+ <fieldType name="ignored" stored="false" indexed="false" class="solr.StrField"/>
+
+ <field name="title" type="string" indexed="true" stored="true"/>
+ <field name="author" type="string" indexed="true" stored="true"/>
+ <field name="text" type="text" indexed="true" stored="true"/>
+ <field name="foo_i" type="int" indexed="true" stored="false"/>
+ <field name="home" type="latLon" indexed="true" stored="true"/>
+</schema>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/solr/collection1/conf/dataimport-solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/solr/collection1/conf/dataimport-solrconfig.xml b/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/solr/collection1/conf/dataimport-solrconfig.xml
new file mode 100644
index 0000000..d426320
--- /dev/null
+++ b/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/solr/collection1/conf/dataimport-solrconfig.xml
@@ -0,0 +1,283 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<config>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+ <indexConfig>
+ <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+ </indexConfig>
+
+ <!-- Used to specify an alternate directory to hold all index data
+ other than the default ./data under the Solr home.
+ If replication is in use, this should match the replication configuration. -->
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <!-- the default high-performance update handler -->
+ <updateHandler class="solr.DirectUpdateHandler2">
+
+ <!-- A prefix of "solr." for class names is an alias that
+ causes solr to search appropriate packages, including
+ org.apache.solr.(search|update|request|core|analysis)
+ -->
+
+ <!-- Limit the number of deletions Solr will buffer during doc updating.
+
+ Setting this lower can help bound memory use during indexing.
+ -->
+ <maxPendingDeletes>100000</maxPendingDeletes>
+
+ </updateHandler>
+
+
+ <query>
+ <!-- Maximum number of clauses in a boolean query... can affect
+ range or prefix queries that expand to big boolean
+ queries. An exception is thrown if exceeded. -->
+ <maxBooleanClauses>1024</maxBooleanClauses>
+
+
+ <!-- Cache used by SolrIndexSearcher for filters (DocSets),
+ unordered sets of *all* documents that match a query.
+ When a new searcher is opened, its caches may be prepopulated
+ or "autowarmed" using data from caches in the old searcher.
+ autowarmCount is the number of items to prepopulate. For LRUCache,
+ the autowarmed items will be the most recently accessed items.
+ Parameters:
+ class - the SolrCache implementation (currently only LRUCache)
+ size - the maximum number of entries in the cache
+ initialSize - the initial capacity (number of entries) of
+ the cache. (seel java.util.HashMap)
+ autowarmCount - the number of entries to prepopulate from
+ and old cache.
+ -->
+ <filterCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="256"/>
+
+ <!-- queryResultCache caches results of searches - ordered lists of
+ document ids (DocList) based on a query, a sort, and the range
+ of documents requested. -->
+ <queryResultCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="256"/>
+
+ <!-- documentCache caches Lucene Document objects (the stored fields for each document).
+ Since Lucene internal document ids are transient, this cache will not be autowarmed. -->
+ <documentCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+
+ <!-- If true, stored fields that are not requested will be loaded lazily.
+
+ This can result in a significant speed improvement if the usual case is to
+ not load all stored fields, especially if the skipped fields are large compressed
+ text fields.
+ -->
+ <enableLazyFieldLoading>true</enableLazyFieldLoading>
+
+ <!-- Example of a generic cache. These caches may be accessed by name
+ through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert().
+ The purpose is to enable easy caching of user/application level data.
+ The regenerator argument should be specified as an implementation
+ of solr.search.CacheRegenerator if autowarming is desired. -->
+ <!--
+ <cache name="myUserCache"
+ class="solr.LRUCache"
+ size="4096"
+ initialSize="1024"
+ autowarmCount="1024"
+ regenerator="org.mycompany.mypackage.MyRegenerator"
+ />
+ -->
+
+ <!-- An optimization that attempts to use a filter to satisfy a search.
+ If the requested sort does not include score, then the filterCache
+ will be checked for a filter matching the query. If found, the filter
+ will be used as the source of document ids, and then the sort will be
+ applied to that.
+ <useFilterForSortedQuery>true</useFilterForSortedQuery>
+ -->
+
+ <!-- An optimization for use with the queryResultCache. When a search
+ is requested, a superset of the requested number of document ids
+ are collected. For example, if a search for a particular query
+ requests matching documents 10 through 19, and queryWindowSize is 50,
+ then documents 0 through 49 will be collected and cached. Any further
+ requests in that range can be satisfied via the cache. -->
+ <queryResultWindowSize>50</queryResultWindowSize>
+
+ <!-- Maximum number of documents to cache for any entry in the
+ queryResultCache. -->
+ <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
+
+ <!-- This entry enables an int hash representation for filters (DocSets)
+ when the number of items in the set is less than maxSize. For smaller
+ sets, this representation is more memory efficient, more efficient to
+ iterate over, and faster to take intersections. -->
+ <HashDocSet maxSize="3000" loadFactor="0.75"/>
+
+ <!-- a newSearcher event is fired whenever a new searcher is being prepared
+ and there is a current searcher handling requests (aka registered). -->
+ <!-- QuerySenderListener takes an array of NamedList and executes a
+ local query request for each NamedList in sequence. -->
+ <!--<listener event="newSearcher" class="solr.QuerySenderListener">-->
+ <!--<arr name="queries">-->
+ <!--<lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
+ <!--<lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
+ <!--<lst><str name="q">static newSearcher warming query from solrconfig.xml</str></lst>-->
+ <!--</arr>-->
+ <!--</listener>-->
+
+ <!-- a firstSearcher event is fired whenever a new searcher is being
+ prepared but there is no current registered searcher to handle
+ requests or to gain autowarming data from. -->
+ <!--<listener event="firstSearcher" class="solr.QuerySenderListener">-->
+ <!--<arr name="queries">-->
+ <!--</arr>-->
+ <!--</listener>-->
+
+ <!-- If a search request comes in and there is no current registered searcher,
+ then immediately register the still warming searcher and use it. If
+ "false" then all requests will block until the first searcher is done
+ warming. -->
+ <useColdSearcher>false</useColdSearcher>
+
+ <!-- Maximum number of searchers that may be warming in the background
+ concurrently. An error is returned if this limit is exceeded. Recommend
+ 1-2 for read-only slaves, higher for masters w/o cache warming. -->
+ <maxWarmingSearchers>4</maxWarmingSearchers>
+
+ </query>
+
+ <requestDispatcher>
+ <!--Make sure your system has some authentication before enabling remote streaming!
+ <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="-1" />
+ -->
+
+ <!-- Set HTTP caching related parameters (for proxy caches and clients).
+
+ To get the behaviour of Solr 1.2 (ie: no caching related headers)
+ use the never304="true" option and do not specify a value for
+ <cacheControl>
+ -->
+ <httpCaching never304="true">
+ <!--httpCaching lastModifiedFrom="openTime"
+ etagSeed="Solr"-->
+ <!-- lastModFrom="openTime" is the default, the Last-Modified value
+ (and validation against If-Modified-Since requests) will all be
+ relative to when the current Searcher was opened.
+ You can change it to lastModFrom="dirLastMod" if you want the
+ value to exactly corrispond to when the physical index was last
+ modified.
+
+ etagSeed="..." is an option you can change to force the ETag
+ header (and validation against If-None-Match requests) to be
+ differnet even if the index has not changed (ie: when making
+ significant changes to your config file)
+
+ lastModifiedFrom and etagSeed are both ignored if you use the
+ never304="true" option.
+ -->
+ <!-- If you include a <cacheControl> directive, it will be used to
+ generate a Cache-Control header, as well as an Expires header
+ if the value contains "max-age="
+
+ By default, no Cache-Control header is generated.
+
+ You can use the <cacheControl> option even if you have set
+ never304="true"
+ -->
+ <!-- <cacheControl>max-age=30, public</cacheControl> -->
+ </httpCaching>
+ </requestDispatcher>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <!-- default values for query parameters -->
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <!--
+ <int name="rows">10</int>
+ <str name="fl">*</str>
+ <str name="version">2.1</str>
+ -->
+ </lst>
+ </requestHandler>
+
+ <requestHandler name="/dataimport" class="org.apache.solr.handler.dataimport.DataImportHandler">
+ </requestHandler>
+
+ <!--
+
+ Search components are registered to SolrCore and used by Search Handlers
+
+ By default, the following components are avaliable:
+
+ <searchComponent name="query" class="org.apache.solr.handler.component.QueryComponent" />
+ <searchComponent name="facet" class="org.apache.solr.handler.component.FacetComponent" />
+ <searchComponent name="mlt" class="org.apache.solr.handler.component.MoreLikeThisComponent" />
+ <searchComponent name="highlight" class="org.apache.solr.handler.component.HighlightComponent" />
+ <searchComponent name="debug" class="org.apache.solr.handler.component.DebugComponent" />
+
+ If you register a searchComponent to one of the standard names, that will be used instead.
+
+ -->
+
+ <requestHandler name="/search" class="org.apache.solr.handler.component.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ </lst>
+ <!--
+ By default, this will register the following components:
+
+ <arr name="components">
+ <str>query</str>
+ <str>facet</str>
+ <str>mlt</str>
+ <str>highlight</str>
+ <str>debug</str>
+ </arr>
+
+ To insert handlers before or after the 'standard' components, use:
+
+ <arr name="first-components">
+ <str>first</str>
+ </arr>
+
+ <arr name="last-components">
+ <str>last</str>
+ </arr>
+
+ -->
+ </requestHandler>
+
+ <!-- config for the admin interface -->
+ <admin>
+ <defaultQuery>*:*</defaultQuery>
+ </admin>
+
+</config>
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/structured.html
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/structured.html b/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/structured.html
new file mode 100644
index 0000000..1037481
--- /dev/null
+++ b/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/structured.html
@@ -0,0 +1,29 @@
+<!DOCTYPE html>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<html>
+<head>
+ <title>Title in the header</title>
+</head>
+<body>
+<h1>H1 Header</h1>
+<div>Basic div</div>
+<div class="classAttribute">Div with attribute</div>
+</body>
+</html>
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/test_jpeg.jpg
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/test_jpeg.jpg b/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/test_jpeg.jpg
new file mode 100644
index 0000000..10d1ebb
Binary files /dev/null and b/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/test_jpeg.jpg differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/test_recursive_embedded.docx
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/test_recursive_embedded.docx b/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/test_recursive_embedded.docx
new file mode 100644
index 0000000..cd562cb
Binary files /dev/null and b/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/test_recursive_embedded.docx differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/test_vsdx.vsdx
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/test_vsdx.vsdx b/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/test_vsdx.vsdx
new file mode 100644
index 0000000..659ecdd
Binary files /dev/null and b/solr/contrib/dataimporthandler-extras/src/test/resources/dihextras/test_vsdx.vsdx differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/build.gradle
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/build.gradle b/solr/contrib/dataimporthandler/build.gradle
new file mode 100644
index 0000000..3c9ce90
--- /dev/null
+++ b/solr/contrib/dataimporthandler/build.gradle
@@ -0,0 +1,25 @@
+apply plugin: 'java'
+
+task jarTest (type: Jar) {
+ from sourceSets.test.output
+ classifier = 'test'
+}
+
+configurations {
+ testOutput
+}
+
+artifacts {
+ testOutput jarTest
+}
+
+dependencies {
+ compile project(':solr:core')
+
+ testCompile project(':solr:test-framework')
+ testCompile library.hsqldb
+ testCompile library.derby
+ testCompile library.mockito_core
+ testCompile library.byte_buddy
+ testCompile library.objenesis
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/derby.log
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/derby.log b/solr/contrib/dataimporthandler/derby.log
new file mode 100644
index 0000000..c383b7a
--- /dev/null
+++ b/solr/contrib/dataimporthandler/derby.log
@@ -0,0 +1,10 @@
+----------------------------------------------------------------
+Fri Nov 02 06:23:46 CDT 2018:
+Booting Derby version The Apache Software Foundation - Apache Derby - 10.9.1.0 - (1344872): instance a816c00e-0166-d42a-2ef3-00000dfa6678
+on database directory memory:/Users/caomanhdat/workspace/lucene-solr-gradle/solr/contrib/dataimporthandler/tempDB with class loader sun.misc.Launcher$AppClassLoader@2a139a55
+Loaded from file:/Users/caomanhdat/.gradle/caches/modules-2/files-2.1/org.apache.derby/derby/10.9.1.0/4538cf5564ab3c262eec65c55fdb13965625589c/derby-10.9.1.0.jar
+java.vendor=Oracle Corporation
+java.runtime.version=1.8.0_191-b12
+user.dir=/Users/caomanhdat/workspace/lucene-solr-gradle/solr/contrib/dataimporthandler
+derby.system.home=null
+Database Class Loader started - derby.database.classpath=''
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinContentStreamDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinContentStreamDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinContentStreamDataSource.java
deleted file mode 100644
index f4b1d7a..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinContentStreamDataSource.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.common.util.ContentStream;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-
-import java.io.InputStream;
-import java.io.IOException;
-import java.util.Properties;
-/**
- * <p> A data source implementation which can be used to read binary stream from content streams. </p> <p> Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a> for more
- * details. </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 3.1
- */
-
-public class BinContentStreamDataSource extends DataSource<InputStream> {
- private ContextImpl context;
- private ContentStream contentStream;
- private InputStream in;
-
-
- @Override
- public void init(Context context, Properties initProps) {
- this.context = (ContextImpl) context;
- }
-
- @Override
- public InputStream getData(String query) {
- contentStream = context.getDocBuilder().getReqParams().getContentStream();
- if (contentStream == null)
- throw new DataImportHandlerException(SEVERE, "No stream available. The request has no body");
- try {
- return in = contentStream.getStream();
- } catch (IOException e) {
- DataImportHandlerException.wrapAndThrow(SEVERE, e);
- return null;
- }
- }
-
- @Override
- public void close() {
- if (contentStream != null) {
- try {
- if (in == null) in = contentStream.getStream();
- in.close();
- } catch (IOException e) {
- /*no op*/
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinFileDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinFileDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinFileDataSource.java
deleted file mode 100644
index dc7a0f5..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinFileDataSource.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-
-import java.io.InputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.util.Properties;
-/**
- * <p>
- * A DataSource which reads from local files
- * </p>
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 3.1
- */
-
-public class BinFileDataSource extends DataSource<InputStream>{
- protected String basePath;
- @Override
- public void init(Context context, Properties initProps) {
- basePath = initProps.getProperty(FileDataSource.BASE_PATH);
- }
-
- @Override
- public InputStream getData(String query) {
- File f = FileDataSource.getFile(basePath,query);
- try {
- return new FileInputStream(f);
- } catch (FileNotFoundException e) {
- wrapAndThrow(SEVERE,e,"Unable to open file "+f.getAbsolutePath());
- return null;
- }
- }
-
- @Override
- public void close() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinURLDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinURLDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinURLDataSource.java
deleted file mode 100644
index c1b4808..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinURLDataSource.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.*;
-import static org.apache.solr.handler.dataimport.URLDataSource.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.InputStream;
-import java.lang.invoke.MethodHandles;
-import java.net.URL;
-import java.net.URLConnection;
-import java.util.Properties;
-/**
- * <p> A data source implementation which can be used to read binary streams using HTTP. </p> <p> Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a> for more
- * details. </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 3.1
- */
-public class BinURLDataSource extends DataSource<InputStream>{
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private String baseUrl;
- private int connectionTimeout = CONNECTION_TIMEOUT;
-
- private int readTimeout = READ_TIMEOUT;
-
- private Context context;
-
- private Properties initProps;
-
- public BinURLDataSource() { }
-
- @Override
- public void init(Context context, Properties initProps) {
- this.context = context;
- this.initProps = initProps;
-
- baseUrl = getInitPropWithReplacements(BASE_URL);
- String cTimeout = getInitPropWithReplacements(CONNECTION_TIMEOUT_FIELD_NAME);
- String rTimeout = getInitPropWithReplacements(READ_TIMEOUT_FIELD_NAME);
- if (cTimeout != null) {
- try {
- connectionTimeout = Integer.parseInt(cTimeout);
- } catch (NumberFormatException e) {
- log.warn("Invalid connection timeout: " + cTimeout);
- }
- }
- if (rTimeout != null) {
- try {
- readTimeout = Integer.parseInt(rTimeout);
- } catch (NumberFormatException e) {
- log.warn("Invalid read timeout: " + rTimeout);
- }
- }
- }
-
- @Override
- public InputStream getData(String query) {
- URL url = null;
- try {
- if (URIMETHOD.matcher(query).find()) url = new URL(query);
- else url = new URL(baseUrl + query);
- log.debug("Accessing URL: " + url.toString());
- URLConnection conn = url.openConnection();
- conn.setConnectTimeout(connectionTimeout);
- conn.setReadTimeout(readTimeout);
- return conn.getInputStream();
- } catch (Exception e) {
- log.error("Exception thrown while getting data", e);
- wrapAndThrow (SEVERE, e, "Exception in invoking url " + url);
- return null;//unreachable
- }
- }
-
- @Override
- public void close() { }
-
- private String getInitPropWithReplacements(String propertyName) {
- final String expr = initProps.getProperty(propertyName);
- if (expr == null) {
- return null;
- }
- return context.replaceTokens(expr);
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/CachePropertyUtil.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/CachePropertyUtil.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/CachePropertyUtil.java
deleted file mode 100644
index 544761f..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/CachePropertyUtil.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-public class CachePropertyUtil {
- public static String getAttributeValueAsString(Context context, String attr) {
- Object o = context.getSessionAttribute(attr, Context.SCOPE_ENTITY);
- if (o == null) {
- o = context.getResolvedEntityAttribute(attr);
- }
- if (o == null && context.getRequestParameters() != null) {
- o = context.getRequestParameters().get(attr);
- }
- if (o == null) {
- return null;
- }
- return o.toString();
- }
-
- public static Object getAttributeValue(Context context, String attr) {
- Object o = context.getSessionAttribute(attr, Context.SCOPE_ENTITY);
- if (o == null) {
- o = context.getResolvedEntityAttribute(attr);
- }
- if (o == null && context.getRequestParameters() != null) {
- o = context.getRequestParameters().get(attr);
- }
- if (o == null) {
- return null;
- }
- return o;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ClobTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ClobTransformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ClobTransformer.java
deleted file mode 100644
index 4bbe046..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ClobTransformer.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.HTMLStripTransformer.TRUE;
-
-import java.io.IOException;
-import java.io.Reader;
-import java.sql.Clob;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * {@link Transformer} instance which converts a {@link Clob} to a {@link String}.
- * <p>
- * Refer to <a href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * <p>
- * <b>This API is experimental and subject to change</b>
- *
- * @since solr 1.4
- */
-public class ClobTransformer extends Transformer {
- @Override
- public Object transformRow(Map<String, Object> aRow, Context context) {
- for (Map<String, String> map : context.getAllEntityFields()) {
- if (!TRUE.equals(map.get(CLOB))) continue;
- String column = map.get(DataImporter.COLUMN);
- String srcCol = map.get(RegexTransformer.SRC_COL_NAME);
- if (srcCol == null)
- srcCol = column;
- Object o = aRow.get(srcCol);
- if (o instanceof List) {
- List<Clob> inputs = (List<Clob>) o;
- List<String> results = new ArrayList<>();
- for (Object input : inputs) {
- if (input instanceof Clob) {
- Clob clob = (Clob) input;
- results.add(readFromClob(clob));
- }
- }
- aRow.put(column, results);
- } else {
- if (o instanceof Clob) {
- Clob clob = (Clob) o;
- aRow.put(column, readFromClob(clob));
- }
- }
- }
- return aRow;
- }
-
- private String readFromClob(Clob clob) {
- Reader reader = FieldReaderDataSource.readCharStream(clob);
- StringBuilder sb = new StringBuilder();
- char[] buf = new char[1024];
- int len;
- try {
- while ((len = reader.read(buf)) != -1) {
- sb.append(buf, 0, len);
- }
- } catch (IOException e) {
- DataImportHandlerException.wrapAndThrow(DataImportHandlerException.SEVERE, e);
- }
- return sb.toString();
- }
-
- public static final String CLOB = "clob";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ConfigParseUtil.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ConfigParseUtil.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ConfigParseUtil.java
deleted file mode 100644
index 179df23..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ConfigParseUtil.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-import org.w3c.dom.Element;
-import org.w3c.dom.NamedNodeMap;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-
-public class ConfigParseUtil {
- public static String getStringAttribute(Element e, String name, String def) {
- String r = e.getAttribute(name);
- if (r == null || "".equals(r.trim()))
- r = def;
- return r;
- }
-
- public static HashMap<String, String> getAllAttributes(Element e) {
- HashMap<String, String> m = new HashMap<>();
- NamedNodeMap nnm = e.getAttributes();
- for (int i = 0; i < nnm.getLength(); i++) {
- m.put(nnm.item(i).getNodeName(), nnm.item(i).getNodeValue());
- }
- return m;
- }
-
- public static String getText(Node elem, StringBuilder buffer) {
- if (elem.getNodeType() != Node.CDATA_SECTION_NODE) {
- NodeList childs = elem.getChildNodes();
- for (int i = 0; i < childs.getLength(); i++) {
- Node child = childs.item(i);
- short childType = child.getNodeType();
- if (childType != Node.COMMENT_NODE
- && childType != Node.PROCESSING_INSTRUCTION_NODE) {
- getText(child, buffer);
- }
- }
- } else {
- buffer.append(elem.getNodeValue());
- }
-
- return buffer.toString();
- }
-
- public static List<Element> getChildNodes(Element e, String byName) {
- List<Element> result = new ArrayList<>();
- NodeList l = e.getChildNodes();
- for (int i = 0; i < l.getLength(); i++) {
- if (e.equals(l.item(i).getParentNode())
- && byName.equals(l.item(i).getNodeName()))
- result.add((Element) l.item(i));
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContentStreamDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContentStreamDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContentStreamDataSource.java
deleted file mode 100644
index 4482160..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContentStreamDataSource.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.common.util.ContentStream;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-
-import java.io.IOException;
-import java.io.Reader;
-import java.util.Properties;
-
-/**
- * A DataSource implementation which reads from the ContentStream of a POST request
- * <p>
- * Refer to <a href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 1.4
- */
-public class ContentStreamDataSource extends DataSource<Reader> {
- private ContextImpl context;
- private ContentStream contentStream;
- private Reader reader;
-
- @Override
- public void init(Context context, Properties initProps) {
- this.context = (ContextImpl) context;
- }
-
- @Override
- public Reader getData(String query) {
- contentStream = context.getDocBuilder().getReqParams().getContentStream();
- if (contentStream == null)
- throw new DataImportHandlerException(SEVERE, "No stream available. The request has no body");
- try {
- return reader = contentStream.getReader();
- } catch (IOException e) {
- DataImportHandlerException.wrapAndThrow(SEVERE, e);
- return null;
- }
- }
-
- @Override
- public void close() {
- if (contentStream != null) {
- try {
- if (reader == null) reader = contentStream.getReader();
- reader.close();
- } catch (IOException e) {
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Context.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Context.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Context.java
deleted file mode 100644
index 4d0cb3a..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Context.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.core.SolrCore;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * <p>
- * This abstract class gives access to all available objects. So any
- * component implemented by a user can have the full power of DataImportHandler
- * </p>
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
- * <p>
- * <b>This API is experimental and subject to change</b>
- *
- * @since solr 1.3
- */
-public abstract class Context {
- public static final String FULL_DUMP = "FULL_DUMP", DELTA_DUMP = "DELTA_DUMP", FIND_DELTA = "FIND_DELTA";
-
- /**
- * An object stored in entity scope is valid only for the current entity for the current document only.
- */
- public static final String SCOPE_ENTITY = "entity";
-
- /**
- * An object stored in global scope is available for the current import only but across entities and documents.
- */
- public static final String SCOPE_GLOBAL = "global";
-
- /**
- * An object stored in document scope is available for the current document only but across entities.
- */
- public static final String SCOPE_DOC = "document";
-
- /**
- * An object stored in 'solrcore' scope is available across imports, entities and documents throughout the life of
- * a solr core. A solr core unload or reload will destroy this data.
- */
- public static final String SCOPE_SOLR_CORE = "solrcore";
-
- /**
- * Get the value of any attribute put into this entity
- *
- * @param name name of the attribute eg: 'name'
- * @return value of named attribute in entity
- */
- public abstract String getEntityAttribute(String name);
-
- /**
- * Get the value of any attribute put into this entity after resolving all variables found in the attribute value
- * @param name name of the attribute
- * @return value of the named attribute after resolving all variables
- */
- public abstract String getResolvedEntityAttribute(String name);
-
- /**
- * Returns all the fields put into an entity. each item (which is a map ) in
- * the list corresponds to one field. each if the map contains the attribute
- * names and values in a field
- *
- * @return all fields in an entity
- */
- public abstract List<Map<String, String>> getAllEntityFields();
-
- /**
- * Returns the VariableResolver used in this entity which can be used to
- * resolve the tokens in ${<namespce.name>}
- *
- * @return a VariableResolver instance
- * @see org.apache.solr.handler.dataimport.VariableResolver
- */
-
- public abstract VariableResolver getVariableResolver();
-
- /**
- * Gets the datasource instance defined for this entity. Do not close() this instance.
- * Transformers should use the getDataSource(String name) method.
- *
- * @return a new DataSource instance as configured for the current entity
- * @see org.apache.solr.handler.dataimport.DataSource
- * @see #getDataSource(String)
- */
- public abstract DataSource getDataSource();
-
- /**
- * Gets a new DataSource instance with a name. Ensure that you close() this after use
- * because this is created just for this method call.
- *
- * @param name Name of the dataSource as defined in the dataSource tag
- * @return a new DataSource instance
- * @see org.apache.solr.handler.dataimport.DataSource
- */
- public abstract DataSource getDataSource(String name);
-
- /**
- * Returns the instance of EntityProcessor used for this entity
- *
- * @return instance of EntityProcessor used for the current entity
- * @see org.apache.solr.handler.dataimport.EntityProcessor
- */
- public abstract EntityProcessor getEntityProcessor();
-
- /**
- * Store values in a certain name and scope (entity, document,global)
- *
- * @param name the key
- * @param val the value
- * @param scope the scope in which the given key, value pair is to be stored
- */
- public abstract void setSessionAttribute(String name, Object val, String scope);
-
- /**
- * get a value by name in the given scope (entity, document,global)
- *
- * @param name the key
- * @param scope the scope from which the value is to be retrieved
- * @return the object stored in the given scope with the given key
- */
- public abstract Object getSessionAttribute(String name, String scope);
-
- /**
- * Get the context instance for the parent entity. works only in the full dump
- * If the current entity is rootmost a null is returned
- *
- * @return parent entity's Context
- */
- public abstract Context getParentContext();
-
- /**
- * The request parameters passed over HTTP for this command the values in the
- * map are either String(for single valued parameters) or List<String> (for
- * multi-valued parameters)
- *
- * @return the request parameters passed in the URL to initiate this process
- */
- public abstract Map<String, Object> getRequestParameters();
-
- /**
- * Returns if the current entity is the root entity
- *
- * @return true if current entity is the root entity, false otherwise
- */
- public abstract boolean isRootEntity();
-
- /**
- * Returns the current process FULL_DUMP, DELTA_DUMP, FIND_DELTA
- *
- * @return the type of the current running process
- */
- public abstract String currentProcess();
-
- /**
- * Exposing the actual SolrCore to the components
- *
- * @return the core
- */
- public abstract SolrCore getSolrCore();
-
- /**
- * Makes available some basic running statistics such as "docCount",
- * "deletedDocCount", "rowCount", "queryCount" and "skipDocCount"
- *
- * @return a Map containing running statistics of the current import
- */
- public abstract Map<String, Object> getStats();
-
- /**
- * Returns the text specified in the script tag in the data-config.xml
- */
- public abstract String getScript();
-
- /**
- * Returns the language of the script as specified in the script tag in data-config.xml
- */
- public abstract String getScriptLanguage();
-
- /**delete a document by id
- */
- public abstract void deleteDoc(String id);
-
- /**delete documents by query
- */
- public abstract void deleteDocByQuery(String query);
-
- /**Use this directly to resolve variable
- * @param var the variable name
- * @return the resolved value
- */
- public abstract Object resolve(String var);
-
- /** Resolve variables in a template
- *
- * @return The string w/ variables resolved
- */
- public abstract String replaceTokens(String template);
-
-}
[06/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java
deleted file mode 100644
index 1a0d888..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.OutputStream;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import junit.framework.Assert;
-
-import org.apache.solr.request.LocalSolrQueryRequest;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-
-/**
- * This sets up an in-memory Sql database with a little sample data.
- */
-public abstract class AbstractDIHJdbcTestCase extends
- AbstractDataImportHandlerTestCase {
-
- protected Database dbToUse;
-
- public enum Database {
- RANDOM, DERBY, HSQLDB
- }
-
- protected boolean skipThisTest = false;
-
- private static final Pattern totalRequestsPattern = Pattern
- .compile(".str name..Total Requests made to DataSource..(\\d+)..str.");
-
- @BeforeClass
- public static void beforeClassDihJdbcTest() throws Exception {
- try {
- Class.forName("org.hsqldb.jdbcDriver").newInstance();
- String oldProp = System.getProperty("derby.stream.error.field");
- System
- .setProperty("derby.stream.error.field",
- "org.apache.solr.handler.dataimport.AbstractDIHJdbcTestCase$DerbyUtil.DEV_NULL");
- Class.forName("org.apache.derby.jdbc.EmbeddedDriver").newInstance();
- if (oldProp != null) {
- System.setProperty("derby.stream.error.field", oldProp);
- }
- } catch (Exception e) {
- throw e;
- }
- initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
- }
-
- @AfterClass
- public static void afterClassDihJdbcTest() throws Exception {
- try {
- DriverManager.getConnection("jdbc:derby:;shutdown=true");
- } catch (SQLException e) {
- // ignore...we might not even be using derby this time...
- }
- }
-
- protected Database setAllowedDatabases() {
- return Database.RANDOM;
- }
-
- @Before
- public void beforeDihJdbcTest() throws Exception {
- skipThisTest = false;
- dbToUse = setAllowedDatabases();
- if (dbToUse == Database.RANDOM) {
- if (random().nextBoolean()) {
- dbToUse = Database.DERBY;
- } else {
- dbToUse = Database.HSQLDB;
- }
- }
-
- clearIndex();
- assertU(commit());
- buildDatabase();
- }
-
- @After
- public void afterDihJdbcTest() throws Exception {
- Connection conn = null;
- Statement s = null;
- try {
- if (dbToUse == Database.DERBY) {
- try {
- conn = DriverManager
- .getConnection("jdbc:derby:memory:derbyDB;drop=true;territory=en_US");
- } catch (SQLException e) {
- if (!"08006".equals(e.getSQLState())) {
- throw e;
- }
- }
- } else if (dbToUse == Database.HSQLDB) {
- conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
- s = conn.createStatement();
- s.executeUpdate("shutdown");
- }
- } catch (SQLException e) {
- if(!skipThisTest) {
- throw e;
- }
- } finally {
- try {
- s.close();
- } catch (Exception ex) {}
- try {
- conn.close();
- } catch (Exception ex) {}
- }
- }
-
- protected Connection newConnection() throws Exception {
- if (dbToUse == Database.DERBY) {
- return DriverManager.getConnection("jdbc:derby:memory:derbyDB;territory=en_US");
- } else if (dbToUse == Database.HSQLDB) {
- return DriverManager.getConnection("jdbc:hsqldb:mem:.");
- }
- throw new AssertionError("Invalid database to use: " + dbToUse);
- }
-
- protected void buildDatabase() throws Exception {
- Connection conn = null;
- try {
- if (dbToUse == Database.DERBY) {
- conn = DriverManager
- .getConnection("jdbc:derby:memory:derbyDB;create=true;territory=en_US");
- } else if (dbToUse == Database.HSQLDB) {
- conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
- } else {
- throw new AssertionError("Invalid database to use: " + dbToUse);
- }
- populateData(conn);
- } catch (SQLException sqe) {
- Throwable cause = sqe;
- while(cause.getCause()!=null) {
- cause = cause.getCause();
- }
- } finally {
- try {
- conn.close();
- } catch (Exception e1) {}
- }
- }
-
- protected void populateData(Connection conn) throws Exception {
- // no-op
- }
-
- public int totalDatabaseRequests(String dihHandlerName) throws Exception {
- LocalSolrQueryRequest request = lrf.makeRequest("indent", "true");
- String response = h.query(dihHandlerName, request);
- Matcher m = totalRequestsPattern.matcher(response);
- Assert.assertTrue("The handler " + dihHandlerName
- + " is not reporting any database requests. ",
- m.find() && m.groupCount() == 1);
- return Integer.parseInt(m.group(1));
- }
-
- public int totalDatabaseRequests() throws Exception {
- return totalDatabaseRequests("/dataimport");
- }
-
- protected LocalSolrQueryRequest generateRequest() {
- return lrf.makeRequest("command", "full-import", "dataConfig",
- generateConfig(), "clean", "true", "commit", "true", "synchronous",
- "true", "indent", "true");
- }
-
- protected abstract String generateConfig();
-
- public static class DerbyUtil {
- public static final OutputStream DEV_NULL = new OutputStream() {
- @Override
- public void write(int b) {}
- };
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java
deleted file mode 100644
index 7b8ff88..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.SuppressForbidden;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.request.LocalSolrQueryRequest;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.update.AddUpdateCommand;
-import org.apache.solr.update.CommitUpdateCommand;
-import org.apache.solr.update.DeleteUpdateCommand;
-import org.apache.solr.update.MergeIndexesCommand;
-import org.apache.solr.update.RollbackUpdateCommand;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
-import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
-import org.junit.Before;
-
-/**
- * <p>
- * Abstract base class for DataImportHandler tests
- * </p>
- * <p>
- * <b>This API is experimental and subject to change</b>
- *
- *
- * @since solr 1.3
- */
-public abstract class AbstractDataImportHandlerTestCase extends
- SolrTestCaseJ4 {
-
- // note, a little twisted that we shadow this static method
- public static void initCore(String config, String schema) throws Exception {
- File testHome = createTempDir("core-home").toFile();
- FileUtils.copyDirectory(getFile("dih/solr"), testHome);
- initCore(config, schema, testHome.getAbsolutePath());
- }
-
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- File home = createTempDir("dih-properties").toFile();
- System.setProperty("solr.solr.home", home.getAbsolutePath());
- }
-
- protected String loadDataConfig(String dataConfigFileName) {
- try {
- SolrCore core = h.getCore();
- return SolrWriter.getResourceAsString(core.getResourceLoader()
- .openResource(dataConfigFileName));
- } catch (IOException e) {
- e.printStackTrace();
- return null;
- }
- }
-
- protected String runFullImport(String dataConfig) throws Exception {
- LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import",
- "debug", "on", "clean", "true", "commit", "true", "dataConfig",
- dataConfig);
- return h.query("/dataimport", request);
- }
-
- protected void runDeltaImport(String dataConfig) throws Exception {
- LocalSolrQueryRequest request = lrf.makeRequest("command", "delta-import",
- "debug", "on", "clean", "false", "commit", "true", "dataConfig",
- dataConfig);
- h.query("/dataimport", request);
- }
-
- /**
- * Redirect {@link SimplePropertiesWriter#filename} to a temporary location
- * and return it.
- */
- protected File redirectTempProperties(DataImporter di) {
- try {
- File tempFile = createTempFile().toFile();
- di.getConfig().getPropertyWriter().getParameters()
- .put(SimplePropertiesWriter.FILENAME, tempFile.getAbsolutePath());
- return tempFile;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Runs a full-import using the given dataConfig and the provided request parameters.
- *
- * By default, debug=on, clean=true and commit=true are passed which can be overridden.
- *
- * @param dataConfig the data-config xml as a string
- * @param extraParams any extra request parameters needed to be passed to DataImportHandler
- * @throws Exception in case of any error
- */
- protected void runFullImport(String dataConfig, Map<String, String> extraParams) throws Exception {
- HashMap<String, String> params = new HashMap<>();
- params.put("command", "full-import");
- params.put("debug", "on");
- params.put("dataConfig", dataConfig);
- params.put("clean", "true");
- params.put("commit", "true");
- params.putAll(extraParams);
- NamedList l = new NamedList();
- for (Map.Entry<String, String> e : params.entrySet()) {
- l.add(e.getKey(),e.getValue());
- }
- LocalSolrQueryRequest request = new LocalSolrQueryRequest(h.getCore(), l);
- h.query("/dataimport", request);
- }
-
- /**
- * Helper for creating a Context instance. Useful for testing Transformers
- */
- @SuppressWarnings("unchecked")
- public static TestContext getContext(EntityProcessorWrapper parent,
- VariableResolver resolver, DataSource parentDataSource,
- String currProcess, final List<Map<String, String>> entityFields,
- final Map<String, String> entityAttrs) {
- if (resolver == null) resolver = new VariableResolver();
- final Context delegate = new ContextImpl(parent, resolver,
- parentDataSource, currProcess,
- new HashMap<>(), null, null);
- return new TestContext(entityAttrs, delegate, entityFields, parent == null);
- }
-
- /**
- * Strings at even index are keys, odd-index strings are values in the
- * returned map
- */
- @SuppressWarnings("unchecked")
- public static Map createMap(Object... args) {
- return Utils.makeMap(args);
- }
-
- @SuppressForbidden(reason = "Needs currentTimeMillis to set modified time for a file")
- public static File createFile(File tmpdir, String name, byte[] content,
- boolean changeModifiedTime) throws IOException {
- File file = new File(tmpdir.getAbsolutePath() + File.separator + name);
- file.deleteOnExit();
- FileOutputStream f = new FileOutputStream(file);
- f.write(content);
- f.close();
- if (changeModifiedTime)
- file.setLastModified(System.currentTimeMillis() - 3600000);
- return file;
- }
-
- public static Map<String, String> getField(String col, String type,
- String re, String srcCol, String splitBy) {
- HashMap<String, String> vals = new HashMap<>();
- vals.put("column", col);
- vals.put("type", type);
- vals.put("regex", re);
- vals.put("sourceColName", srcCol);
- vals.put("splitBy", splitBy);
- return vals;
- }
-
- static class TestContext extends Context {
- private final Map<String, String> entityAttrs;
- private final Context delegate;
- private final List<Map<String, String>> entityFields;
- private final boolean root;
- String script,scriptlang;
-
- public TestContext(Map<String, String> entityAttrs, Context delegate,
- List<Map<String, String>> entityFields, boolean root) {
- this.entityAttrs = entityAttrs;
- this.delegate = delegate;
- this.entityFields = entityFields;
- this.root = root;
- }
-
- @Override
- public String getEntityAttribute(String name) {
- return entityAttrs == null ? delegate.getEntityAttribute(name) : entityAttrs.get(name);
- }
-
- @Override
- public String getResolvedEntityAttribute(String name) {
- return entityAttrs == null ? delegate.getResolvedEntityAttribute(name) :
- delegate.getVariableResolver().replaceTokens(entityAttrs.get(name));
- }
-
- @Override
- public List<Map<String, String>> getAllEntityFields() {
- return entityFields == null ? delegate.getAllEntityFields()
- : entityFields;
- }
-
- @Override
- public VariableResolver getVariableResolver() {
- return delegate.getVariableResolver();
- }
-
- @Override
- public DataSource getDataSource() {
- return delegate.getDataSource();
- }
-
- @Override
- public boolean isRootEntity() {
- return root;
- }
-
- @Override
- public String currentProcess() {
- return delegate.currentProcess();
- }
-
- @Override
- public Map<String, Object> getRequestParameters() {
- return delegate.getRequestParameters();
- }
-
- @Override
- public EntityProcessor getEntityProcessor() {
- return null;
- }
-
- @Override
- public void setSessionAttribute(String name, Object val, String scope) {
- delegate.setSessionAttribute(name, val, scope);
- }
-
- @Override
- public Object getSessionAttribute(String name, String scope) {
- return delegate.getSessionAttribute(name, scope);
- }
-
- @Override
- public Context getParentContext() {
- return delegate.getParentContext();
- }
-
- @Override
- public DataSource getDataSource(String name) {
- return delegate.getDataSource(name);
- }
-
- @Override
- public SolrCore getSolrCore() {
- return delegate.getSolrCore();
- }
-
- @Override
- public Map<String, Object> getStats() {
- return delegate.getStats();
- }
-
-
- @Override
- public String getScript() {
- return script == null ? delegate.getScript() : script;
- }
-
- @Override
- public String getScriptLanguage() {
- return scriptlang == null ? delegate.getScriptLanguage() : scriptlang;
- }
-
- @Override
- public void deleteDoc(String id) {
-
- }
-
- @Override
- public void deleteDocByQuery(String query) {
-
- }
-
- @Override
- public Object resolve(String var) {
- return delegate.resolve(var);
- }
-
- @Override
- public String replaceTokens(String template) {
- return delegate.replaceTokens(template);
- }
- }
-
- public static class TestUpdateRequestProcessorFactory extends UpdateRequestProcessorFactory {
-
- @Override
- public UpdateRequestProcessor getInstance(SolrQueryRequest req,
- SolrQueryResponse rsp, UpdateRequestProcessor next) {
- return new TestUpdateRequestProcessor(next);
- }
-
- }
-
- public static class TestUpdateRequestProcessor extends UpdateRequestProcessor {
-
- public static boolean finishCalled = false;
- public static boolean processAddCalled = false;
- public static boolean processCommitCalled = false;
- public static boolean processDeleteCalled = false;
- public static boolean mergeIndexesCalled = false;
- public static boolean rollbackCalled = false;
-
- public static void reset() {
- finishCalled = false;
- processAddCalled = false;
- processCommitCalled = false;
- processDeleteCalled = false;
- mergeIndexesCalled = false;
- rollbackCalled = false;
- }
-
- public TestUpdateRequestProcessor(UpdateRequestProcessor next) {
- super(next);
- reset();
- }
-
- @Override
- public void finish() throws IOException {
- finishCalled = true;
- super.finish();
- }
-
- @Override
- public void processAdd(AddUpdateCommand cmd) throws IOException {
- processAddCalled = true;
- super.processAdd(cmd);
- }
-
- @Override
- public void processCommit(CommitUpdateCommand cmd) throws IOException {
- processCommitCalled = true;
- super.processCommit(cmd);
- }
-
- @Override
- public void processDelete(DeleteUpdateCommand cmd) throws IOException {
- processDeleteCalled = true;
- super.processDelete(cmd);
- }
-
- @Override
- public void processMergeIndexes(MergeIndexesCommand cmd) throws IOException {
- mergeIndexesCalled = true;
- super.processMergeIndexes(cmd);
- }
-
- @Override
- public void processRollback(RollbackUpdateCommand cmd) throws IOException {
- rollbackCalled = true;
- super.processRollback(cmd);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java
deleted file mode 100644
index 2774044..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java
+++ /dev/null
@@ -1,845 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import junit.framework.Assert;
-import org.apache.solr.common.util.SuppressForbidden;
-import org.junit.After;
-import org.junit.Before;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.lang.invoke.MethodHandles;
-import java.nio.file.Files;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.sql.Timestamp;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-
-public abstract class AbstractSqlEntityProcessorTestCase extends
- AbstractDIHJdbcTestCase {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- protected boolean underlyingDataModified;
- protected boolean useSimpleCaches;
- protected boolean countryEntity;
- protected boolean countryCached;
- protected boolean countryZipper;
- protected boolean sportsEntity;
- protected boolean sportsCached;
- protected boolean sportsZipper;
-
- protected boolean wrongPeopleOrder ;
- protected boolean wrongSportsOrder ;
- protected boolean wrongCountryOrder;
-
- protected String rootTransformerName;
- protected boolean countryTransformer;
- protected boolean sportsTransformer;
- protected String fileLocation;
- protected String fileName;
-
- @Before
- public void beforeSqlEntitiyProcessorTestCase() throws Exception {
- File tmpdir = createTempDir().toFile();
- fileLocation = tmpdir.getPath();
- fileName = "the.properties";
- }
-
- @After
- public void afterSqlEntitiyProcessorTestCase() throws Exception {
- useSimpleCaches = false;
- countryEntity = false;
- countryCached = false;
- countryZipper = false;
- sportsEntity = false;
- sportsCached = false;
- sportsZipper = false;
-
- wrongPeopleOrder = false;
- wrongSportsOrder = false;
- wrongCountryOrder= false;
-
- rootTransformerName = null;
- countryTransformer = false;
- sportsTransformer = false;
- underlyingDataModified = false;
-
- //If an Assume was tripped while setting up the test,
- //the file might not ever have been created...
- if(fileLocation!=null) {
- Files.deleteIfExists(new File(fileLocation + File.separatorChar + fileName).toPath());
- Files.deleteIfExists(new File(fileLocation).toPath());
- }
- }
-
- protected void logPropertiesFile() {
- Map<String,String> init = new HashMap<>();
- init.put("filename", fileName);
- init.put("directory", fileLocation);
- SimplePropertiesWriter spw = new SimplePropertiesWriter();
- spw.init(new DataImporter(), init);
- Map<String,Object> props = spw.readIndexerProperties();
- if(props!=null) {
- StringBuilder sb = new StringBuilder();
- sb.append("\ndataimporter.properties: \n");
- for(Map.Entry<String,Object> entry : props.entrySet()) {
- sb.append(" > key=" + entry.getKey() + " / value=" + entry.getValue() + "\n");
- }
- log.debug(sb.toString());
- }
- }
-
- protected abstract String deltaQueriesCountryTable();
-
- protected abstract String deltaQueriesPersonTable();
-
- protected void singleEntity(int numToExpect) throws Exception {
- h.query("/dataimport", generateRequest());
- assertQ("There should be 1 document per person in the database: "
- + totalPeople(), req("*:*"), "//*[@numFound='" + totalPeople() + "']");
- Assert.assertTrue("Expecting " + numToExpect
- + " database calls, but DIH reported " + totalDatabaseRequests(),
- totalDatabaseRequests() == numToExpect);
- }
-
- protected void simpleTransform(int numToExpect) throws Exception {
- rootTransformerName = "AddAColumnTransformer";
- h.query("/dataimport", generateRequest());
- assertQ(
- "There should be 1 document with a transformer-added column per person is the database: "
- + totalPeople(), req("AddAColumn_s:Added"), "//*[@numFound='"
- + totalPeople() + "']");
- Assert.assertTrue("Expecting " + numToExpect
- + " database calls, but DIH reported " + totalDatabaseRequests(),
- totalDatabaseRequests() == numToExpect);
- }
-
- /**
- * A delta update will not clean up documents added by a transformer even if
- * the parent document that the transformer used to base the new documents
- * were deleted
- */
- protected void complexTransform(int numToExpect, int numDeleted)
- throws Exception {
- rootTransformerName = "TripleThreatTransformer";
- h.query("/dataimport", generateRequest());
- int totalDocs = ((totalPeople() * 3) + (numDeleted * 2));
- int totalAddedDocs = (totalPeople() + numDeleted);
- assertQ(
- req("q", "*:*", "rows", "" + (totalPeople() * 3), "sort", "id asc"),
- "//*[@numFound='" + totalDocs + "']");
- assertQ(req("id:TripleThreat-1-*"), "//*[@numFound='" + totalAddedDocs
- + "']");
- assertQ(req("id:TripleThreat-2-*"), "//*[@numFound='" + totalAddedDocs
- + "']");
- if (personNameExists("Michael") && countryCodeExists("NR")) {
- assertQ(
- "Michael and NR are assured to be in the database. Therefore the transformer should have added leahciM and RN on the same document as id:TripleThreat-1-3",
- req("+id:TripleThreat-1-3 +NAME_mult_s:Michael +NAME_mult_s:leahciM +COUNTRY_CODES_mult_s:NR +COUNTRY_CODES_mult_s:RN"),
- "//*[@numFound='1']");
- }
- assertQ(req("AddAColumn_s:Added"), "//*[@numFound='" + totalAddedDocs
- + "']");
- Assert.assertTrue("Expecting " + numToExpect
- + " database calls, but DIH reported " + totalDatabaseRequests(),
- totalDatabaseRequests() == numToExpect);
- }
-
- protected void withChildEntities(boolean cached, boolean checkDatabaseRequests)
- throws Exception {
- rootTransformerName = random().nextBoolean() ? null
- : "AddAColumnTransformer";
- int numChildren = random().nextInt(1) + 1;
- int numDatabaseRequests = 1;
- if (underlyingDataModified) {
- if (countryEntity) {
- if (cached) {
- numDatabaseRequests++;
- } else {
- numDatabaseRequests += totalPeople();
- }
- }
- if (sportsEntity) {
- if (cached) {
- numDatabaseRequests++;
- } else {
- numDatabaseRequests += totalPeople();
- }
- }
- } else {
- countryEntity = true;
- sportsEntity = true;
- if(countryZipper||sportsZipper){// zipper tests fully cover nums of children
- countryEntity = countryZipper;
- sportsEntity = sportsZipper;
- }else{// apply default randomization on cached cases
- if (numChildren == 1) {
- countryEntity = random().nextBoolean();
- sportsEntity = !countryEntity;
- }
- }
- if (countryEntity) {
- countryTransformer = random().nextBoolean();
- if (cached) {
- numDatabaseRequests++;
- countryCached = true;
- } else {
- numDatabaseRequests += totalPeople();
- }
- }
- if (sportsEntity) {
- sportsTransformer = random().nextBoolean();
- if (cached) {
- numDatabaseRequests++;
- sportsCached = true;
- } else {
- numDatabaseRequests += totalPeople();
- }
- }
- }
- h.query("/dataimport", generateRequest());
-
- assertQ("There should be 1 document per person in the database: "
- + totalPeople(), req("*:*"), "//*[@numFound='" + (totalPeople()) + "']");
- if (!underlyingDataModified
- && "AddAColumnTransformer".equals(rootTransformerName)) {
- assertQ(
- "There should be 1 document with a transformer-added column per person is the database: "
- + totalPeople(), req("AddAColumn_s:Added"), "//*[@numFound='"
- + (totalPeople()) + "']");
- }
- if (countryEntity) {
- {
- String[] people = getStringsFromQuery("SELECT NAME FROM PEOPLE WHERE DELETED != 'Y'");
- String man = people[random().nextInt(people.length)];
- String[] countryNames = getStringsFromQuery("SELECT C.COUNTRY_NAME FROM PEOPLE P "
- + "INNER JOIN COUNTRIES C ON P.COUNTRY_CODE=C.CODE "
- + "WHERE P.DELETED!='Y' AND C.DELETED!='Y' AND P.NAME='" + man + "'");
-
- assertQ(req("{!term f=NAME_mult_s}"+ man), "//*[@numFound='1']",
- countryNames.length>0?
- "//doc/str[@name='COUNTRY_NAME_s']='" + countryNames[random().nextInt(countryNames.length)] + "'"
- :"//doc[count(*[@name='COUNTRY_NAME_s'])=0]");
- }
- {
- String[] countryCodes = getStringsFromQuery("SELECT CODE FROM COUNTRIES WHERE DELETED != 'Y'");
- String theCode = countryCodes[random().nextInt(countryCodes.length)];
- int num = numberPeopleByCountryCode(theCode);
- if(num>0){
- String nrName = countryNameByCode(theCode);
- assertQ(req("COUNTRY_CODES_mult_s:"+theCode), "//*[@numFound='" + num + "']",
- "//doc/str[@name='COUNTRY_NAME_s']='" + nrName + "'");
- }else{ // no one lives there anyway
- assertQ(req("COUNTRY_CODES_mult_s:"+theCode), "//*[@numFound='" + num + "']");
- }
- }
- if (countryTransformer && !underlyingDataModified) {
- assertQ(req("countryAdded_s:country_added"), "//*[@numFound='"
- + totalPeople() + "']");
- }
- }
- if (sportsEntity) {
- if (!underlyingDataModified) {
- assertQ(req("SPORT_NAME_mult_s:Sailing"), "//*[@numFound='2']");
- }
- String [] names = getStringsFromQuery("SELECT NAME FROM PEOPLE WHERE DELETED != 'Y'");
- String name = names[random().nextInt(names.length)];
- int personId = getIntFromQuery("SELECT ID FROM PEOPLE WHERE DELETED != 'Y' AND NAME='"+name+"'");
- String[] michaelsSports = sportNamesByPersonId(personId);
-
- String[] xpath = new String[michaelsSports.length + 1];
- xpath[0] = "//*[@numFound='1']";
- int i = 1;
- for (String ms : michaelsSports) {
- xpath[i] = "//doc/arr[@name='SPORT_NAME_mult_s']/str='"//[" + i + "]='" don't care about particular order
- + ms + "'";
- i++;
- }
- assertQ(req("NAME_mult_s:" + name.replaceAll("\\W", "\\\\$0")),
- xpath);
- if (!underlyingDataModified && sportsTransformer) {
- assertQ(req("sportsAdded_s:sport_added"), "//*[@numFound='"
- + (totalSportsmen()) + "']");
- }
- assertQ("checking orphan sport is absent",
- req("{!term f=SPORT_NAME_mult_s}No Fishing"), "//*[@numFound='0']");
- }
- if (checkDatabaseRequests) {
- Assert.assertTrue("Expecting " + numDatabaseRequests
- + " database calls, but DIH reported " + totalDatabaseRequests(),
- totalDatabaseRequests() == numDatabaseRequests);
- }
- }
-
- protected void simpleCacheChildEntities(boolean checkDatabaseRequests)
- throws Exception {
- useSimpleCaches = true;
- countryEntity = true;
- sportsEntity = true;
- countryCached = true;
- sportsCached = true;
- int dbRequestsMoreThan = 3;
- int dbRequestsLessThan = totalPeople() * 2 + 1;
- h.query("/dataimport", generateRequest());
- assertQ(req("*:*"), "//*[@numFound='" + (totalPeople()) + "']");
- if (!underlyingDataModified
- || (personNameExists("Samantha") && "Nauru"
- .equals(countryNameByCode("NR")))) {
- assertQ(req("NAME_mult_s:Samantha"), "//*[@numFound='1']",
- "//doc/str[@name='COUNTRY_NAME_s']='Nauru'");
- }
- if (!underlyingDataModified) {
- assertQ(req("COUNTRY_CODES_mult_s:NR"), "//*[@numFound='2']",
- "//doc/str[@name='COUNTRY_NAME_s']='Nauru'");
- assertQ(req("SPORT_NAME_mult_s:Sailing"), "//*[@numFound='2']");
- }
- String[] michaelsSports = sportNamesByPersonId(3);
- if (!underlyingDataModified || michaelsSports.length > 0) {
- String[] xpath = new String[michaelsSports.length + 1];
- xpath[0] = "//*[@numFound='1']";
- int i = 1;
- for (String ms : michaelsSports) {
- xpath[i] = "//doc/arr[@name='SPORT_NAME_mult_s']/str[" + i + "]='" + ms
- + "'";
- i++;
- }
- assertQ(req("NAME_mult_s:Michael"), xpath);
- }
- if (checkDatabaseRequests) {
- Assert.assertTrue("Expecting more than " + dbRequestsMoreThan
- + " database calls, but DIH reported " + totalDatabaseRequests(),
- totalDatabaseRequests() > dbRequestsMoreThan);
- Assert.assertTrue("Expecting fewer than " + dbRequestsLessThan
- + " database calls, but DIH reported " + totalDatabaseRequests(),
- totalDatabaseRequests() < dbRequestsLessThan);
- }
- }
-
-
- private int getIntFromQuery(String query) throws Exception {
- Connection conn = null;
- Statement s = null;
- ResultSet rs = null;
- try {
- conn = newConnection();
- s = conn.createStatement();
- rs = s.executeQuery(query);
- if (rs.next()) {
- return rs.getInt(1);
- }
- return 0;
- } catch (SQLException e) {
- throw e;
- } finally {
- try {
- rs.close();
- } catch (Exception ex) {}
- try {
- s.close();
- } catch (Exception ex) {}
- try {
- conn.close();
- } catch (Exception ex) {}
- }
- }
-
- private String[] getStringsFromQuery(String query) throws Exception {
- Connection conn = null;
- Statement s = null;
- ResultSet rs = null;
- try {
- conn = newConnection();
- s = conn.createStatement();
- rs = s.executeQuery(query);
- List<String> results = new ArrayList<>();
- while (rs.next()) {
- results.add(rs.getString(1));
- }
- return results.toArray(new String[results.size()]);
- } catch (SQLException e) {
- throw e;
- } finally {
- try {
- rs.close();
- } catch (Exception ex) {}
- try {
- s.close();
- } catch (Exception ex) {}
- try {
- conn.close();
- } catch (Exception ex) {}
- }
- }
-
- public int totalCountries() throws Exception {
- return getIntFromQuery("SELECT COUNT(1) FROM COUNTRIES WHERE DELETED != 'Y' ");
- }
-
- public int totalPeople() throws Exception {
- return getIntFromQuery("SELECT COUNT(1) FROM PEOPLE WHERE DELETED != 'Y' ");
- }
-
- public int totalSportsmen() throws Exception {
- return getIntFromQuery("SELECT COUNT(*) FROM PEOPLE WHERE "
- + "EXISTS(SELECT ID FROM PEOPLE_SPORTS WHERE PERSON_ID=PEOPLE.ID AND PEOPLE_SPORTS.DELETED != 'Y')"
- + " AND PEOPLE.DELETED != 'Y'");
- }
-
- public boolean countryCodeExists(String cc) throws Exception {
- return getIntFromQuery("SELECT COUNT(1) country_name FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='"
- + cc + "'") > 0;
- }
-
- public String countryNameByCode(String cc) throws Exception {
- String[] s = getStringsFromQuery("SELECT country_name FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='"
- + cc + "'");
- return s.length == 0 ? null : s[0];
- }
-
- public int numberPeopleByCountryCode(String cc) throws Exception {
- return getIntFromQuery("Select count(1) " + "from people p "
- + "inner join countries c on p.country_code=c.code "
- + "where p.deleted!='Y' and c.deleted!='Y' and c.code='" + cc + "'");
- }
-
- public String[] sportNamesByPersonId(int personId) throws Exception {
- return getStringsFromQuery("SELECT ps.SPORT_NAME "
- + "FROM people_sports ps "
- + "INNER JOIN PEOPLE p ON p.id = ps.person_Id "
- + "WHERE ps.DELETED != 'Y' AND p.DELETED != 'Y' " + "AND ps.person_id="
- + personId + " " + "ORDER BY ps.id");
- }
-
- public boolean personNameExists(String pn) throws Exception {
- return getIntFromQuery("SELECT COUNT(1) FROM PEOPLE WHERE DELETED != 'Y' AND NAME='"
- + pn + "'") > 0;
- }
-
- public String personNameById(int id) throws Exception {
- String[] nameArr = getStringsFromQuery("SELECT NAME FROM PEOPLE WHERE ID="
- + id);
- if (nameArr.length == 0) {
- return null;
- }
- return nameArr[0];
- }
-
- @SuppressForbidden(reason = "Needs currentTimeMillis to set change time for SQL query")
- public IntChanges modifySomePeople() throws Exception {
- underlyingDataModified = true;
- int numberToChange = random().nextInt(people.length + 1);
- Set<Integer> changeSet = new HashSet<>();
- Set<Integer> deleteSet = new HashSet<>();
- Set<Integer> addSet = new HashSet<>();
- Connection conn = null;
- PreparedStatement change = null;
- PreparedStatement delete = null;
- PreparedStatement add = null;
- // One second in the future ensures a change time after the last import (DIH
- // uses second precision only)
- Timestamp theTime = new Timestamp(System.currentTimeMillis() + 1000);
- log.debug("PEOPLE UPDATE USING TIMESTAMP: "
- + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.ROOT)
- .format(theTime));
- try {
- conn = newConnection();
- change = conn
- .prepareStatement("update people set name=?, last_modified=? where id=?");
- delete = conn
- .prepareStatement("update people set deleted='Y', last_modified=? where id=?");
- add = conn
- .prepareStatement("insert into people (id,name,country_code,last_modified) values (?,?,'ZZ',?)");
- for (int i = 0; i < numberToChange; i++) {
- int tryIndex = random().nextInt(people.length);
- Integer id = (Integer) people[tryIndex][0];
- if (!changeSet.contains(id) && !deleteSet.contains(id)) {
- boolean changeDontDelete = random().nextBoolean();
- if (changeDontDelete) {
- changeSet.add(id);
- change.setString(1, "MODIFIED " + people[tryIndex][1]);
- change.setTimestamp(2, theTime);
- change.setInt(3, id);
- Assert.assertEquals(1, change.executeUpdate());
- } else {
- deleteSet.add(id);
- delete.setTimestamp(1, theTime);
- delete.setInt(2, id);
- Assert.assertEquals(1, delete.executeUpdate());
- }
- }
- }
- int numberToAdd = random().nextInt(3);
- for (int i = 0; i < numberToAdd; i++) {
- int tryIndex = random().nextInt(people.length);
- Integer id = (Integer) people[tryIndex][0];
- Integer newId = id + 1000;
- String newDesc = "ADDED " + people[tryIndex][1];
- if (!addSet.contains(newId)) {
- addSet.add(newId);
- add.setInt(1, newId);
- add.setString(2, newDesc);
- add.setTimestamp(3, theTime);
- Assert.assertEquals(1, add.executeUpdate());
- }
- }
- conn.commit();
- } catch (SQLException e) {
- throw e;
- } finally {
- try {
- change.close();
- } catch (Exception ex) {}
- try {
- conn.close();
- } catch (Exception ex) {}
- }
- IntChanges c = new IntChanges();
- c.changedKeys = changeSet.toArray(new Integer[changeSet.size()]);
- c.deletedKeys = deleteSet.toArray(new Integer[deleteSet.size()]);
- c.addedKeys = addSet.toArray(new Integer[addSet.size()]);
- return c;
- }
-
- @SuppressForbidden(reason = "Needs currentTimeMillis to set change time for SQL query")
- public String[] modifySomeCountries() throws Exception {
- underlyingDataModified = true;
- int numberToChange = random().nextInt(countries.length + 1);
- Set<String> changeSet = new HashSet<>();
- Connection conn = null;
- PreparedStatement change = null;
- // One second in the future ensures a change time after the last import (DIH
- // uses second precision only)
- Timestamp theTime = new Timestamp(System.currentTimeMillis() + 1000);
- log.debug("COUNTRY UPDATE USING TIMESTAMP: "
- + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.ROOT)
- .format(theTime));
- try {
- conn = newConnection();
- change = conn
- .prepareStatement("update countries set country_name=?, last_modified=? where code=?");
- for (int i = 0; i < numberToChange; i++) {
- int tryIndex = random().nextInt(countries.length);
- String code = countries[tryIndex][0];
- if (!changeSet.contains(code)) {
- changeSet.add(code);
- change.setString(1, "MODIFIED " + countries[tryIndex][1]);
- change.setTimestamp(2, theTime);
- change.setString(3, code);
- Assert.assertEquals(1, change.executeUpdate());
-
- }
- }
- } catch (SQLException e) {
- throw e;
- } finally {
- try {
- change.close();
- } catch (Exception ex) {}
- try {
- conn.close();
- } catch (Exception ex) {}
- }
- return changeSet.toArray(new String[changeSet.size()]);
- }
-
- static class IntChanges {
- public Integer[] changedKeys;
- public Integer[] deletedKeys;
- public Integer[] addedKeys;
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- if(changedKeys!=null) {
- sb.append("changes: ");
- for(int i : changedKeys) {
- sb.append(i).append(" ");
- }
- }
- if(deletedKeys!=null) {
- sb.append("deletes: ");
- for(int i : deletedKeys) {
- sb.append(i).append(" ");
- }
- }
- if(addedKeys!=null) {
- sb.append("adds: ");
- for(int i : addedKeys) {
- sb.append(i).append(" ");
- }
- }
- return sb.toString();
- }
- }
-
- @Override
- protected String generateConfig() {
- String ds = null;
- if (dbToUse == Database.DERBY) {
- ds = "derby";
- } else if (dbToUse == Database.HSQLDB) {
- ds = "hsqldb";
- } else {
- throw new AssertionError("Invalid database to use: " + dbToUse);
- }
- StringBuilder sb = new StringBuilder();
- sb.append("\n<dataConfig> \n");
- sb.append("<propertyWriter type=''SimplePropertiesWriter'' directory=''" + fileLocation + "'' filename=''" + fileName + "'' />\n");
- sb.append("<dataSource name=''hsqldb'' driver=''org.hsqldb.jdbcDriver'' url=''jdbc:hsqldb:mem:.'' /> \n");
- sb.append("<dataSource name=''derby'' driver=''org.apache.derby.jdbc.EmbeddedDriver'' url=''jdbc:derby:memory:derbyDB;territory=en_US'' /> \n");
- sb.append("<document name=''TestSqlEntityProcessor''> \n");
- sb.append("<entity name=''People'' ");
- sb.append("pk=''" + (random().nextBoolean() ? "ID" : "People.ID") + "'' ");
- sb.append("processor=''SqlEntityProcessor'' ");
- sb.append("dataSource=''" + ds + "'' ");
- sb.append(rootTransformerName != null ? "transformer=''"
- + rootTransformerName + "'' " : "");
-
- sb.append("query=''SELECT ID, NAME, COUNTRY_CODE FROM PEOPLE WHERE DELETED != 'Y' "
- +((sportsZipper||countryZipper?"ORDER BY ID":"")
- +(wrongPeopleOrder? " DESC":""))+"'' ");
-
- sb.append(deltaQueriesPersonTable());
- sb.append("> \n");
-
- sb.append("<field column=''NAME'' name=''NAME_mult_s'' /> \n");
- sb.append("<field column=''COUNTRY_CODE'' name=''COUNTRY_CODES_mult_s'' /> \n");
-
- if (countryEntity) {
- sb.append("<entity name=''Countries'' ");
- sb.append("pk=''" + (random().nextBoolean() ? "CODE" : "Countries.CODE")
- + "'' ");
- sb.append("dataSource=''" + ds + "'' ");
- sb.append(countryTransformer ? "transformer=''AddAColumnTransformer'' "
- + "newColumnName=''countryAdded_s'' newColumnValue=''country_added'' "
- : "");
- if (countryCached) {
- sb.append("processor=''SqlEntityProcessor'' cacheImpl=''SortedMapBackedCache'' ");
- if (useSimpleCaches) {
- sb.append("query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='${People.COUNTRY_CODE}' ''>\n");
- } else {
-
- if(countryZipper){// really odd join btw. it sends duped countries
- sb.append(random().nextBoolean() ? "cacheKey=''ID'' cacheLookup=''People.ID'' "
- : "where=''ID=People.ID'' ");
- sb.append("join=''zipper'' query=''SELECT PEOPLE.ID, CODE, COUNTRY_NAME FROM COUNTRIES"
- + " JOIN PEOPLE ON COUNTRIES.CODE=PEOPLE.COUNTRY_CODE "
- + "WHERE PEOPLE.DELETED != 'Y' ORDER BY PEOPLE.ID "+
- (wrongCountryOrder ? " DESC":"")
- + "'' ");
- }else{
- sb.append(random().nextBoolean() ? "cacheKey=''CODE'' cacheLookup=''People.COUNTRY_CODE'' "
- : "where=''CODE=People.COUNTRY_CODE'' ");
- sb.append("query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES'' ");
- }
- sb.append("> \n");
- }
- } else {
- sb.append("processor=''SqlEntityProcessor'' query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='${People.COUNTRY_CODE}' '' ");
- sb.append(deltaQueriesCountryTable());
- sb.append("> \n");
- }
- sb.append("<field column=''CODE'' name=''COUNTRY_CODE_s'' /> \n");
- sb.append("<field column=''COUNTRY_NAME'' name=''COUNTRY_NAME_s'' /> \n");
- sb.append("</entity> \n");
- }
- if (sportsEntity) {
- sb.append("<entity name=''Sports'' ");
- sb.append("dataSource=''" + ds + "'' ");
- sb.append(sportsTransformer ? "transformer=''AddAColumnTransformer'' "
- + "newColumnName=''sportsAdded_s'' newColumnValue=''sport_added'' "
- : "");
- if (sportsCached) {
- sb.append("processor=''SqlEntityProcessor'' cacheImpl=''SortedMapBackedCache'' ");
- if (useSimpleCaches) {
- sb.append("query=''SELECT ID, SPORT_NAME FROM PEOPLE_SPORTS WHERE DELETED != 'Y' AND PERSON_ID=${People.ID} ORDER BY ID'' ");
- } else {
- sb.append(random().nextBoolean() ? "cacheKey=''PERSON_ID'' cacheLookup=''People.ID'' "
- : "where=''PERSON_ID=People.ID'' ");
- if(sportsZipper){
- sb.append("join=''zipper'' query=''SELECT ID, PERSON_ID, SPORT_NAME FROM PEOPLE_SPORTS ORDER BY PERSON_ID"
- + (wrongSportsOrder?" DESC" : "")+
- "'' ");
- }
- else{
- sb.append("query=''SELECT ID, PERSON_ID, SPORT_NAME FROM PEOPLE_SPORTS ORDER BY ID'' ");
- }
- }
- } else {
- sb.append("processor=''SqlEntityProcessor'' query=''SELECT ID, SPORT_NAME FROM PEOPLE_SPORTS WHERE DELETED != 'Y' AND PERSON_ID=${People.ID} ORDER BY ID'' ");
- }
- sb.append("> \n");
- sb.append("<field column=''SPORT_NAME'' name=''SPORT_NAME_mult_s'' /> \n");
- sb.append("<field column=''id'' name=''SPORT_ID_mult_s'' /> \n");
- sb.append("</entity> \n");
- }
-
- sb.append("</entity> \n");
- sb.append("</document> \n");
- sb.append("</dataConfig> \n");
- String config = sb.toString().replaceAll("[']{2}", "\"");
- log.debug(config);
- return config;
- }
-
- @SuppressForbidden(reason = "Needs currentTimeMillis to set change time for SQL query")
- @Override
- protected void populateData(Connection conn) throws Exception {
- Statement s = null;
- PreparedStatement ps = null;
- Timestamp theTime = new Timestamp(System.currentTimeMillis() - 10000); // 10 seconds ago
- try {
- s = conn.createStatement();
- s.executeUpdate("create table countries(code varchar(3) not null primary key, country_name varchar(50), deleted char(1) default 'N', last_modified timestamp not null)");
- s.executeUpdate("create table people(id int not null primary key, name varchar(50), country_code char(2), deleted char(1) default 'N', last_modified timestamp not null)");
- s.executeUpdate("create table people_sports(id int not null primary key, person_id int, sport_name varchar(50), deleted char(1) default 'N', last_modified timestamp not null)");
- log.debug("INSERTING DB DATA USING TIMESTAMP: "
- + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.ROOT)
- .format(theTime));
- ps = conn
- .prepareStatement("insert into countries (code, country_name, last_modified) values (?,?,?)");
- for (String[] country : countries) {
- ps.setString(1, country[0]);
- ps.setString(2, country[1]);
- ps.setTimestamp(3, theTime);
- Assert.assertEquals(1, ps.executeUpdate());
- }
- ps.close();
-
- ps = conn
- .prepareStatement("insert into people (id, name, country_code, last_modified) values (?,?,?,?)");
- for (Object[] person : people) {
- ps.setInt(1, (Integer) person[0]);
- ps.setString(2, (String) person[1]);
- ps.setString(3, (String) person[2]);
- ps.setTimestamp(4, theTime);
- Assert.assertEquals(1, ps.executeUpdate());
- }
- ps.close();
-
- ps = conn
- .prepareStatement("insert into people_sports (id, person_id, sport_name, last_modified) values (?,?,?,?)");
- for (Object[] sport : people_sports) {
- ps.setInt(1, (Integer) sport[0]);
- ps.setInt(2, (Integer) sport[1]);
- ps.setString(3, (String) sport[2]);
- ps.setTimestamp(4, theTime);
- Assert.assertEquals(1, ps.executeUpdate());
- }
- ps.close();
- conn.commit();
- conn.close();
- } catch (Exception e) {
- throw e;
- } finally {
- try {
- ps.close();
- } catch (Exception ex) {}
- try {
- s.close();
- } catch (Exception ex) {}
- try {
- conn.close();
- } catch (Exception ex) {}
- }
- }
- public static final String[][] countries = {
- {"NA", "Namibia"},
- {"NC", "New Caledonia"},
- {"NE", "Niger"},
- {"NF", "Norfolk Island"},
- {"NG", "Nigeria"},
- {"NI", "Nicaragua"},
- {"NL", "Netherlands"},
- {"NO", "Norway"},
- {"NP", "Nepal"},
- {"NR", "Nauru"},
- {"NU", "Niue"},
- {"NZ", "New Zealand"}
- };
-
- public static final Object[][] people = {
- {1,"Jacob","NZ"},
- {2,"Ethan","NU"},
- {3,"Michael","NR"},
- {4,"Jayden","NP"},
- {5,"William","NO"},
- {6,"Alexander","NL"},
- {7,"Noah","NI"},
- {8,"Daniel","NG"},
- {9,"Aiden","NF"},
-
- {21,"Anthony","NE"}, // there is no ID=10 anymore
-
- {11,"Emma","NL"},
- {12,"Grace","NI"},
- {13,"Hailey","NG"},
- {14,"Isabella","NF"},
- {15,"Lily","NE"},
- {16,"Madison","NC"},
- {17,"Mia","NA"},
- {18,"Natalie","NZ"},
- {19,"Olivia","NU"},
- {20,"Samantha","NR"}
- };
-
- public static final Object[][] people_sports = {
- {100, 1, "Swimming"},
- {200, 2, "Triathlon"},
- {300, 3, "Water polo"},
- {310, 3, "Underwater rugby"},
- {320, 3, "Kayaking"},
- {400, 4, "Snorkeling"},
- {500, 5, "Synchronized diving"},
- {600, 6, "Underwater rugby"},
- {700, 7, "Boating"},
- {800, 8, "Bodyboarding"},
- {900, 9, "Canoeing"},
-
- {1000, 10, "No Fishing"}, // orhpaned sport
- //
-
- {1100, 11, "Jet Ski"},
- {1110, 11, "Rowing"},
- {1120, 11, "Sailing"},
- {1200, 12, "Kayaking"},
- {1210, 12, "Canoeing"},
- {1300, 13, "Kite surfing"},
- {1400, 14, "Parasailing"},
- {1500, 15, "Rafting"},
- //{1600, 16, "Rowing"}, Madison has no sport
- {1700, 17, "Sailing"},
- {1800, 18, "White Water Rafting"},
- {1900, 19, "Water skiing"},
- {2000, 20, "Windsurfing"},
- {2100, 21, "Concrete diving"},
- {2110, 21, "Bubble rugby"}
- };
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AddAColumnTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AddAColumnTransformer.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AddAColumnTransformer.java
deleted file mode 100644
index 5e665d4..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AddAColumnTransformer.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.Map;
-
-public class AddAColumnTransformer extends Transformer {
- @Override
- public Object transformRow(Map<String,Object> aRow, Context context) {
- String colName = context.getEntityAttribute("newColumnName");
- colName = colName==null ? "AddAColumn_s" : colName;
- String colValue = context.getEntityAttribute("newColumnValue");
- colValue = colValue==null ? "Added" : colValue;
- aRow.put(colName, colValue);
- return aRow;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/DestroyCountCache.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/DestroyCountCache.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/DestroyCountCache.java
deleted file mode 100644
index d14f43e..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/DestroyCountCache.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.hamcrest.CoreMatchers.nullValue;
-
-import java.util.IdentityHashMap;
-import java.util.Map;
-
-import org.junit.Assert;
-
-public class DestroyCountCache extends SortedMapBackedCache {
- static Map<DIHCache,DIHCache> destroyed = new IdentityHashMap<>();
-
- @Override
- public void destroy() {
- super.destroy();
- Assert.assertThat(destroyed.put(this, this), nullValue());
- }
-
- public DestroyCountCache() {}
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockInitialContextFactory.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockInitialContextFactory.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockInitialContextFactory.java
deleted file mode 100644
index 7568210..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockInitialContextFactory.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.HashMap;
-import java.util.Hashtable;
-import java.util.Map;
-
-import javax.naming.NamingException;
-import javax.naming.spi.InitialContextFactory;
-
-import static org.mockito.Mockito.*;
-
-public class MockInitialContextFactory implements InitialContextFactory {
- private static final Map<String, Object> objects = new HashMap<>();
- private final javax.naming.Context context;
-
- public MockInitialContextFactory() {
- context = mock(javax.naming.Context.class);
-
- try {
- when(context.lookup(anyString())).thenAnswer(invocation -> objects.get(invocation.getArgument(0)));
-
- } catch (NamingException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public javax.naming.Context getInitialContext(Hashtable env) {
- return context;
- }
-
- public static void bind(String name, Object obj) {
- objects.put(name, obj);
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java
deleted file mode 100644
index 42e5f7d..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
-
-import java.util.List;
-
-public class MockSolrEntityProcessor extends SolrEntityProcessor {
-
- private final List<SolrTestCaseJ4.Doc> docsData;
-// private final int rows;
- private int queryCount = 0;
-
- private int rows;
-
- private int start = 0;
-
- public MockSolrEntityProcessor(List<SolrTestCaseJ4.Doc> docsData, int rows) {
- this.docsData = docsData;
- this.rows = rows;
- }
-
- //@Override
- //protected SolrDocumentList doQuery(int start) {
- // queryCount++;
- // return getDocs(start, rows);
- // }
-
- @Override
- protected void buildIterator() {
- if (rowIterator==null || (!rowIterator.hasNext() && ((SolrDocumentListIterator)rowIterator).hasMoreRows())){
- queryCount++;
- SolrDocumentList docs = getDocs(start, rows);
- rowIterator = new SolrDocumentListIterator(docs);
- start += docs.size();
- }
- }
-
- private SolrDocumentList getDocs(int start, int rows) {
- SolrDocumentList docs = new SolrDocumentList();
- docs.setNumFound(docsData.size());
- docs.setStart(start);
-
- int endIndex = start + rows;
- int end = docsData.size() < endIndex ? docsData.size() : endIndex;
- for (int i = start; i < end; i++) {
- SolrDocument doc = new SolrDocument();
- SolrTestCaseJ4.Doc testDoc = docsData.get(i);
- doc.addField("id", testDoc.id);
- doc.addField("description", testDoc.getValues("description"));
- docs.add(doc);
- }
- return docs;
- }
-
- public int getQueryCount() {
- return queryCount;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockStringDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockStringDataSource.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockStringDataSource.java
deleted file mode 100644
index 7c9a6d1..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockStringDataSource.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.handler.dataimport;
-
-
-import java.io.Reader;
-import java.io.StringReader;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-public class MockStringDataSource extends DataSource<Reader> {
-
- private static Map<String, String> cache = new HashMap<>();
-
- public static void setData(String query,
- String data) {
- cache.put(query, data);
- }
-
- public static void clearCache() {
- cache.clear();
- }
- @Override
- public void init(Context context, Properties initProps) {
-
- }
-
- @Override
- public Reader getData(String query) {
- return new StringReader(cache.get(query));
- }
-
- @Override
- public void close() {
- cache.clear();
-
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
deleted file mode 100644
index 986a8cd..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
-import java.text.SimpleDateFormat;
-import java.util.*;
-
-/**
- * <p> Test for Evaluators </p>
- *
- *
- * @since solr 1.3
- */
-public class TestBuiltInEvaluators extends AbstractDataImportHandlerTestCase {
- private static final String ENCODING = StandardCharsets.UTF_8.name();
-
- VariableResolver resolver;
-
- Map<String, String> sqlTests;
-
- Map<String, String> urlTests;
-
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- resolver = new VariableResolver();
-
- sqlTests = new HashMap<>();
-
- sqlTests.put("foo\"", "foo\"\"");
- sqlTests.put("foo\\", "foo\\\\");
- sqlTests.put("foo'", "foo''");
- sqlTests.put("foo''", "foo''''");
- sqlTests.put("'foo\"", "''foo\"\"");
- sqlTests.put("\"Albert D'souza\"", "\"\"Albert D''souza\"\"");
-
- urlTests = new HashMap<>();
-
- urlTests.put("*:*", URLEncoder.encode("*:*", ENCODING));
- urlTests.put("price:[* TO 200]", URLEncoder.encode("price:[* TO 200]",
- ENCODING));
- urlTests.put("review:\"hybrid sedan\"", URLEncoder.encode(
- "review:\"hybrid sedan\"", ENCODING));
- }
-
-
- @Test
- public void testSqlEscapingEvaluator() {
- Evaluator sqlEscaper = new SqlEscapingEvaluator();
- runTests(sqlTests, sqlEscaper);
- }
-
-
- @Test
- public void testUrlEvaluator() throws Exception {
- Evaluator urlEvaluator = new UrlEvaluator();
- runTests(urlTests, urlEvaluator);
- }
-
- @Test
- public void parseParams() {
- Map<String,Object> m = new HashMap<>();
- m.put("b","B");
- VariableResolver vr = new VariableResolver();
- vr.addNamespace("a",m);
- List<Object> l = (new Evaluator() {
- @Override
- public String evaluate(String expression, Context context) {
- return null;
- }
- }).parseParams(" 1 , a.b, 'hello!', 'ds,o,u\'za',",vr);
- assertEquals(1d,l.get(0));
- assertEquals("B",((Evaluator.VariableWrapper)l.get(1)).resolve());
- assertEquals("hello!",l.get(2));
- assertEquals("ds,o,u'za",l.get(3));
- }
-
- @Test
- public void testEscapeSolrQueryFunction() {
- final VariableResolver resolver = new VariableResolver();
- Map<String,Object> m= new HashMap<>();
- m.put("query","c:t");
- resolver.setEvaluators(new DataImporter().getEvaluators(Collections.<Map<String,String>>emptyList()));
-
- resolver.addNamespace("e",m);
- String s = resolver
- .replaceTokens("${dataimporter.functions.escapeQueryChars(e.query)}");
- org.junit.Assert.assertEquals("c\\:t", s);
-
- }
-
- private Date twoDaysAgo(Locale l, TimeZone tz) {
- Calendar calendar = Calendar.getInstance(tz, l);
- calendar.add(Calendar.DAY_OF_YEAR, -2);
- return calendar.getTime();
- }
-
- @Test
- public void testDateFormatEvaluator() {
- Evaluator dateFormatEval = new DateFormatEvaluator();
- ContextImpl context = new ContextImpl(null, resolver, null,
- Context.FULL_DUMP, Collections.<String,Object> emptyMap(), null, null);
-
- Locale rootLocale = Locale.ROOT;
- Locale defaultLocale = Locale.getDefault();
- TimeZone defaultTz = TimeZone.getDefault();
-
- {
- SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd HH", rootLocale);
- String sdf = sdfDate.format(twoDaysAgo(rootLocale, defaultTz));
- String dfe = dateFormatEval.evaluate("'NOW-2DAYS','yyyy-MM-dd HH'", context);
- assertEquals(sdf,dfe);
- }
- {
- SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd HH", defaultLocale);
- String sdf = sdfDate.format(twoDaysAgo(defaultLocale, TimeZone.getDefault()));
- String dfe = dateFormatEval.evaluate(
- "'NOW-2DAYS','yyyy-MM-dd HH','" + defaultLocale.toLanguageTag() + "'", context);
- assertEquals(sdf,dfe);
- for(String tzStr : TimeZone.getAvailableIDs()) {
- TimeZone tz = TimeZone.getTimeZone(tzStr);
- sdfDate.setTimeZone(tz);
- sdf = sdfDate.format(twoDaysAgo(defaultLocale, tz));
- dfe = dateFormatEval.evaluate(
- "'NOW-2DAYS','yyyy-MM-dd HH','" + defaultLocale.toLanguageTag() + "','" + tzStr + "'", context);
- assertEquals(sdf,dfe);
- }
- }
-
- Date d = new Date();
- Map<String,Object> map = new HashMap<>();
- map.put("key", d);
- resolver.addNamespace("A", map);
-
- assertEquals(
- new SimpleDateFormat("yyyy-MM-dd HH:mm", rootLocale).format(d),
- dateFormatEval.evaluate("A.key, 'yyyy-MM-dd HH:mm'", context));
- assertEquals(
- new SimpleDateFormat("yyyy-MM-dd HH:mm", defaultLocale).format(d),
- dateFormatEval.evaluate("A.key, 'yyyy-MM-dd HH:mm','" + defaultLocale.toLanguageTag() + "'", context));
- SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm", defaultLocale);
- for(String tzStr : TimeZone.getAvailableIDs()) {
- TimeZone tz = TimeZone.getTimeZone(tzStr);
- sdf.setTimeZone(tz);
- assertEquals(
- sdf.format(d),
- dateFormatEval.evaluate(
- "A.key, 'yyyy-MM-dd HH:mm','" + defaultLocale.toLanguageTag() + "', '" + tzStr + "'", context));
-
- }
-
-
- }
-
- private void runTests(Map<String, String> tests, Evaluator evaluator) {
- ContextImpl ctx = new ContextImpl(null, resolver, null, Context.FULL_DUMP, Collections.<String, Object>emptyMap(), null, null);
- for (Map.Entry<String, String> entry : tests.entrySet()) {
- Map<String, Object> values = new HashMap<>();
- values.put("key", entry.getKey());
- resolver.addNamespace("A", values);
-
- String expected = entry.getValue();
- String actual = evaluator.evaluate("A.key", ctx);
- assertEquals(expected, actual);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestClobTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestClobTransformer.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestClobTransformer.java
deleted file mode 100644
index fe00d49..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestClobTransformer.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.junit.Test;
-
-import java.io.StringReader;
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.Method;
-import java.lang.reflect.Proxy;
-import java.sql.Clob;
-import java.util.*;
-
-/**
- * Test for ClobTransformer
- *
- *
- * @see org.apache.solr.handler.dataimport.ClobTransformer
- * @since solr 1.4
- */
-public class TestClobTransformer extends AbstractDataImportHandlerTestCase {
- @Test
- public void simple() throws Exception {
- List<Map<String, String>> flds = new ArrayList<>();
- Map<String, String> f = new HashMap<>();
- // <field column="dsc" clob="true" name="description" />
- f.put(DataImporter.COLUMN, "dsc");
- f.put(ClobTransformer.CLOB, "true");
- f.put(DataImporter.NAME, "description");
- flds.add(f);
- Context ctx = getContext(null, new VariableResolver(), null, Context.FULL_DUMP, flds, Collections.EMPTY_MAP);
- Transformer t = new ClobTransformer();
- Map<String, Object> row = new HashMap<>();
- Clob clob = (Clob) Proxy.newProxyInstance(this.getClass().getClassLoader(), new Class[]{Clob.class}, new InvocationHandler() {
- @Override
- public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
- if (method.getName().equals("getCharacterStream")) {
- return new StringReader("hello!");
- }
- return null;
- }
- });
-
- row.put("dsc", clob);
- t.transformRow(row, ctx);
- assertEquals("hello!", row.get("dsc"));
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
deleted file mode 100644
index 06fd51c..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.request.DirectXmlRequest;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.UpdateParams;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.nio.file.Files;
-import java.util.List;
-import java.util.Properties;
-
-/**
- * Test for ContentStreamDataSource
- *
- *
- * @since solr 1.4
- */
-public class TestContentStreamDataSource extends AbstractDataImportHandlerTestCase {
- private static final String CONF_DIR = "dih/solr/collection1/conf/";
- private static final String ROOT_DIR = "dih/solr/";
- SolrInstance instance = null;
- JettySolrRunner jetty;
-
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- instance = new SolrInstance("inst", null);
- instance.setUp();
- jetty = createJetty(instance);
- }
-
- @Override
- @After
- public void tearDown() throws Exception {
- jetty.stop();
- super.tearDown();
- }
-
- @Test
- public void testSimple() throws Exception {
- DirectXmlRequest req = new DirectXmlRequest("/dataimport", xml);
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("command", "full-import");
- params.set("clean", "false");
- req.setParams(params);
- try (HttpSolrClient solrClient = getHttpSolrClient(buildUrl(jetty.getLocalPort(), "/solr/collection1"))) {
- solrClient.request(req);
- ModifiableSolrParams qparams = new ModifiableSolrParams();
- qparams.add("q", "*:*");
- QueryResponse qres = solrClient.query(qparams);
- SolrDocumentList results = qres.getResults();
- assertEquals(2, results.getNumFound());
- SolrDocument doc = results.get(0);
- assertEquals("1", doc.getFieldValue("id"));
- assertEquals("Hello C1", ((List) doc.getFieldValue("desc")).get(0));
- }
- }
-
- @Test
- public void testCommitWithin() throws Exception {
- DirectXmlRequest req = new DirectXmlRequest("/dataimport", xml);
- ModifiableSolrParams params = params("command", "full-import",
- "clean", "false", UpdateParams.COMMIT, "false",
- UpdateParams.COMMIT_WITHIN, "1000");
- req.setParams(params);
- try (HttpSolrClient solrServer = getHttpSolrClient(buildUrl(jetty.getLocalPort(), "/solr/collection1"))) {
- solrServer.request(req);
- Thread.sleep(100);
- ModifiableSolrParams queryAll = params("q", "*", "df", "desc");
- QueryResponse qres = solrServer.query(queryAll);
- SolrDocumentList results = qres.getResults();
- assertEquals(0, results.getNumFound());
- Thread.sleep(1000);
- for (int i = 0; i < 10; i++) {
- qres = solrServer.query(queryAll);
- results = qres.getResults();
- if (2 == results.getNumFound()) {
- return;
- }
- Thread.sleep(500);
- }
- }
- fail("Commit should have occured but it did not");
- }
-
- private static class SolrInstance {
- String name;
- Integer port;
- File homeDir;
- File confDir;
- File dataDir;
-
- /**
- * if masterPort is null, this instance is a master -- otherwise this instance is a slave, and assumes the master is
- * on localhost at the specified port.
- */
- public SolrInstance(String name, Integer port) {
- this.name = name;
- this.port = port;
- }
-
- public String getHomeDir() {
- return homeDir.toString();
- }
-
- public String getSchemaFile() {
- return CONF_DIR + "dataimport-schema.xml";
- }
-
- public String getConfDir() {
- return confDir.toString();
- }
-
- public String getDataDir() {
- return dataDir.toString();
- }
-
- public String getSolrConfigFile() {
- return CONF_DIR + "contentstream-solrconfig.xml";
- }
-
- public String getSolrXmlFile() {
- return ROOT_DIR + "solr.xml";
- }
-
-
- public void setUp() throws Exception {
- homeDir = createTempDir("inst").toFile();
- dataDir = new File(homeDir + "/collection1", "data");
- confDir = new File(homeDir + "/collection1", "conf");
-
- homeDir.mkdirs();
- dataDir.mkdirs();
- confDir.mkdirs();
-
- FileUtils.copyFile(getFile(getSolrXmlFile()), new File(homeDir, "solr.xml"));
- File f = new File(confDir, "solrconfig.xml");
- FileUtils.copyFile(getFile(getSolrConfigFile()), f);
- f = new File(confDir, "schema.xml");
-
- FileUtils.copyFile(getFile(getSchemaFile()), f);
- f = new File(confDir, "data-config.xml");
- FileUtils.copyFile(getFile(CONF_DIR + "dataconfig-contentstream.xml"), f);
-
- Files.createFile(homeDir.toPath().resolve("collection1/core.properties"));
- }
-
- }
-
- private JettySolrRunner createJetty(SolrInstance instance) throws Exception {
- Properties nodeProperties = new Properties();
- nodeProperties.setProperty("solr.data.dir", instance.getDataDir());
- JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), nodeProperties, buildJettyConfig("/solr"));
- jetty.start();
- return jetty;
- }
-
- static String xml = "<root>\n"
- + "<b>\n"
- + " <id>1</id>\n"
- + " <c>Hello C1</c>\n"
- + "</b>\n"
- + "<b>\n"
- + " <id>2</id>\n"
- + " <c>Hello C2</c>\n"
- + "</b>\n" + "</root>";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContextImpl.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContextImpl.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContextImpl.java
deleted file mode 100644
index 4193550..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContextImpl.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.HashMap;
-
-import org.apache.solr.handler.dataimport.RequestInfo;
-import org.junit.Test;
-
-public class TestContextImpl extends AbstractDataImportHandlerTestCase {
-
- @Test
- public void testEntityScope() {
- ContextImpl ctx = new ContextImpl(null, new VariableResolver(), null, "something", new HashMap<String,Object>(), null, null);
- String lala = new String("lala");
- ctx.setSessionAttribute("huhu", lala, Context.SCOPE_ENTITY);
- Object got = ctx.getSessionAttribute("huhu", Context.SCOPE_ENTITY);
-
- assertEquals(lala, got);
-
- }
- @Test
- public void testCoreScope() {
- DataImporter di = new DataImporter();
- di.loadAndInit("<dataConfig><document /></dataConfig>");
- DocBuilder db = new DocBuilder(di, new SolrWriter(null, null),new SimplePropertiesWriter(), new RequestInfo(null, new HashMap<String,Object>(), null));
- ContextImpl ctx = new ContextImpl(null, new VariableResolver(), null, "something", new HashMap<String,Object>(), null, db);
- String lala = new String("lala");
- ctx.setSessionAttribute("huhu", lala, Context.SCOPE_SOLR_CORE);
- Object got = ctx.getSessionAttribute("huhu", Context.SCOPE_SOLR_CORE);
- assertEquals(lala, got);
-
- }
- @Test
- public void testDocumentScope() {
- ContextImpl ctx = new ContextImpl(null, new VariableResolver(), null, "something", new HashMap<String,Object>(), null, null);
- ctx.setDoc(new DocBuilder.DocWrapper());
- String lala = new String("lala");
- ctx.setSessionAttribute("huhu", lala, Context.SCOPE_DOC);
- Object got = ctx.getSessionAttribute("huhu", Context.SCOPE_DOC);
-
- assertEquals(lala, got);
-
- }
- @Test
- public void testGlobalScope() {
- ContextImpl ctx = new ContextImpl(null, new VariableResolver(), null, "something", new HashMap<String,Object>(), null, null);
- String lala = new String("lala");
- ctx.setSessionAttribute("huhu", lala, Context.SCOPE_GLOBAL);
- Object got = ctx.getSessionAttribute("huhu", Context.SCOPE_GLOBAL);
-
- assertEquals(lala, got);
-
- }
-
-}
[10/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockInitialContextFactory.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockInitialContextFactory.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockInitialContextFactory.java
new file mode 100644
index 0000000..7568210
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockInitialContextFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.HashMap;
+import java.util.Hashtable;
+import java.util.Map;
+
+import javax.naming.NamingException;
+import javax.naming.spi.InitialContextFactory;
+
+import static org.mockito.Mockito.*;
+
+public class MockInitialContextFactory implements InitialContextFactory {
+ private static final Map<String, Object> objects = new HashMap<>();
+ private final javax.naming.Context context;
+
+ public MockInitialContextFactory() {
+ context = mock(javax.naming.Context.class);
+
+ try {
+ when(context.lookup(anyString())).thenAnswer(invocation -> objects.get(invocation.getArgument(0)));
+
+ } catch (NamingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public javax.naming.Context getInitialContext(Hashtable env) {
+ return context;
+ }
+
+ public static void bind(String name, Object obj) {
+ objects.put(name, obj);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java
new file mode 100644
index 0000000..42e5f7d
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+
+import java.util.List;
+
+public class MockSolrEntityProcessor extends SolrEntityProcessor {
+
+ private final List<SolrTestCaseJ4.Doc> docsData;
+// private final int rows;
+ private int queryCount = 0;
+
+ private int rows;
+
+ private int start = 0;
+
+ public MockSolrEntityProcessor(List<SolrTestCaseJ4.Doc> docsData, int rows) {
+ this.docsData = docsData;
+ this.rows = rows;
+ }
+
+ //@Override
+ //protected SolrDocumentList doQuery(int start) {
+ // queryCount++;
+ // return getDocs(start, rows);
+ // }
+
+ @Override
+ protected void buildIterator() {
+ if (rowIterator==null || (!rowIterator.hasNext() && ((SolrDocumentListIterator)rowIterator).hasMoreRows())){
+ queryCount++;
+ SolrDocumentList docs = getDocs(start, rows);
+ rowIterator = new SolrDocumentListIterator(docs);
+ start += docs.size();
+ }
+ }
+
+ private SolrDocumentList getDocs(int start, int rows) {
+ SolrDocumentList docs = new SolrDocumentList();
+ docs.setNumFound(docsData.size());
+ docs.setStart(start);
+
+ int endIndex = start + rows;
+ int end = docsData.size() < endIndex ? docsData.size() : endIndex;
+ for (int i = start; i < end; i++) {
+ SolrDocument doc = new SolrDocument();
+ SolrTestCaseJ4.Doc testDoc = docsData.get(i);
+ doc.addField("id", testDoc.id);
+ doc.addField("description", testDoc.getValues("description"));
+ docs.add(doc);
+ }
+ return docs;
+ }
+
+ public int getQueryCount() {
+ return queryCount;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockStringDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockStringDataSource.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockStringDataSource.java
new file mode 100644
index 0000000..7c9a6d1
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockStringDataSource.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.dataimport;
+
+
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+public class MockStringDataSource extends DataSource<Reader> {
+
+ private static Map<String, String> cache = new HashMap<>();
+
+ public static void setData(String query,
+ String data) {
+ cache.put(query, data);
+ }
+
+ public static void clearCache() {
+ cache.clear();
+ }
+ @Override
+ public void init(Context context, Properties initProps) {
+
+ }
+
+ @Override
+ public Reader getData(String query) {
+ return new StringReader(cache.get(query));
+ }
+
+ @Override
+ public void close() {
+ cache.clear();
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
new file mode 100644
index 0000000..986a8cd
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+/**
+ * <p> Test for Evaluators </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestBuiltInEvaluators extends AbstractDataImportHandlerTestCase {
+ private static final String ENCODING = StandardCharsets.UTF_8.name();
+
+ VariableResolver resolver;
+
+ Map<String, String> sqlTests;
+
+ Map<String, String> urlTests;
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ resolver = new VariableResolver();
+
+ sqlTests = new HashMap<>();
+
+ sqlTests.put("foo\"", "foo\"\"");
+ sqlTests.put("foo\\", "foo\\\\");
+ sqlTests.put("foo'", "foo''");
+ sqlTests.put("foo''", "foo''''");
+ sqlTests.put("'foo\"", "''foo\"\"");
+ sqlTests.put("\"Albert D'souza\"", "\"\"Albert D''souza\"\"");
+
+ urlTests = new HashMap<>();
+
+ urlTests.put("*:*", URLEncoder.encode("*:*", ENCODING));
+ urlTests.put("price:[* TO 200]", URLEncoder.encode("price:[* TO 200]",
+ ENCODING));
+ urlTests.put("review:\"hybrid sedan\"", URLEncoder.encode(
+ "review:\"hybrid sedan\"", ENCODING));
+ }
+
+
+ @Test
+ public void testSqlEscapingEvaluator() {
+ Evaluator sqlEscaper = new SqlEscapingEvaluator();
+ runTests(sqlTests, sqlEscaper);
+ }
+
+
+ @Test
+ public void testUrlEvaluator() throws Exception {
+ Evaluator urlEvaluator = new UrlEvaluator();
+ runTests(urlTests, urlEvaluator);
+ }
+
+ @Test
+ public void parseParams() {
+ Map<String,Object> m = new HashMap<>();
+ m.put("b","B");
+ VariableResolver vr = new VariableResolver();
+ vr.addNamespace("a",m);
+ List<Object> l = (new Evaluator() {
+ @Override
+ public String evaluate(String expression, Context context) {
+ return null;
+ }
+ }).parseParams(" 1 , a.b, 'hello!', 'ds,o,u\'za',",vr);
+ assertEquals(1d,l.get(0));
+ assertEquals("B",((Evaluator.VariableWrapper)l.get(1)).resolve());
+ assertEquals("hello!",l.get(2));
+ assertEquals("ds,o,u'za",l.get(3));
+ }
+
+ @Test
+ public void testEscapeSolrQueryFunction() {
+ final VariableResolver resolver = new VariableResolver();
+ Map<String,Object> m= new HashMap<>();
+ m.put("query","c:t");
+ resolver.setEvaluators(new DataImporter().getEvaluators(Collections.<Map<String,String>>emptyList()));
+
+ resolver.addNamespace("e",m);
+ String s = resolver
+ .replaceTokens("${dataimporter.functions.escapeQueryChars(e.query)}");
+ org.junit.Assert.assertEquals("c\\:t", s);
+
+ }
+
+ private Date twoDaysAgo(Locale l, TimeZone tz) {
+ Calendar calendar = Calendar.getInstance(tz, l);
+ calendar.add(Calendar.DAY_OF_YEAR, -2);
+ return calendar.getTime();
+ }
+
+ @Test
+ public void testDateFormatEvaluator() {
+ Evaluator dateFormatEval = new DateFormatEvaluator();
+ ContextImpl context = new ContextImpl(null, resolver, null,
+ Context.FULL_DUMP, Collections.<String,Object> emptyMap(), null, null);
+
+ Locale rootLocale = Locale.ROOT;
+ Locale defaultLocale = Locale.getDefault();
+ TimeZone defaultTz = TimeZone.getDefault();
+
+ {
+ SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd HH", rootLocale);
+ String sdf = sdfDate.format(twoDaysAgo(rootLocale, defaultTz));
+ String dfe = dateFormatEval.evaluate("'NOW-2DAYS','yyyy-MM-dd HH'", context);
+ assertEquals(sdf,dfe);
+ }
+ {
+ SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd HH", defaultLocale);
+ String sdf = sdfDate.format(twoDaysAgo(defaultLocale, TimeZone.getDefault()));
+ String dfe = dateFormatEval.evaluate(
+ "'NOW-2DAYS','yyyy-MM-dd HH','" + defaultLocale.toLanguageTag() + "'", context);
+ assertEquals(sdf,dfe);
+ for(String tzStr : TimeZone.getAvailableIDs()) {
+ TimeZone tz = TimeZone.getTimeZone(tzStr);
+ sdfDate.setTimeZone(tz);
+ sdf = sdfDate.format(twoDaysAgo(defaultLocale, tz));
+ dfe = dateFormatEval.evaluate(
+ "'NOW-2DAYS','yyyy-MM-dd HH','" + defaultLocale.toLanguageTag() + "','" + tzStr + "'", context);
+ assertEquals(sdf,dfe);
+ }
+ }
+
+ Date d = new Date();
+ Map<String,Object> map = new HashMap<>();
+ map.put("key", d);
+ resolver.addNamespace("A", map);
+
+ assertEquals(
+ new SimpleDateFormat("yyyy-MM-dd HH:mm", rootLocale).format(d),
+ dateFormatEval.evaluate("A.key, 'yyyy-MM-dd HH:mm'", context));
+ assertEquals(
+ new SimpleDateFormat("yyyy-MM-dd HH:mm", defaultLocale).format(d),
+ dateFormatEval.evaluate("A.key, 'yyyy-MM-dd HH:mm','" + defaultLocale.toLanguageTag() + "'", context));
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm", defaultLocale);
+ for(String tzStr : TimeZone.getAvailableIDs()) {
+ TimeZone tz = TimeZone.getTimeZone(tzStr);
+ sdf.setTimeZone(tz);
+ assertEquals(
+ sdf.format(d),
+ dateFormatEval.evaluate(
+ "A.key, 'yyyy-MM-dd HH:mm','" + defaultLocale.toLanguageTag() + "', '" + tzStr + "'", context));
+
+ }
+
+
+ }
+
+ private void runTests(Map<String, String> tests, Evaluator evaluator) {
+ ContextImpl ctx = new ContextImpl(null, resolver, null, Context.FULL_DUMP, Collections.<String, Object>emptyMap(), null, null);
+ for (Map.Entry<String, String> entry : tests.entrySet()) {
+ Map<String, Object> values = new HashMap<>();
+ values.put("key", entry.getKey());
+ resolver.addNamespace("A", values);
+
+ String expected = entry.getValue();
+ String actual = evaluator.evaluate("A.key", ctx);
+ assertEquals(expected, actual);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestClobTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestClobTransformer.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestClobTransformer.java
new file mode 100644
index 0000000..fe00d49
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestClobTransformer.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.Test;
+
+import java.io.StringReader;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.sql.Clob;
+import java.util.*;
+
+/**
+ * Test for ClobTransformer
+ *
+ *
+ * @see org.apache.solr.handler.dataimport.ClobTransformer
+ * @since solr 1.4
+ */
+public class TestClobTransformer extends AbstractDataImportHandlerTestCase {
+ @Test
+ public void simple() throws Exception {
+ List<Map<String, String>> flds = new ArrayList<>();
+ Map<String, String> f = new HashMap<>();
+ // <field column="dsc" clob="true" name="description" />
+ f.put(DataImporter.COLUMN, "dsc");
+ f.put(ClobTransformer.CLOB, "true");
+ f.put(DataImporter.NAME, "description");
+ flds.add(f);
+ Context ctx = getContext(null, new VariableResolver(), null, Context.FULL_DUMP, flds, Collections.EMPTY_MAP);
+ Transformer t = new ClobTransformer();
+ Map<String, Object> row = new HashMap<>();
+ Clob clob = (Clob) Proxy.newProxyInstance(this.getClass().getClassLoader(), new Class[]{Clob.class}, new InvocationHandler() {
+ @Override
+ public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+ if (method.getName().equals("getCharacterStream")) {
+ return new StringReader("hello!");
+ }
+ return null;
+ }
+ });
+
+ row.put("dsc", clob);
+ t.transformRow(row, ctx);
+ assertEquals("hello!", row.get("dsc"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
new file mode 100644
index 0000000..06fd51c
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.DirectXmlRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.UpdateParams;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * Test for ContentStreamDataSource
+ *
+ *
+ * @since solr 1.4
+ */
+public class TestContentStreamDataSource extends AbstractDataImportHandlerTestCase {
+ private static final String CONF_DIR = "dih/solr/collection1/conf/";
+ private static final String ROOT_DIR = "dih/solr/";
+ SolrInstance instance = null;
+ JettySolrRunner jetty;
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ instance = new SolrInstance("inst", null);
+ instance.setUp();
+ jetty = createJetty(instance);
+ }
+
+ @Override
+ @After
+ public void tearDown() throws Exception {
+ jetty.stop();
+ super.tearDown();
+ }
+
+ @Test
+ public void testSimple() throws Exception {
+ DirectXmlRequest req = new DirectXmlRequest("/dataimport", xml);
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("command", "full-import");
+ params.set("clean", "false");
+ req.setParams(params);
+ try (HttpSolrClient solrClient = getHttpSolrClient(buildUrl(jetty.getLocalPort(), "/solr/collection1"))) {
+ solrClient.request(req);
+ ModifiableSolrParams qparams = new ModifiableSolrParams();
+ qparams.add("q", "*:*");
+ QueryResponse qres = solrClient.query(qparams);
+ SolrDocumentList results = qres.getResults();
+ assertEquals(2, results.getNumFound());
+ SolrDocument doc = results.get(0);
+ assertEquals("1", doc.getFieldValue("id"));
+ assertEquals("Hello C1", ((List) doc.getFieldValue("desc")).get(0));
+ }
+ }
+
+ @Test
+ public void testCommitWithin() throws Exception {
+ DirectXmlRequest req = new DirectXmlRequest("/dataimport", xml);
+ ModifiableSolrParams params = params("command", "full-import",
+ "clean", "false", UpdateParams.COMMIT, "false",
+ UpdateParams.COMMIT_WITHIN, "1000");
+ req.setParams(params);
+ try (HttpSolrClient solrServer = getHttpSolrClient(buildUrl(jetty.getLocalPort(), "/solr/collection1"))) {
+ solrServer.request(req);
+ Thread.sleep(100);
+ ModifiableSolrParams queryAll = params("q", "*", "df", "desc");
+ QueryResponse qres = solrServer.query(queryAll);
+ SolrDocumentList results = qres.getResults();
+ assertEquals(0, results.getNumFound());
+ Thread.sleep(1000);
+ for (int i = 0; i < 10; i++) {
+ qres = solrServer.query(queryAll);
+ results = qres.getResults();
+ if (2 == results.getNumFound()) {
+ return;
+ }
+ Thread.sleep(500);
+ }
+ }
+ fail("Commit should have occured but it did not");
+ }
+
+ private static class SolrInstance {
+ String name;
+ Integer port;
+ File homeDir;
+ File confDir;
+ File dataDir;
+
+ /**
+ * if masterPort is null, this instance is a master -- otherwise this instance is a slave, and assumes the master is
+ * on localhost at the specified port.
+ */
+ public SolrInstance(String name, Integer port) {
+ this.name = name;
+ this.port = port;
+ }
+
+ public String getHomeDir() {
+ return homeDir.toString();
+ }
+
+ public String getSchemaFile() {
+ return CONF_DIR + "dataimport-schema.xml";
+ }
+
+ public String getConfDir() {
+ return confDir.toString();
+ }
+
+ public String getDataDir() {
+ return dataDir.toString();
+ }
+
+ public String getSolrConfigFile() {
+ return CONF_DIR + "contentstream-solrconfig.xml";
+ }
+
+ public String getSolrXmlFile() {
+ return ROOT_DIR + "solr.xml";
+ }
+
+
+ public void setUp() throws Exception {
+ homeDir = createTempDir("inst").toFile();
+ dataDir = new File(homeDir + "/collection1", "data");
+ confDir = new File(homeDir + "/collection1", "conf");
+
+ homeDir.mkdirs();
+ dataDir.mkdirs();
+ confDir.mkdirs();
+
+ FileUtils.copyFile(getFile(getSolrXmlFile()), new File(homeDir, "solr.xml"));
+ File f = new File(confDir, "solrconfig.xml");
+ FileUtils.copyFile(getFile(getSolrConfigFile()), f);
+ f = new File(confDir, "schema.xml");
+
+ FileUtils.copyFile(getFile(getSchemaFile()), f);
+ f = new File(confDir, "data-config.xml");
+ FileUtils.copyFile(getFile(CONF_DIR + "dataconfig-contentstream.xml"), f);
+
+ Files.createFile(homeDir.toPath().resolve("collection1/core.properties"));
+ }
+
+ }
+
+ private JettySolrRunner createJetty(SolrInstance instance) throws Exception {
+ Properties nodeProperties = new Properties();
+ nodeProperties.setProperty("solr.data.dir", instance.getDataDir());
+ JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), nodeProperties, buildJettyConfig("/solr"));
+ jetty.start();
+ return jetty;
+ }
+
+ static String xml = "<root>\n"
+ + "<b>\n"
+ + " <id>1</id>\n"
+ + " <c>Hello C1</c>\n"
+ + "</b>\n"
+ + "<b>\n"
+ + " <id>2</id>\n"
+ + " <c>Hello C2</c>\n"
+ + "</b>\n" + "</root>";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestContextImpl.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestContextImpl.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestContextImpl.java
new file mode 100644
index 0000000..4193550
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestContextImpl.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.HashMap;
+
+import org.apache.solr.handler.dataimport.RequestInfo;
+import org.junit.Test;
+
+public class TestContextImpl extends AbstractDataImportHandlerTestCase {
+
+ @Test
+ public void testEntityScope() {
+ ContextImpl ctx = new ContextImpl(null, new VariableResolver(), null, "something", new HashMap<String,Object>(), null, null);
+ String lala = new String("lala");
+ ctx.setSessionAttribute("huhu", lala, Context.SCOPE_ENTITY);
+ Object got = ctx.getSessionAttribute("huhu", Context.SCOPE_ENTITY);
+
+ assertEquals(lala, got);
+
+ }
+ @Test
+ public void testCoreScope() {
+ DataImporter di = new DataImporter();
+ di.loadAndInit("<dataConfig><document /></dataConfig>");
+ DocBuilder db = new DocBuilder(di, new SolrWriter(null, null),new SimplePropertiesWriter(), new RequestInfo(null, new HashMap<String,Object>(), null));
+ ContextImpl ctx = new ContextImpl(null, new VariableResolver(), null, "something", new HashMap<String,Object>(), null, db);
+ String lala = new String("lala");
+ ctx.setSessionAttribute("huhu", lala, Context.SCOPE_SOLR_CORE);
+ Object got = ctx.getSessionAttribute("huhu", Context.SCOPE_SOLR_CORE);
+ assertEquals(lala, got);
+
+ }
+ @Test
+ public void testDocumentScope() {
+ ContextImpl ctx = new ContextImpl(null, new VariableResolver(), null, "something", new HashMap<String,Object>(), null, null);
+ ctx.setDoc(new DocBuilder.DocWrapper());
+ String lala = new String("lala");
+ ctx.setSessionAttribute("huhu", lala, Context.SCOPE_DOC);
+ Object got = ctx.getSessionAttribute("huhu", Context.SCOPE_DOC);
+
+ assertEquals(lala, got);
+
+ }
+ @Test
+ public void testGlobalScope() {
+ ContextImpl ctx = new ContextImpl(null, new VariableResolver(), null, "something", new HashMap<String,Object>(), null, null);
+ String lala = new String("lala");
+ ctx.setSessionAttribute("huhu", lala, Context.SCOPE_GLOBAL);
+ Object got = ctx.getSessionAttribute("huhu", Context.SCOPE_GLOBAL);
+
+ assertEquals(lala, got);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java
new file mode 100644
index 0000000..7a56fd2
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.handler.dataimport.config.DIHConfiguration;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.w3c.dom.Document;
+import org.xml.sax.InputSource;
+
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * <p>
+ * Test for DataConfig
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestDataConfig extends AbstractDataImportHandlerTestCase {
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("dataimport-nodatasource-solrconfig.xml", "dataimport-schema.xml");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testDataConfigWithDataSource() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(loadDataConfig("data-config-with-datasource.xml"));
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ }
+
+ @Test
+ public void testBasic() throws Exception {
+ javax.xml.parsers.DocumentBuilder builder = DocumentBuilderFactory
+ .newInstance().newDocumentBuilder();
+ Document doc = builder.parse(new InputSource(new StringReader(xml)));
+ DataImporter di = new DataImporter();
+ DIHConfiguration dc = di.readFromXml(doc);
+ assertEquals("atrimlisting", dc.getEntities().get(0).getName());
+ }
+
+ private static final String xml = "<dataConfig>\n"
+ + "\t<document name=\"autos\" >\n"
+ + "\t\t<entity name=\"atrimlisting\" pk=\"acode\"\n"
+ + "\t\t\tquery=\"select acode,make,model,year,msrp,category,image,izmo_image_url,price_range_low,price_range_high,invoice_range_low,invoice_range_high from atrimlisting\"\n"
+ + "\t\t\tdeltaQuery=\"select acode from atrimlisting where last_modified > '${indexer.last_index_time}'\">\n"
+ +
+
+ "\t\t</entity>\n" +
+
+ "\t</document>\n" + "</dataConfig>";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java
new file mode 100644
index 0000000..a1e85d7
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.Test;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+/**
+ * <p>
+ * Test for DateFormatTransformer
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestDateFormatTransformer extends AbstractDataImportHandlerTestCase {
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testTransformRow_SingleRow() throws Exception {
+ List<Map<String, String>> fields = new ArrayList<>();
+ fields.add(createMap(DataImporter.COLUMN, "lastModified"));
+ fields.add(createMap(DataImporter.COLUMN,
+ "dateAdded", RegexTransformer.SRC_COL_NAME, "lastModified",
+ DateFormatTransformer.DATE_TIME_FMT, "${xyz.myDateFormat}"));
+
+ SimpleDateFormat format = new SimpleDateFormat("MM/dd/yyyy", Locale.ROOT);
+ Date now = format.parse(format.format(new Date()));
+
+ Map<String,Object> row = createMap("lastModified", format.format(now));
+
+ VariableResolver resolver = new VariableResolver();
+ resolver.addNamespace("e", row);
+ resolver.addNamespace("xyz", createMap("myDateFormat", "MM/dd/yyyy"));
+
+ Context context = getContext(null, resolver,
+ null, Context.FULL_DUMP, fields, null);
+ new DateFormatTransformer().transformRow(row, context);
+ assertEquals(now, row.get("dateAdded"));
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testTransformRow_MultipleRows() throws Exception {
+ List<Map<String, String>> fields = new ArrayList<>();
+ fields.add(createMap(DataImporter.COLUMN, "lastModified"));
+ fields.add(createMap(DataImporter.COLUMN,
+ "dateAdded", RegexTransformer.SRC_COL_NAME, "lastModified",
+ DateFormatTransformer.DATE_TIME_FMT, "MM/dd/yyyy hh:mm:ss.SSS"));
+
+ SimpleDateFormat format = new SimpleDateFormat("MM/dd/yyyy hh:mm:ss.SSS", Locale.ROOT);
+ Date now1 = format.parse(format.format(new Date()));
+ Date now2 = format.parse(format.format(new Date()));
+
+ Map<String,Object> row = new HashMap<>();
+ List<String> list = new ArrayList<>();
+ list.add(format.format(now1));
+ list.add(format.format(now2));
+ row.put("lastModified", list);
+
+ VariableResolver resolver = new VariableResolver();
+ resolver.addNamespace("e", row);
+
+ Context context = getContext(null, resolver,
+ null, Context.FULL_DUMP, fields, null);
+ new DateFormatTransformer().transformRow(row, context);
+ List<Object> output = new ArrayList<>();
+ output.add(now1);
+ output.add(now2);
+ assertEquals(output, row.get("dateAdded"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java
new file mode 100644
index 0000000..7f7278e
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java
@@ -0,0 +1,335 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.handler.dataimport.config.DIHConfiguration;
+import org.apache.solr.handler.dataimport.config.Entity;
+
+import org.junit.After;
+import org.junit.Test;
+
+import java.util.*;
+
+/**
+ * <p>
+ * Test for DocBuilder
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+// See: https://issues.apache.org/jira/browse/SOLR-12028 Tests cannot remove files on Windows machines occasionally
+public class TestDocBuilder extends AbstractDataImportHandlerTestCase {
+
+ @Override
+ @After
+ public void tearDown() throws Exception {
+ MockDataSource.clearCache();
+ MockStringDataSource.clearCache();
+ super.tearDown();
+ }
+
+ @Test
+ public void loadClass() throws Exception {
+ @SuppressWarnings("unchecked")
+ Class<Transformer> clz = DocBuilder.loadClass("RegexTransformer", null);
+ assertNotNull(clz);
+ }
+
+ @Test
+ public void singleEntityNoRows() {
+ DataImporter di = new DataImporter();
+ di.loadAndInit(dc_singleEntity);
+ DIHConfiguration cfg = di.getConfig();
+ Entity ent = cfg.getEntities().get(0);
+ MockDataSource.setIterator("select * from x", new ArrayList<Map<String, Object>>().iterator());
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals(Boolean.TRUE, swi.deleteAllCalled);
+ assertEquals(Boolean.TRUE, swi.commitCalled);
+ assertEquals(Boolean.TRUE, swi.finishCalled);
+ assertEquals(0, swi.docs.size());
+ assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
+ assertEquals(0, di.getDocBuilder().importStatistics.docCount.get());
+ assertEquals(0, di.getDocBuilder().importStatistics.rowsCount.get());
+ }
+
+ @Test
+ public void testDeltaImportNoRows_MustNotCommit() {
+ DataImporter di = new DataImporter();
+ di.loadAndInit(dc_deltaConfig);
+ redirectTempProperties(di);
+
+ DIHConfiguration cfg = di.getConfig();
+ Entity ent = cfg.getEntities().get(0);
+ MockDataSource.setIterator("select * from x", new ArrayList<Map<String, Object>>().iterator());
+ MockDataSource.setIterator("select id from x", new ArrayList<Map<String, Object>>().iterator());
+ RequestInfo rp = new RequestInfo(null, createMap("command", "delta-import"), null);
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals(Boolean.FALSE, swi.deleteAllCalled);
+ assertEquals(Boolean.FALSE, swi.commitCalled);
+ assertEquals(Boolean.TRUE, swi.finishCalled);
+ assertEquals(0, swi.docs.size());
+ assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
+ assertEquals(0, di.getDocBuilder().importStatistics.docCount.get());
+ assertEquals(0, di.getDocBuilder().importStatistics.rowsCount.get());
+ }
+
+ @Test
+ public void singleEntityOneRow() {
+ DataImporter di = new DataImporter();
+ di.loadAndInit(dc_singleEntity);
+ DIHConfiguration cfg = di.getConfig();
+ Entity ent = cfg.getEntities().get(0);
+ List<Map<String, Object>> l = new ArrayList<>();
+ l.add(createMap("id", 1, "desc", "one"));
+ MockDataSource.setIterator("select * from x", l.iterator());
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals(Boolean.TRUE, swi.deleteAllCalled);
+ assertEquals(Boolean.TRUE, swi.commitCalled);
+ assertEquals(Boolean.TRUE, swi.finishCalled);
+ assertEquals(1, swi.docs.size());
+ assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
+ assertEquals(1, di.getDocBuilder().importStatistics.docCount.get());
+ assertEquals(1, di.getDocBuilder().importStatistics.rowsCount.get());
+
+ for (int i = 0; i < l.size(); i++) {
+ Map<String, Object> map = l.get(i);
+ SolrInputDocument doc = swi.docs.get(i);
+ for (Map.Entry<String, Object> entry : map.entrySet()) {
+ assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
+ }
+ }
+ }
+
+ @Test
+ public void testImportCommand() {
+ DataImporter di = new DataImporter();
+ di.loadAndInit(dc_singleEntity);
+ DIHConfiguration cfg = di.getConfig();
+ Entity ent = cfg.getEntities().get(0);
+ List<Map<String, Object>> l = new ArrayList<>();
+ l.add(createMap("id", 1, "desc", "one"));
+ MockDataSource.setIterator("select * from x", l.iterator());
+ RequestInfo rp = new RequestInfo(null, createMap("command", "import"), null);
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals(Boolean.FALSE, swi.deleteAllCalled);
+ assertEquals(Boolean.TRUE, swi.commitCalled);
+ assertEquals(Boolean.TRUE, swi.finishCalled);
+ assertEquals(1, swi.docs.size());
+ assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
+ assertEquals(1, di.getDocBuilder().importStatistics.docCount.get());
+ assertEquals(1, di.getDocBuilder().importStatistics.rowsCount.get());
+
+ for (int i = 0; i < l.size(); i++) {
+ Map<String, Object> map = (Map<String, Object>) l.get(i);
+ SolrInputDocument doc = swi.docs.get(i);
+ for (Map.Entry<String, Object> entry : map.entrySet()) {
+ assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
+ }
+ }
+ }
+
+ @Test
+ public void singleEntityMultipleRows() {
+ DataImporter di = new DataImporter();
+ di.loadAndInit(dc_singleEntity);
+ DIHConfiguration cfg = di.getConfig();
+ Entity ent = cfg.getEntities().get(0);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
+ List<Map<String, Object>> l = new ArrayList<>();
+ l.add(createMap("id", 1, "desc", "one"));
+ l.add(createMap("id", 2, "desc", "two"));
+ l.add(createMap("id", 3, "desc", "three"));
+
+ MockDataSource.setIterator("select * from x", l.iterator());
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals(Boolean.TRUE, swi.deleteAllCalled);
+ assertEquals(Boolean.TRUE, swi.commitCalled);
+ assertEquals(Boolean.TRUE, swi.finishCalled);
+ assertEquals(3, swi.docs.size());
+ for (int i = 0; i < l.size(); i++) {
+ Map<String, Object> map = (Map<String, Object>) l.get(i);
+ SolrInputDocument doc = swi.docs.get(i);
+ for (Map.Entry<String, Object> entry : map.entrySet()) {
+ assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
+ }
+ assertEquals(map.get("desc"), doc.getFieldValue("desc_s"));
+ }
+ assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
+ assertEquals(3, di.getDocBuilder().importStatistics.docCount.get());
+ assertEquals(3, di.getDocBuilder().importStatistics.rowsCount.get());
+ }
+
+ @Test
+ public void templateXPath() {
+ DataImporter di = new DataImporter();
+ di.loadAndInit(dc_variableXpath);
+ DIHConfiguration cfg = di.getConfig();
+
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
+ List<Map<String, Object>> l = new ArrayList<>();
+ l.add(createMap("id", 1, "name", "iphone", "manufacturer", "Apple"));
+ l.add(createMap("id", 2, "name", "ipad", "manufacturer", "Apple"));
+ l.add(createMap("id", 3, "name", "pixel", "manufacturer", "Google"));
+
+ MockDataSource.setIterator("select * from x", l.iterator());
+
+ List<Map<String,Object>> nestedData = new ArrayList<>();
+ nestedData.add(createMap("founded", "Cupertino, California, U.S", "year", "1976", "year2", "1976"));
+ nestedData.add(createMap("founded", "Cupertino, California, U.S", "year", "1976", "year2", "1976"));
+ nestedData.add(createMap("founded", "Menlo Park, California, U.S", "year", "1998", "year2", "1998"));
+
+ MockStringDataSource.setData("companies.xml", xml_attrVariableXpath);
+ MockStringDataSource.setData("companies2.xml", xml_variableXpath);
+ MockStringDataSource.setData("companies3.xml", xml_variableForEach);
+
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals(Boolean.TRUE, swi.deleteAllCalled);
+ assertEquals(Boolean.TRUE, swi.commitCalled);
+ assertEquals(Boolean.TRUE, swi.finishCalled);
+ assertEquals(3, swi.docs.size());
+ for (int i = 0; i < l.size(); i++) {
+ SolrInputDocument doc = swi.docs.get(i);
+
+ Map<String, Object> map = l.get(i);
+ for (Map.Entry<String, Object> entry : map.entrySet()) {
+ assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
+ }
+
+ map = nestedData.get(i);
+ for (Map.Entry<String, Object> entry : map.entrySet()) {
+ assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
+ }
+ }
+ assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
+ assertEquals(3, di.getDocBuilder().importStatistics.docCount.get());
+ }
+
+ static class SolrWriterImpl extends SolrWriter {
+ List<SolrInputDocument> docs = new ArrayList<>();
+
+ Boolean deleteAllCalled = Boolean.FALSE;
+
+ Boolean commitCalled = Boolean.FALSE;
+
+ Boolean finishCalled = Boolean.FALSE;
+
+ public SolrWriterImpl() {
+ super(null, null);
+ }
+
+ @Override
+ public boolean upload(SolrInputDocument doc) {
+ return docs.add(doc);
+ }
+
+ @Override
+ public void doDeleteAll() {
+ deleteAllCalled = Boolean.TRUE;
+ }
+
+ @Override
+ public void commit(boolean b) {
+ commitCalled = Boolean.TRUE;
+ }
+
+ @Override
+ public void close() {
+ finishCalled = Boolean.TRUE;
+ }
+ }
+
+ public static final String dc_singleEntity = "<dataConfig>\n"
+ + "<dataSource type=\"MockDataSource\"/>\n"
+ + " <document name=\"X\" >\n"
+ + " <entity name=\"x\" query=\"select * from x\">\n"
+ + " <field column=\"id\"/>\n"
+ + " <field column=\"desc\"/>\n"
+ + " <field column=\"desc\" name=\"desc_s\" />" + " </entity>\n"
+ + " </document>\n" + "</dataConfig>";
+
+ public static final String dc_deltaConfig = "<dataConfig>\n"
+ + "<dataSource type=\"MockDataSource\"/>\n"
+ + " <document name=\"X\" >\n"
+ + " <entity name=\"x\" query=\"select * from x\" deltaQuery=\"select id from x\">\n"
+ + " <field column=\"id\"/>\n"
+ + " <field column=\"desc\"/>\n"
+ + " <field column=\"desc\" name=\"desc_s\" />" + " </entity>\n"
+ + " </document>\n" + "</dataConfig>";
+
+ public static final String dc_variableXpath = "<dataConfig>\n"
+ + "<dataSource type=\"MockDataSource\"/>\n"
+ + "<dataSource name=\"xml\" type=\"MockStringDataSource\"/>\n"
+ + " <document name=\"X\" >\n"
+ + " <entity name=\"x\" query=\"select * from x\">\n"
+ + " <field column=\"id\"/>\n"
+ + " <field column=\"name\"/>\n"
+ + " <field column=\"manufacturer\"/>"
+ + " <entity name=\"c1\" url=\"companies.xml\" dataSource=\"xml\" forEach=\"/companies/company\" processor=\"XPathEntityProcessor\">"
+ + " <field column=\"year\" xpath=\"/companies/company/year[@name='p_${x.manufacturer}_s']\" />"
+ + " </entity>"
+ + " <entity name=\"c2\" url=\"companies2.xml\" dataSource=\"xml\" forEach=\"/companies/company\" processor=\"XPathEntityProcessor\">"
+ + " <field column=\"founded\" xpath=\"/companies/company/p_${x.manufacturer}_s/founded\" />"
+ + " </entity>"
+ + " <entity name=\"c3\" url=\"companies3.xml\" dataSource=\"xml\" forEach=\"/companies/${x.manufacturer}\" processor=\"XPathEntityProcessor\">"
+ + " <field column=\"year2\" xpath=\"/companies/${x.manufacturer}/year\" />"
+ + " </entity>"
+ + " </entity>\n"
+ + " </document>\n" + "</dataConfig>";
+
+
+ public static final String xml_variableForEach = "<companies>\n" +
+ "\t<Apple>\n" +
+ "\t\t<year>1976</year>\n" +
+ "\t</Apple>\n" +
+ "\t<Google>\n" +
+ "\t\t<year>1998</year>\n" +
+ "\t</Google>\n" +
+ "</companies>";
+
+ public static final String xml_variableXpath = "<companies>\n" +
+ "\t<company>\n" +
+ "\t\t<p_Apple_s>\n" +
+ "\t\t\t<founded>Cupertino, California, U.S</founded>\n" +
+ "\t\t</p_Apple_s>\t\t\n" +
+ "\t</company>\n" +
+ "\t<company>\n" +
+ "\t\t<p_Google_s>\n" +
+ "\t\t\t<founded>Menlo Park, California, U.S</founded>\n" +
+ "\t\t</p_Google_s>\n" +
+ "\t</company>\n" +
+ "</companies>";
+
+ public static final String xml_attrVariableXpath = "<companies>\n" +
+ "\t<company>\n" +
+ "\t\t<year name='p_Apple_s'>1976</year>\n" +
+ "\t</company>\n" +
+ "\t<company>\n" +
+ "\t\t<year name='p_Google_s'>1998</year>\t\t\n" +
+ "\t</company>\n" +
+ "</companies>";
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java
new file mode 100644
index 0000000..27865cd
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java
@@ -0,0 +1,433 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * <p>
+ * Test for DocBuilder using the test harness
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestDocBuilder2 extends AbstractDataImportHandlerTestCase {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testSingleEntity() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(loadDataConfig("single-entity-data-config.xml"));
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+
+ assertTrue("Update request processor processAdd was not called", TestUpdateRequestProcessor.processAddCalled);
+ assertTrue("Update request processor processCommit was not callled", TestUpdateRequestProcessor.processCommitCalled);
+ assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testSingleEntity_CaseInsensitive() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desC", "one"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(DATA_CONFIG_WITH_CASE_INSENSITIVE_FIELDS);
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertTrue("Start event listener was not called", StartEventListener.executed);
+ assertTrue("End event listener was not called", EndEventListener.executed);
+ assertTrue("Update request processor processAdd was not called", TestUpdateRequestProcessor.processAddCalled);
+ assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
+ }
+
+ @Test
+ public void testErrorHandler() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "FORCE_ERROR", "true"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(DATA_CONFIG_WITH_ERROR_HANDLER);
+
+ assertTrue("Error event listener was not called", ErrorEventListener.executed);
+ assertTrue(ErrorEventListener.lastException.getMessage().contains("ForcedException"));
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testDynamicFields() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(DATA_CONFIG_WITH_DYNAMIC_TRANSFORMER);
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("dynamic_s:test"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testRequestParamsAsVariable() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "101", "desc", "ApacheSolr"));
+ MockDataSource.setIterator("select * from books where category='search'", rows.iterator());
+
+ LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import",
+ "debug", "on", "clean", "true", "commit", "true",
+ "category", "search",
+ "dataConfig", REQUEST_PARAM_AS_VARIABLE);
+ h.query("/dataimport", request);
+ assertQ(req("desc:ApacheSolr"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testDynamicFieldNames() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("mypk", "101", "text", "ApacheSolr"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import",
+ "debug", "on", "clean", "true", "commit", "true",
+ "dataConfig", DATA_CONFIG_WITH_DYNAMIC_FIELD_NAMES);
+ h.query("/dataimport", request);
+ assertQ(req("id:101"), "//*[@numFound='1']", "//*[@name='101_s']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testRequestParamsAsFieldName() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("mypk", "101", "text", "ApacheSolr"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import",
+ "debug", "on", "clean", "true", "commit", "true",
+ "mypk", "id", "text", "desc",
+ "dataConfig", DATA_CONFIG_WITH_TEMPLATIZED_FIELD_NAMES);
+ h.query("/dataimport", request);
+ assertQ(req("id:101"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testContext() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(loadDataConfig("data-config-with-transformer.xml"));
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testSkipDoc() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ rows.add(createMap("id", "2", "desc", "two", DocBuilder.SKIP_DOC, "true"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(DATA_CONFIG_WITH_DYNAMIC_TRANSFORMER);
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='0']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testSkipRow() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ rows.add(createMap("id", "2", "desc", "two", DocBuilder.SKIP_ROW, "true"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(DATA_CONFIG_WITH_DYNAMIC_TRANSFORMER);
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='0']");
+
+ MockDataSource.clearCache();
+
+ rows = new ArrayList();
+ rows.add(createMap("id", "3", "desc", "one"));
+ rows.add(createMap("id", "4", "desc", "two"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ rows = new ArrayList();
+ rows.add(createMap("name_s", "abcd"));
+ MockDataSource.setIterator("3", rows.iterator());
+
+ rows = new ArrayList();
+ rows.add(createMap("name_s", "xyz", DocBuilder.SKIP_ROW, "true"));
+ MockDataSource.setIterator("4", rows.iterator());
+
+ runFullImport(DATA_CONFIG_WITH_TWO_ENTITIES);
+ assertQ(req("id:3"), "//*[@numFound='1']");
+ assertQ(req("id:4"), "//*[@numFound='1']");
+ assertQ(req("name_s:abcd"), "//*[@numFound='1']");
+ assertQ(req("name_s:xyz"), "//*[@numFound='0']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testStopTransform() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ rows.add(createMap("id", "2", "desc", "two", "$stopTransform", "true"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(DATA_CONFIG_FOR_SKIP_TRANSFORM);
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='1']");
+ assertQ(req("name_s:xyz"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testDeleteDocs() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ rows.add(createMap("id", "2", "desc", "two"));
+ rows.add(createMap("id", "3", "desc", "two", DocBuilder.DELETE_DOC_BY_ID, "2"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(DATA_CONFIG_FOR_SKIP_TRANSFORM);
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='0']");
+ assertQ(req("id:3"), "//*[@numFound='1']");
+
+ assertTrue("Update request processor processDelete was not called", TestUpdateRequestProcessor.processDeleteCalled);
+ assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
+
+ MockDataSource.clearCache();
+ rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ rows.add(createMap("id", "2", "desc", "one"));
+ rows.add(createMap("id", "3", "desc", "two", DocBuilder.DELETE_DOC_BY_QUERY, "desc:one"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(DATA_CONFIG_FOR_SKIP_TRANSFORM);
+
+ assertQ(req("id:1"), "//*[@numFound='0']");
+ assertQ(req("id:2"), "//*[@numFound='0']");
+ assertQ(req("id:3"), "//*[@numFound='1']");
+
+ assertTrue("Update request processor processDelete was not called", TestUpdateRequestProcessor.processDeleteCalled);
+ assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
+
+ MockDataSource.clearCache();
+ rows = new ArrayList();
+ rows.add(createMap(DocBuilder.DELETE_DOC_BY_ID, "3"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+ runFullImport(DATA_CONFIG_FOR_SKIP_TRANSFORM, createMap("clean","false"));
+ assertQ(req("id:3"), "//*[@numFound='0']");
+
+ assertTrue("Update request processor processDelete was not called", TestUpdateRequestProcessor.processDeleteCalled);
+ assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
+
+ }
+
+ @Test
+ @Ignore("Fix Me. See SOLR-4103.")
+ public void testFileListEntityProcessor_lastIndexTime() throws Exception {
+ File tmpdir = createTempDir().toFile();
+
+ Map<String, String> params = createMap("baseDir", tmpdir.getAbsolutePath());
+
+ createFile(tmpdir, "a.xml", "a.xml".getBytes(StandardCharsets.UTF_8), true);
+ createFile(tmpdir, "b.xml", "b.xml".getBytes(StandardCharsets.UTF_8), true);
+ createFile(tmpdir, "c.props", "c.props".getBytes(StandardCharsets.UTF_8), true);
+ runFullImport(DATA_CONFIG_FILE_LIST, params);
+ assertQ(req("*:*"), "//*[@numFound='3']");
+
+ // Add a new file after a full index is done
+ createFile(tmpdir, "t.xml", "t.xml".getBytes(StandardCharsets.UTF_8), false);
+ runFullImport(DATA_CONFIG_FILE_LIST, params);
+ // we should find only 1 because by default clean=true is passed
+ // and this particular import should find only one file t.xml
+ assertQ(req("*:*"), "//*[@numFound='1']");
+ }
+
+ public static class MockTransformer extends Transformer {
+ @Override
+ public Object transformRow(Map<String, Object> row, Context context) {
+ assertTrue("Context gave incorrect data source", context.getDataSource("mockDs") instanceof MockDataSource2);
+ return row;
+ }
+ }
+
+ public static class AddDynamicFieldTransformer extends Transformer {
+ @Override
+ public Object transformRow(Map<String, Object> row, Context context) {
+ // Add a dynamic field
+ row.put("dynamic_s", "test");
+ return row;
+ }
+ }
+
+ public static class ForcedExceptionTransformer extends Transformer {
+ @Override
+ public Object transformRow(Map<String, Object> row, Context context) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "ForcedException");
+ }
+ }
+
+ public static class MockDataSource2 extends MockDataSource {
+
+ }
+
+ public static class StartEventListener implements EventListener {
+ public static boolean executed = false;
+
+ @Override
+ public void onEvent(Context ctx) {
+ executed = true;
+ }
+ }
+
+ public static class EndEventListener implements EventListener {
+ public static boolean executed = false;
+
+ @Override
+ public void onEvent(Context ctx) {
+ executed = true;
+ }
+ }
+
+ public static class ErrorEventListener implements EventListener {
+ public static boolean executed = false;
+ public static Exception lastException = null;
+
+ @Override
+ public void onEvent(Context ctx) {
+ executed = true;
+ lastException = ((ContextImpl) ctx).getLastException();
+ }
+ }
+
+ private static final String REQUEST_PARAM_AS_VARIABLE = "<dataConfig>\n" +
+ " <dataSource type=\"MockDataSource\" />\n" +
+ " <document>\n" +
+ " <entity name=\"books\" query=\"select * from books where category='${dataimporter.request.category}'\">\n" +
+ " <field column=\"id\" />\n" +
+ " <field column=\"desc\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private static final String DATA_CONFIG_WITH_DYNAMIC_TRANSFORMER = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
+ " <document>\n" +
+ " <entity name=\"books\" query=\"select * from x\"" +
+ " transformer=\"TestDocBuilder2$AddDynamicFieldTransformer\">\n" +
+ " <field column=\"id\" />\n" +
+ " <field column=\"desc\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private static final String DATA_CONFIG_FOR_SKIP_TRANSFORM = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
+ " <document>\n" +
+ " <entity name=\"books\" query=\"select * from x\"" +
+ " transformer=\"TemplateTransformer\">\n" +
+ " <field column=\"id\" />\n" +
+ " <field column=\"desc\" />\n" +
+ " <field column=\"name_s\" template=\"xyz\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private static final String DATA_CONFIG_WITH_TWO_ENTITIES = "<dataConfig><dataSource type=\"MockDataSource\"/>\n" +
+ " <document>\n" +
+ " <entity name=\"books\" query=\"select * from x\">" +
+ " <field column=\"id\" />\n" +
+ " <field column=\"desc\" />\n" +
+ " <entity name=\"authors\" query=\"${books.id}\">" +
+ " <field column=\"name_s\" />" +
+ " </entity>" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private static final String DATA_CONFIG_WITH_CASE_INSENSITIVE_FIELDS = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
+ " <document onImportStart=\"TestDocBuilder2$StartEventListener\" onImportEnd=\"TestDocBuilder2$EndEventListener\">\n" +
+ " <entity name=\"books\" query=\"select * from x\">\n" +
+ " <field column=\"ID\" />\n" +
+ " <field column=\"Desc\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private static final String DATA_CONFIG_WITH_ERROR_HANDLER = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
+ " <document onError=\"TestDocBuilder2$ErrorEventListener\">\n" +
+ " <entity name=\"books\" query=\"select * from x\" transformer=\"TestDocBuilder2$ForcedExceptionTransformer\">\n" +
+ " <field column=\"id\" />\n" +
+ " <field column=\"FORCE_ERROR\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private static final String DATA_CONFIG_WITH_TEMPLATIZED_FIELD_NAMES = "<dataConfig><dataSource type=\"MockDataSource\"/>\n" +
+ " <document>\n" +
+ " <entity name=\"books\" query=\"select * from x\">\n" +
+ " <field column=\"mypk\" name=\"${dih.request.mypk}\" />\n" +
+ " <field column=\"text\" name=\"${dih.request.text}\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private static final String DATA_CONFIG_WITH_DYNAMIC_FIELD_NAMES = "<dataConfig><dataSource type=\"MockDataSource\"/>\n" +
+ " <document>\n" +
+ " <entity name=\"books\" query=\"select * from x\">\n" +
+ " <field column=\"mypk\" name=\"id\" />\n" +
+ " <field column=\"text\" name=\"${books.mypk}_s\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private static final String DATA_CONFIG_FILE_LIST = "<dataConfig>\n" +
+ "\t<document>\n" +
+ "\t\t<entity name=\"x\" processor=\"FileListEntityProcessor\" \n" +
+ "\t\t\t\tfileName=\".*\" newerThan=\"${dih.last_index_time}\" \n" +
+ "\t\t\t\tbaseDir=\"${dih.request.baseDir}\" transformer=\"TemplateTransformer\">\n" +
+ "\t\t\t<field column=\"id\" template=\"${x.file}\" />\n" +
+ "\t\t</entity>\n" +
+ "\t</document>\n" +
+ "</dataConfig>";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java
new file mode 100644
index 0000000..623e49e
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * <p>
+ * Test for EntityProcessorBase
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestEntityProcessorBase extends AbstractDataImportHandlerTestCase {
+
+ @Test
+ public void multiTransformer() {
+ List<Map<String, String>> fields = new ArrayList<>();
+ Map<String, String> entity = new HashMap<>();
+ entity.put("transformer", T1.class.getName() + "," + T2.class.getName()
+ + "," + T3.class.getName());
+ fields.add(getField("A", null, null, null, null));
+ fields.add(getField("B", null, null, null, null));
+
+ Context context = getContext(null, null, new MockDataSource(), Context.FULL_DUMP,
+ fields, entity);
+ Map<String, Object> src = new HashMap<>();
+ src.put("A", "NA");
+ src.put("B", "NA");
+ EntityProcessorWrapper sep = new EntityProcessorWrapper(new SqlEntityProcessor(), null, null);
+ sep.init(context);
+ Map<String, Object> res = sep.applyTransformer(src);
+ assertNotNull(res.get("T1"));
+ assertNotNull(res.get("T2"));
+ assertNotNull(res.get("T3"));
+ }
+
+ static class T1 extends Transformer {
+
+ @Override
+ public Object transformRow(Map<String, Object> aRow, Context context) {
+ aRow.put("T1", "T1 called");
+ return aRow;
+
+ }
+ }
+
+ static class T2 extends Transformer {
+
+ @Override
+ public Object transformRow(Map<String, Object> aRow, Context context) {
+ aRow.put("T2", "T2 called");
+ return aRow;
+ }
+ }
+
+ static class T3 {
+
+ public Object transformRow(Map<String, Object> aRow) {
+ aRow.put("T3", "T3 called");
+ return aRow;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEphemeralCache.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEphemeralCache.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEphemeralCache.java
new file mode 100644
index 0000000..ddccafe
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEphemeralCache.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.*;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestEphemeralCache extends AbstractDataImportHandlerTestCase {
+
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
+ }
+
+ @Before
+ public void reset() {
+ DestroyCountCache.destroyed.clear();
+ setupMockData();
+ }
+
+ @Test
+ public void test() throws Exception {
+ assertFullImport(getDataConfigDotXml());
+ }
+
+ @SuppressWarnings("unchecked")
+ private void setupMockData() {
+ List parentRows = new ArrayList();
+ parentRows.add(createMap("id", new BigDecimal("1"), "parent_s", "one"));
+ parentRows.add(createMap("id", new BigDecimal("2"), "parent_s", "two"));
+ parentRows.add(createMap("id", new BigDecimal("3"), "parent_s", "three"));
+ parentRows.add(createMap("id", new BigDecimal("4"), "parent_s", "four"));
+ parentRows.add(createMap("id", new BigDecimal("5"), "parent_s", "five"));
+
+ List child1Rows = new ArrayList();
+ child1Rows.add(createMap("id", new BigDecimal("6"), "child1a_mult_s", "this is the number six."));
+ child1Rows.add(createMap("id", new BigDecimal("5"), "child1a_mult_s", "this is the number five."));
+ child1Rows.add(createMap("id", new BigDecimal("6"), "child1a_mult_s", "let's sing a song of six."));
+ child1Rows.add(createMap("id", new BigDecimal("3"), "child1a_mult_s", "three"));
+ child1Rows.add(createMap("id", new BigDecimal("3"), "child1a_mult_s", "III"));
+ child1Rows.add(createMap("id", new BigDecimal("3"), "child1a_mult_s", "3"));
+ child1Rows.add(createMap("id", new BigDecimal("3"), "child1a_mult_s", "|||"));
+ child1Rows.add(createMap("id", new BigDecimal("1"), "child1a_mult_s", "one"));
+ child1Rows.add(createMap("id", new BigDecimal("1"), "child1a_mult_s", "uno"));
+ child1Rows.add(createMap("id", new BigDecimal("2"), "child1b_s", "CHILD1B", "child1a_mult_s", "this is the number two."));
+
+ List child2Rows = new ArrayList();
+ child2Rows.add(createMap("id", new BigDecimal("6"), "child2a_mult_s", "Child 2 says, 'this is the number six.'"));
+ child2Rows.add(createMap("id", new BigDecimal("5"), "child2a_mult_s", "Child 2 says, 'this is the number five.'"));
+ child2Rows.add(createMap("id", new BigDecimal("6"), "child2a_mult_s", "Child 2 says, 'let's sing a song of six.'"));
+ child2Rows.add(createMap("id", new BigDecimal("3"), "child2a_mult_s", "Child 2 says, 'three'"));
+ child2Rows.add(createMap("id", new BigDecimal("3"), "child2a_mult_s", "Child 2 says, 'III'"));
+ child2Rows.add(createMap("id", new BigDecimal("3"), "child2b_s", "CHILD2B", "child2a_mult_s", "Child 2 says, '3'"));
+ child2Rows.add(createMap("id", new BigDecimal("3"), "child2a_mult_s", "Child 2 says, '|||'"));
+ child2Rows.add(createMap("id", new BigDecimal("1"), "child2a_mult_s", "Child 2 says, 'one'"));
+ child2Rows.add(createMap("id", new BigDecimal("1"), "child2a_mult_s", "Child 2 says, 'uno'"));
+ child2Rows.add(createMap("id", new BigDecimal("2"), "child2a_mult_s", "Child 2 says, 'this is the number two.'"));
+
+ MockDataSource.setIterator("SELECT * FROM PARENT", parentRows.iterator());
+ MockDataSource.setIterator("SELECT * FROM CHILD_1", child1Rows.iterator());
+ MockDataSource.setIterator("SELECT * FROM CHILD_2", child2Rows.iterator());
+
+ }
+ private String getDataConfigDotXml() {
+ return
+ "<dataConfig>" +
+ " <dataSource type=\"MockDataSource\" />" +
+ " <document>" +
+ " <entity " +
+ " name=\"PARENT\"" +
+ " processor=\"SqlEntityProcessor\"" +
+ " cacheImpl=\"org.apache.solr.handler.dataimport.DestroyCountCache\"" +
+ " cacheName=\"PARENT\"" +
+ " query=\"SELECT * FROM PARENT\" " +
+ " >" +
+ " <entity" +
+ " name=\"CHILD_1\"" +
+ " processor=\"SqlEntityProcessor\"" +
+ " cacheImpl=\"org.apache.solr.handler.dataimport.DestroyCountCache\"" +
+ " cacheName=\"CHILD\"" +
+ " cacheKey=\"id\"" +
+ " cacheLookup=\"PARENT.id\"" +
+ " fieldNames=\"id, child1a_mult_s, child1b_s\"" +
+ " fieldTypes=\"BIGDECIMAL, STRING, STRING\"" +
+ " query=\"SELECT * FROM CHILD_1\" " +
+ " />" +
+ " <entity" +
+ " name=\"CHILD_2\"" +
+ " processor=\"SqlEntityProcessor\"" +
+ " cacheImpl=\"org.apache.solr.handler.dataimport.DestroyCountCache\"" +
+ " cacheKey=\"id\"" +
+ " cacheLookup=\"PARENT.id\"" +
+ " query=\"SELECT * FROM CHILD_2\" " +
+ " />" +
+ " </entity>" +
+ " </document>" +
+ "</dataConfig>"
+ ;
+ }
+
+ private void assertFullImport(String dataConfig) throws Exception {
+ runFullImport(dataConfig);
+
+ assertQ(req("*:*"), "//*[@numFound='5']");
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:6"), "//*[@numFound='0']");
+ assertQ(req("parent_s:four"), "//*[@numFound='1']");
+ assertQ(req("child1a_mult_s:this\\ is\\ the\\ numbe*"), "//*[@numFound='2']");
+ assertQ(req("child2a_mult_s:Child\\ 2\\ say*"), "//*[@numFound='4']");
+ assertQ(req("child1b_s:CHILD1B"), "//*[@numFound='1']");
+ assertQ(req("child2b_s:CHILD2B"), "//*[@numFound='1']");
+ assertQ(req("child1a_mult_s:one"), "//*[@numFound='1']");
+ assertQ(req("child1a_mult_s:uno"), "//*[@numFound='1']");
+ assertQ(req("child1a_mult_s:(uno OR one)"), "//*[@numFound='1']");
+
+ assertThat(DestroyCountCache.destroyed.size(), is(3));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestErrorHandling.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestErrorHandling.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestErrorHandling.java
new file mode 100644
index 0000000..1ea1ad4
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestErrorHandling.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+/**
+ * Tests exception handling during imports in DataImportHandler
+ *
+ *
+ * @since solr 1.4
+ */
+public class TestErrorHandling extends AbstractDataImportHandlerTestCase {
+
+ //TODO: fix this test to not require FSDirectory.
+ static String savedFactory;
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ savedFactory = System.getProperty("solr.DirectoryFactory");
+ System.setProperty("solr.directoryFactory", "solr.MockFSDirectoryFactory");
+ initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
+ ignoreException("Unexpected close tag");
+ }
+
+ @AfterClass
+ public static void afterClass() {
+ if (savedFactory == null) {
+ System.clearProperty("solr.directoryFactory");
+ } else {
+ System.setProperty("solr.directoryFactory", savedFactory);
+ }
+ }
+
+ @Before @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ clearIndex();
+ assertU(commit());
+ }
+
+ public void testMalformedStreamingXml() throws Exception {
+ StringDataSource.xml = malformedXml;
+ runFullImport(dataConfigWithStreaming);
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='1']");
+ }
+
+ public void testMalformedNonStreamingXml() throws Exception {
+ StringDataSource.xml = malformedXml;
+ runFullImport(dataConfigWithoutStreaming);
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='1']");
+ }
+
+ public void testAbortOnError() throws Exception {
+ StringDataSource.xml = malformedXml;
+ runFullImport(dataConfigAbortOnError);
+ assertQ(req("*:*"), "//*[@numFound='0']");
+ }
+
+ public void testTransformerErrorContinue() throws Exception {
+ StringDataSource.xml = wellformedXml;
+ List<Map<String, Object>> rows = new ArrayList<>();
+ rows.add(createMap("id", "3", "desc", "exception-transformer"));
+ MockDataSource.setIterator("select * from foo", rows.iterator());
+ runFullImport(dataConfigWithTransformer);
+ assertQ(req("*:*"), "//*[@numFound='3']");
+ }
+
+ public void testExternalEntity() throws Exception {
+ StringDataSource.xml = wellformedXml;
+ // This should not fail as external entities are replaced by an empty string during parsing:
+ runFullImport(dataConfigWithEntity);
+ assertQ(req("*:*"), "//*[@numFound='3']");
+ }
+
+ public static class StringDataSource extends DataSource<Reader> {
+ public static String xml = "";
+
+ @Override
+ public void init(Context context, Properties initProps) {
+ }
+
+ @Override
+ public Reader getData(String query) {
+ return new StringReader(xml);
+ }
+
+ @Override
+ public void close() {
+
+ }
+ }
+
+ public static class ExceptionTransformer extends Transformer {
+ @Override
+ public Object transformRow(Map<String, Object> row, Context context) {
+ throw new RuntimeException("Test exception");
+ }
+ }
+
+ private String dataConfigWithStreaming = "<dataConfig>\n" +
+ " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
+ " <document>\n" +
+ " <entity name=\"node\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" stream=\"true\" forEach=\"/root/node\" onError=\"skip\">\n" +
+ " <field column=\"id\" xpath=\"/root/node/id\" />\n" +
+ " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private String dataConfigWithoutStreaming = "<dataConfig>\n" +
+ " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
+ " <document>\n" +
+ " <entity name=\"node\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" forEach=\"/root/node\" onError=\"skip\">\n" +
+ " <field column=\"id\" xpath=\"/root/node/id\" />\n" +
+ " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private String dataConfigAbortOnError = "<dataConfig>\n" +
+ " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
+ " <document>\n" +
+ " <entity name=\"node\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" forEach=\"/root/node\" onError=\"abort\">\n" +
+ " <field column=\"id\" xpath=\"/root/node/id\" />\n" +
+ " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private String dataConfigWithTransformer = "<dataConfig>\n" +
+ " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
+ "<dataSource type=\"MockDataSource\"/>" +
+ " <document>\n" +
+ " <entity name=\"node\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" forEach=\"/root/node\">\n" +
+ " <field column=\"id\" xpath=\"/root/node/id\" />\n" +
+ " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
+ " <entity name=\"child\" query=\"select * from foo\" transformer=\"TestErrorHandling$ExceptionTransformer\" onError=\"continue\">\n" +
+ " </entity>" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private String dataConfigWithEntity = "<!DOCTYPE dataConfig [\n" +
+ " <!ENTITY internalTerm \"node\">\n" +
+ " <!ENTITY externalTerm SYSTEM \"foo://bar.xyz/external\">\n" +
+ "]><dataConfig>\n" +
+ " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
+ " <document>\n" +
+ " <entity name=\"&internalTerm;\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" forEach=\"/root/node\" onError=\"skip\">\n" +
+ " <field column=\"id\" xpath=\"/root/node/id\">&externalTerm;</field>\n" +
+ " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private String malformedXml = "<root>\n" +
+ " <node>\n" +
+ " <id>1</id>\n" +
+ " <desc>test1</desc>\n" +
+ " </node>\n" +
+ " <node>\n" +
+ " <id>2</id>\n" +
+ " <desc>test2</desc>\n" +
+ " </node>\n" +
+ " <node>\n" +
+ " <id/>3</id>\n" +
+ " <desc>test3</desc>\n" +
+ " </node>\n" +
+ "</root>";
+
+ private String wellformedXml = "<root>\n" +
+ " <node>\n" +
+ " <id>1</id>\n" +
+ " <desc>test1</desc>\n" +
+ " </node>\n" +
+ " <node>\n" +
+ " <id>2</id>\n" +
+ " <desc>test2</desc>\n" +
+ " </node>\n" +
+ " <node>\n" +
+ " <id>3</id>\n" +
+ " <desc>test3</desc>\n" +
+ " </node>\n" +
+ "</root>";
+}
[21/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugLogger.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugLogger.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugLogger.java
deleted file mode 100644
index 2fd9303..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugLogger.java
+++ /dev/null
@@ -1,285 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-import org.apache.solr.common.util.NamedList;
-
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.text.MessageFormat;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Stack;
-
-/**
- * <p>
- * Implements most of the interactive development functionality
- * </p>
- * <p/>
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
- * <p/>
- * <b>This API is experimental and subject to change</b>
- *
- * @since solr 1.3
- */
-class DebugLogger {
- private Stack<DebugInfo> debugStack;
-
- NamedList output;
-// private final SolrWriter writer1;
-
- private static final String LINE = "---------------------------------------------";
-
- private MessageFormat fmt = new MessageFormat(
- "----------- row #{0}-------------", Locale.ROOT);
-
- boolean enabled = true;
-
- public DebugLogger() {
-// writer = solrWriter;
- output = new NamedList();
- debugStack = new Stack<DebugInfo>() {
-
- @Override
- public DebugInfo pop() {
- if (size() == 1)
- throw new DataImportHandlerException(
- DataImportHandlerException.SEVERE, "Stack is becoming empty");
- return super.pop();
- }
- };
- debugStack.push(new DebugInfo(null, DIHLogLevels.NONE, null));
- output = debugStack.peek().lst;
- }
-
- private DebugInfo peekStack() {
- return debugStack.isEmpty() ? null : debugStack.peek();
- }
-
- public void log(DIHLogLevels event, String name, Object row) {
- if (event == DIHLogLevels.DISABLE_LOGGING) {
- enabled = false;
- return;
- } else if (event == DIHLogLevels.ENABLE_LOGGING) {
- enabled = true;
- return;
- }
-
- if (!enabled && event != DIHLogLevels.START_ENTITY
- && event != DIHLogLevels.END_ENTITY) {
- return;
- }
-
- if (event == DIHLogLevels.START_DOC) {
- debugStack.push(new DebugInfo(null, DIHLogLevels.START_DOC, peekStack()));
- } else if (DIHLogLevels.START_ENTITY == event) {
- debugStack
- .push(new DebugInfo(name, DIHLogLevels.START_ENTITY, peekStack()));
- } else if (DIHLogLevels.ENTITY_OUT == event
- || DIHLogLevels.PRE_TRANSFORMER_ROW == event) {
- if (debugStack.peek().type == DIHLogLevels.START_ENTITY
- || debugStack.peek().type == DIHLogLevels.START_DOC) {
- debugStack.peek().lst.add(null, fmt.format(new Object[]{++debugStack
- .peek().rowCount}));
- addToNamedList(debugStack.peek().lst, row);
- debugStack.peek().lst.add(null, LINE);
- }
- } else if (event == DIHLogLevels.ROW_END) {
- popAllTransformers();
- } else if (DIHLogLevels.END_ENTITY == event) {
- while (debugStack.pop().type != DIHLogLevels.START_ENTITY)
- ;
- } else if (DIHLogLevels.END_DOC == event) {
- while (debugStack.pop().type != DIHLogLevels.START_DOC)
- ;
- } else if (event == DIHLogLevels.TRANSFORMER_EXCEPTION) {
- debugStack.push(new DebugInfo(name, event, peekStack()));
- debugStack.peek().lst.add("EXCEPTION",
- getStacktraceString((Exception) row));
- } else if (DIHLogLevels.TRANSFORMED_ROW == event) {
- debugStack.push(new DebugInfo(name, event, peekStack()));
- debugStack.peek().lst.add(null, LINE);
- addToNamedList(debugStack.peek().lst, row);
- debugStack.peek().lst.add(null, LINE);
- if (row instanceof DataImportHandlerException) {
- DataImportHandlerException dataImportHandlerException = (DataImportHandlerException) row;
- dataImportHandlerException.debugged = true;
- }
- } else if (DIHLogLevels.ENTITY_META == event) {
- popAllTransformers();
- debugStack.peek().lst.add(name, row);
- } else if (DIHLogLevels.ENTITY_EXCEPTION == event) {
- if (row instanceof DataImportHandlerException) {
- DataImportHandlerException dihe = (DataImportHandlerException) row;
- if (dihe.debugged)
- return;
- dihe.debugged = true;
- }
-
- popAllTransformers();
- debugStack.peek().lst.add("EXCEPTION",
- getStacktraceString((Exception) row));
- }
- }
-
- private void popAllTransformers() {
- while (true) {
- DIHLogLevels type = debugStack.peek().type;
- if (type == DIHLogLevels.START_DOC || type == DIHLogLevels.START_ENTITY)
- break;
- debugStack.pop();
- }
- }
-
- private void addToNamedList(NamedList nl, Object row) {
- if (row instanceof List) {
- List list = (List) row;
- NamedList l = new NamedList();
- nl.add(null, l);
- for (Object o : list) {
- Map<String, Object> map = (Map<String, Object>) o;
- for (Map.Entry<String, Object> entry : map.entrySet())
- nl.add(entry.getKey(), entry.getValue());
- }
- } else if (row instanceof Map) {
- Map<String, Object> map = (Map<String, Object>) row;
- for (Map.Entry<String, Object> entry : map.entrySet())
- nl.add(entry.getKey(), entry.getValue());
- }
- }
-
- DataSource wrapDs(final DataSource ds) {
- return new DataSource() {
- @Override
- public void init(Context context, Properties initProps) {
- ds.init(context, initProps);
- }
-
- @Override
- public void close() {
- ds.close();
- }
-
- @Override
- public Object getData(String query) {
- log(DIHLogLevels.ENTITY_META, "query", query);
- long start = System.nanoTime();
- try {
- return ds.getData(query);
- } catch (DataImportHandlerException de) {
- log(DIHLogLevels.ENTITY_EXCEPTION,
- null, de);
- throw de;
- } catch (Exception e) {
- log(DIHLogLevels.ENTITY_EXCEPTION,
- null, e);
- DataImportHandlerException de = new DataImportHandlerException(
- DataImportHandlerException.SEVERE, "", e);
- de.debugged = true;
- throw de;
- } finally {
- log(DIHLogLevels.ENTITY_META, "time-taken", DocBuilder
- .getTimeElapsedSince(start));
- }
- }
- };
- }
-
- Transformer wrapTransformer(final Transformer t) {
- return new Transformer() {
- @Override
- public Object transformRow(Map<String, Object> row, Context context) {
- log(DIHLogLevels.PRE_TRANSFORMER_ROW, null, row);
- String tName = getTransformerName(t);
- Object result = null;
- try {
- result = t.transformRow(row, context);
- log(DIHLogLevels.TRANSFORMED_ROW, tName, result);
- } catch (DataImportHandlerException de) {
- log(DIHLogLevels.TRANSFORMER_EXCEPTION, tName, de);
- de.debugged = true;
- throw de;
- } catch (Exception e) {
- log(DIHLogLevels.TRANSFORMER_EXCEPTION, tName, e);
- DataImportHandlerException de = new DataImportHandlerException(DataImportHandlerException.SEVERE, "", e);
- de.debugged = true;
- throw de;
- }
- return result;
- }
- };
- }
-
- public static String getStacktraceString(Exception e) {
- StringWriter sw = new StringWriter();
- e.printStackTrace(new PrintWriter(sw));
- return sw.toString();
- }
-
- static String getTransformerName(Transformer t) {
- Class transClass = t.getClass();
- if (t instanceof EntityProcessorWrapper.ReflectionTransformer) {
- return ((EntityProcessorWrapper.ReflectionTransformer) t).trans;
- }
- if (t instanceof ScriptTransformer) {
- ScriptTransformer scriptTransformer = (ScriptTransformer) t;
- return "script:" + scriptTransformer.getFunctionName();
- }
- if (transClass.getPackage().equals(DebugLogger.class.getPackage())) {
- return transClass.getSimpleName();
- } else {
- return transClass.getName();
- }
- }
-
- private static class DebugInfo {
- String name;
-
- int tCount, rowCount;
-
- NamedList lst;
-
- DIHLogLevels type;
-
- DebugInfo parent;
-
- public DebugInfo(String name, DIHLogLevels type, DebugInfo parent) {
- this.name = name;
- this.type = type;
- this.parent = parent;
- lst = new NamedList();
- if (parent != null) {
- String displayName = null;
- if (type == DIHLogLevels.START_ENTITY) {
- displayName = "entity:" + name;
- } else if (type == DIHLogLevels.TRANSFORMED_ROW
- || type == DIHLogLevels.TRANSFORMER_EXCEPTION) {
- displayName = "transformer:" + name;
- } else if (type == DIHLogLevels.START_DOC) {
- this.name = displayName = "document#" + SolrWriter.getDocCount();
- }
- parent.lst.add(displayName, lst);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
deleted file mode 100644
index 164cf70..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
+++ /dev/null
@@ -1,988 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.handler.dataimport.config.ConfigNameConstants;
-import org.apache.solr.handler.dataimport.config.DIHConfiguration;
-import org.apache.solr.handler.dataimport.config.Entity;
-import org.apache.solr.handler.dataimport.config.EntityField;
-
-import static org.apache.solr.handler.dataimport.SolrWriter.LAST_INDEX_KEY;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-
-import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.schema.SchemaField;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.invoke.MethodHandles;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * <p> {@link DocBuilder} is responsible for creating Solr documents out of the given configuration. It also maintains
- * statistics information. It depends on the {@link EntityProcessor} implementations to fetch data. </p>
- * <p>
- * <b>This API is experimental and subject to change</b>
- *
- * @since solr 1.3
- */
-public class DocBuilder {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private static final AtomicBoolean WARNED_ABOUT_INDEX_TIME_BOOSTS = new AtomicBoolean();
-
- private static final Date EPOCH = new Date(0);
- public static final String DELETE_DOC_BY_ID = "$deleteDocById";
- public static final String DELETE_DOC_BY_QUERY = "$deleteDocByQuery";
- public static final String DOC_BOOST = "$docBoost";
- public static final String SKIP_DOC = "$skipDoc";
- public static final String SKIP_ROW = "$skipRow";
-
- DataImporter dataImporter;
-
- private DIHConfiguration config;
-
- private EntityProcessorWrapper currentEntityProcessorWrapper;
-
- @SuppressWarnings("unchecked")
- private Map statusMessages = Collections.synchronizedMap(new LinkedHashMap());
-
- public Statistics importStatistics = new Statistics();
-
- DIHWriter writer;
-
- boolean verboseDebug = false;
-
- Map<String, Object> session = new HashMap<>();
-
- static final ThreadLocal<DocBuilder> INSTANCE = new ThreadLocal<>();
- private Map<String, Object> persistedProperties;
-
- private DIHProperties propWriter;
- private DebugLogger debugLogger;
- private final RequestInfo reqParams;
-
- public DocBuilder(DataImporter dataImporter, DIHWriter solrWriter, DIHProperties propWriter, RequestInfo reqParams) {
- INSTANCE.set(this);
- this.dataImporter = dataImporter;
- this.reqParams = reqParams;
- this.propWriter = propWriter;
- DataImporter.QUERY_COUNT.set(importStatistics.queryCount);
- verboseDebug = reqParams.isDebug() && reqParams.getDebugInfo().verbose;
- persistedProperties = propWriter.readIndexerProperties();
-
- writer = solrWriter;
- ContextImpl ctx = new ContextImpl(null, null, null, null, reqParams.getRawParams(), null, this);
- if (writer != null) {
- writer.init(ctx);
- }
- }
-
-
- DebugLogger getDebugLogger(){
- if (debugLogger == null) {
- debugLogger = new DebugLogger();
- }
- return debugLogger;
- }
-
- private VariableResolver getVariableResolver() {
- try {
- VariableResolver resolver = null;
- String epoch = propWriter.convertDateToString(EPOCH);
- if(dataImporter != null && dataImporter.getCore() != null
- && dataImporter.getCore().getResourceLoader().getCoreProperties() != null){
- resolver = new VariableResolver(dataImporter.getCore().getResourceLoader().getCoreProperties());
- } else {
- resolver = new VariableResolver();
- }
- resolver.setEvaluators(dataImporter.getEvaluators());
- Map<String, Object> indexerNamespace = new HashMap<>();
- if (persistedProperties.get(LAST_INDEX_TIME) != null) {
- indexerNamespace.put(LAST_INDEX_TIME, persistedProperties.get(LAST_INDEX_TIME));
- } else {
- // set epoch
- indexerNamespace.put(LAST_INDEX_TIME, epoch);
- }
- indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime());
- indexerNamespace.put("request", new HashMap<>(reqParams.getRawParams()));
- indexerNamespace.put("handlerName", dataImporter.getHandlerName());
- for (Entity entity : dataImporter.getConfig().getEntities()) {
- Map<String, Object> entityNamespace = new HashMap<>();
- String key = SolrWriter.LAST_INDEX_KEY;
- Object lastIndex = persistedProperties.get(entity.getName() + "." + key);
- if (lastIndex != null) {
- entityNamespace.put(SolrWriter.LAST_INDEX_KEY, lastIndex);
- } else {
- entityNamespace.put(SolrWriter.LAST_INDEX_KEY, epoch);
- }
- indexerNamespace.put(entity.getName(), entityNamespace);
- }
- resolver.addNamespace(ConfigNameConstants.IMPORTER_NS_SHORT, indexerNamespace);
- resolver.addNamespace(ConfigNameConstants.IMPORTER_NS, indexerNamespace);
- return resolver;
- } catch (Exception e) {
- wrapAndThrow(SEVERE, e);
- // unreachable statement
- return null;
- }
- }
-
- private void invokeEventListener(String className) {
- invokeEventListener(className, null);
- }
-
-
- private void invokeEventListener(String className, Exception lastException) {
- try {
- EventListener listener = (EventListener) loadClass(className, dataImporter.getCore()).newInstance();
- notifyListener(listener, lastException);
- } catch (Exception e) {
- wrapAndThrow(SEVERE, e, "Unable to load class : " + className);
- }
- }
-
- private void notifyListener(EventListener listener, Exception lastException) {
- String currentProcess;
- if (dataImporter.getStatus() == DataImporter.Status.RUNNING_DELTA_DUMP) {
- currentProcess = Context.DELTA_DUMP;
- } else {
- currentProcess = Context.FULL_DUMP;
- }
- ContextImpl ctx = new ContextImpl(null, getVariableResolver(), null, currentProcess, session, null, this);
- ctx.setLastException(lastException);
- listener.onEvent(ctx);
- }
-
- @SuppressWarnings("unchecked")
- public void execute() {
- List<EntityProcessorWrapper> epwList = null;
- try {
- dataImporter.store(DataImporter.STATUS_MSGS, statusMessages);
- config = dataImporter.getConfig();
- final AtomicLong startTime = new AtomicLong(System.nanoTime());
- statusMessages.put(TIME_ELAPSED, new Object() {
- @Override
- public String toString() {
- return getTimeElapsedSince(startTime.get());
- }
- });
-
- statusMessages.put(DataImporter.MSG.TOTAL_QUERIES_EXECUTED,
- importStatistics.queryCount);
- statusMessages.put(DataImporter.MSG.TOTAL_ROWS_EXECUTED,
- importStatistics.rowsCount);
- statusMessages.put(DataImporter.MSG.TOTAL_DOC_PROCESSED,
- importStatistics.docCount);
- statusMessages.put(DataImporter.MSG.TOTAL_DOCS_SKIPPED,
- importStatistics.skipDocCount);
-
- List<String> entities = reqParams.getEntitiesToRun();
-
- // Trigger onImportStart
- if (config.getOnImportStart() != null) {
- invokeEventListener(config.getOnImportStart());
- }
- AtomicBoolean fullCleanDone = new AtomicBoolean(false);
- //we must not do a delete of *:* multiple times if there are multiple root entities to be run
- Map<String,Object> lastIndexTimeProps = new HashMap<>();
- lastIndexTimeProps.put(LAST_INDEX_KEY, dataImporter.getIndexStartTime());
-
- epwList = new ArrayList<>(config.getEntities().size());
- for (Entity e : config.getEntities()) {
- epwList.add(getEntityProcessorWrapper(e));
- }
- for (EntityProcessorWrapper epw : epwList) {
- if (entities != null && !entities.contains(epw.getEntity().getName()))
- continue;
- lastIndexTimeProps.put(epw.getEntity().getName() + "." + LAST_INDEX_KEY, propWriter.getCurrentTimestamp());
- currentEntityProcessorWrapper = epw;
- String delQuery = epw.getEntity().getAllAttributes().get("preImportDeleteQuery");
- if (dataImporter.getStatus() == DataImporter.Status.RUNNING_DELTA_DUMP) {
- cleanByQuery(delQuery, fullCleanDone);
- doDelta();
- delQuery = epw.getEntity().getAllAttributes().get("postImportDeleteQuery");
- if (delQuery != null) {
- fullCleanDone.set(false);
- cleanByQuery(delQuery, fullCleanDone);
- }
- } else {
- cleanByQuery(delQuery, fullCleanDone);
- doFullDump();
- delQuery = epw.getEntity().getAllAttributes().get("postImportDeleteQuery");
- if (delQuery != null) {
- fullCleanDone.set(false);
- cleanByQuery(delQuery, fullCleanDone);
- }
- }
- }
-
- if (stop.get()) {
- // Dont commit if aborted using command=abort
- statusMessages.put("Aborted", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).format(new Date()));
- handleError("Aborted", null);
- } else {
- // Do not commit unnecessarily if this is a delta-import and no documents were created or deleted
- if (!reqParams.isClean()) {
- if (importStatistics.docCount.get() > 0 || importStatistics.deletedDocCount.get() > 0) {
- finish(lastIndexTimeProps);
- }
- } else {
- // Finished operation normally, commit now
- finish(lastIndexTimeProps);
- }
-
- if (config.getOnImportEnd() != null) {
- invokeEventListener(config.getOnImportEnd());
- }
- }
-
- statusMessages.remove(TIME_ELAPSED);
- statusMessages.put(DataImporter.MSG.TOTAL_DOC_PROCESSED, ""+ importStatistics.docCount.get());
- if(importStatistics.failedDocCount.get() > 0)
- statusMessages.put(DataImporter.MSG.TOTAL_FAILED_DOCS, ""+ importStatistics.failedDocCount.get());
-
- statusMessages.put("Time taken", getTimeElapsedSince(startTime.get()));
- log.info("Time taken = " + getTimeElapsedSince(startTime.get()));
- } catch(Exception e)
- {
- throw new RuntimeException(e);
- } finally
- {
- if (writer != null) {
- writer.close();
- }
- if (epwList != null) {
- closeEntityProcessorWrappers(epwList);
- }
- if(reqParams.isDebug()) {
- reqParams.getDebugInfo().debugVerboseOutput = getDebugLogger().output;
- }
- }
- }
- private void closeEntityProcessorWrappers(List<EntityProcessorWrapper> epwList) {
- for(EntityProcessorWrapper epw : epwList) {
- epw.close();
- if(epw.getDatasource()!=null) {
- epw.getDatasource().close();
- }
- closeEntityProcessorWrappers(epw.getChildren());
- }
- }
-
- @SuppressWarnings("unchecked")
- private void finish(Map<String,Object> lastIndexTimeProps) {
- log.info("Import completed successfully");
- statusMessages.put("", "Indexing completed. Added/Updated: "
- + importStatistics.docCount + " documents. Deleted "
- + importStatistics.deletedDocCount + " documents.");
- if(reqParams.isCommit()) {
- writer.commit(reqParams.isOptimize());
- addStatusMessage("Committed");
- if (reqParams.isOptimize())
- addStatusMessage("Optimized");
- }
- try {
- propWriter.persist(lastIndexTimeProps);
- } catch (Exception e) {
- log.error("Could not write property file", e);
- statusMessages.put("error", "Could not write property file. Delta imports will not work. " +
- "Make sure your conf directory is writable");
- }
- }
-
- void handleError(String message, Exception e) {
- if (!dataImporter.getCore().getCoreContainer().isZooKeeperAware()) {
- writer.rollback();
- }
-
- statusMessages.put(message, "Indexing error");
- addStatusMessage(message);
- if ((config != null) && (config.getOnError() != null)) {
- invokeEventListener(config.getOnError(), e);
- }
- }
-
- private void doFullDump() {
- addStatusMessage("Full Dump Started");
- buildDocument(getVariableResolver(), null, null, currentEntityProcessorWrapper, true, null);
- }
-
- @SuppressWarnings("unchecked")
- private void doDelta() {
- addStatusMessage("Delta Dump started");
- VariableResolver resolver = getVariableResolver();
-
- if (config.getDeleteQuery() != null) {
- writer.deleteByQuery(config.getDeleteQuery());
- }
-
- addStatusMessage("Identifying Delta");
- log.info("Starting delta collection.");
- Set<Map<String, Object>> deletedKeys = new HashSet<>();
- Set<Map<String, Object>> allPks = collectDelta(currentEntityProcessorWrapper, resolver, deletedKeys);
- if (stop.get())
- return;
- addStatusMessage("Deltas Obtained");
- addStatusMessage("Building documents");
- if (!deletedKeys.isEmpty()) {
- allPks.removeAll(deletedKeys);
- deleteAll(deletedKeys);
- // Make sure that documents are not re-created
- }
- deletedKeys = null;
- writer.setDeltaKeys(allPks);
-
- statusMessages.put("Total Changed Documents", allPks.size());
- VariableResolver vri = getVariableResolver();
- Iterator<Map<String, Object>> pkIter = allPks.iterator();
- while (pkIter.hasNext()) {
- Map<String, Object> map = pkIter.next();
- vri.addNamespace(ConfigNameConstants.IMPORTER_NS_SHORT + ".delta", map);
- buildDocument(vri, null, map, currentEntityProcessorWrapper, true, null);
- pkIter.remove();
- // check for abort
- if (stop.get())
- break;
- }
-
- if (!stop.get()) {
- log.info("Delta Import completed successfully");
- }
- }
-
- private void deleteAll(Set<Map<String, Object>> deletedKeys) {
- log.info("Deleting stale documents ");
- Iterator<Map<String, Object>> iter = deletedKeys.iterator();
- while (iter.hasNext()) {
- Map<String, Object> map = iter.next();
- String keyName = currentEntityProcessorWrapper.getEntity().isDocRoot() ? currentEntityProcessorWrapper.getEntity().getPk() : currentEntityProcessorWrapper.getEntity().getSchemaPk();
- Object key = map.get(keyName);
- if(key == null) {
- keyName = findMatchingPkColumn(keyName, map);
- key = map.get(keyName);
- }
- if(key == null) {
- log.warn("no key was available for deleted pk query. keyName = " + keyName);
- continue;
- }
- writer.deleteDoc(key);
- importStatistics.deletedDocCount.incrementAndGet();
- iter.remove();
- }
- }
-
- @SuppressWarnings("unchecked")
- public void addStatusMessage(String msg) {
- statusMessages.put(msg, new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).format(new Date()));
- }
-
- private void resetEntity(EntityProcessorWrapper epw) {
- epw.setInitialized(false);
- for (EntityProcessorWrapper child : epw.getChildren()) {
- resetEntity(child);
- }
-
- }
-
- private void buildDocument(VariableResolver vr, DocWrapper doc,
- Map<String,Object> pk, EntityProcessorWrapper epw, boolean isRoot,
- ContextImpl parentCtx) {
- List<EntityProcessorWrapper> entitiesToDestroy = new ArrayList<>();
- try {
- buildDocument(vr, doc, pk, epw, isRoot, parentCtx, entitiesToDestroy);
- } catch (Exception e) {
- throw new RuntimeException(e);
- } finally {
- for (EntityProcessorWrapper entityWrapper : entitiesToDestroy) {
- entityWrapper.destroy();
- }
- resetEntity(epw);
- }
- }
-
- @SuppressWarnings("unchecked")
- private void buildDocument(VariableResolver vr, DocWrapper doc,
- Map<String, Object> pk, EntityProcessorWrapper epw, boolean isRoot,
- ContextImpl parentCtx, List<EntityProcessorWrapper> entitiesToDestroy) {
-
- ContextImpl ctx = new ContextImpl(epw, vr, null,
- pk == null ? Context.FULL_DUMP : Context.DELTA_DUMP,
- session, parentCtx, this);
- epw.init(ctx);
- if (!epw.isInitialized()) {
- entitiesToDestroy.add(epw);
- epw.setInitialized(true);
- }
-
- if (reqParams.getStart() > 0) {
- getDebugLogger().log(DIHLogLevels.DISABLE_LOGGING, null, null);
- }
-
- if (verboseDebug) {
- getDebugLogger().log(DIHLogLevels.START_ENTITY, epw.getEntity().getName(), null);
- }
-
- int seenDocCount = 0;
-
- try {
- while (true) {
- if (stop.get())
- return;
- if(importStatistics.docCount.get() > (reqParams.getStart() + reqParams.getRows())) break;
- try {
- seenDocCount++;
-
- if (seenDocCount > reqParams.getStart()) {
- getDebugLogger().log(DIHLogLevels.ENABLE_LOGGING, null, null);
- }
-
- if (verboseDebug && epw.getEntity().isDocRoot()) {
- getDebugLogger().log(DIHLogLevels.START_DOC, epw.getEntity().getName(), null);
- }
- if (doc == null && epw.getEntity().isDocRoot()) {
- doc = new DocWrapper();
- ctx.setDoc(doc);
- Entity e = epw.getEntity();
- while (e.getParentEntity() != null) {
- addFields(e.getParentEntity(), doc, (Map<String, Object>) vr
- .resolve(e.getParentEntity().getName()), vr);
- e = e.getParentEntity();
- }
- }
-
- Map<String, Object> arow = epw.nextRow();
- if (arow == null) {
- break;
- }
-
- // Support for start parameter in debug mode
- if (epw.getEntity().isDocRoot()) {
- if (seenDocCount <= reqParams.getStart())
- continue;
- if (seenDocCount > reqParams.getStart() + reqParams.getRows()) {
- log.info("Indexing stopped at docCount = " + importStatistics.docCount);
- break;
- }
- }
-
- if (verboseDebug) {
- getDebugLogger().log(DIHLogLevels.ENTITY_OUT, epw.getEntity().getName(), arow);
- }
- importStatistics.rowsCount.incrementAndGet();
-
- DocWrapper childDoc = null;
- if (doc != null) {
- if (epw.getEntity().isChild()) {
- childDoc = new DocWrapper();
- handleSpecialCommands(arow, childDoc);
- addFields(epw.getEntity(), childDoc, arow, vr);
- doc.addChildDocument(childDoc);
- } else {
- handleSpecialCommands(arow, doc);
- vr.addNamespace(epw.getEntity().getName(), arow);
- addFields(epw.getEntity(), doc, arow, vr);
- vr.removeNamespace(epw.getEntity().getName());
- }
- }
- if (epw.getEntity().getChildren() != null) {
- vr.addNamespace(epw.getEntity().getName(), arow);
- for (EntityProcessorWrapper child : epw.getChildren()) {
- if (childDoc != null) {
- buildDocument(vr, childDoc,
- child.getEntity().isDocRoot() ? pk : null, child, false, ctx, entitiesToDestroy);
- } else {
- buildDocument(vr, doc,
- child.getEntity().isDocRoot() ? pk : null, child, false, ctx, entitiesToDestroy);
- }
- }
- vr.removeNamespace(epw.getEntity().getName());
- }
- if (epw.getEntity().isDocRoot()) {
- if (stop.get())
- return;
- if (!doc.isEmpty()) {
- boolean result = writer.upload(doc);
- if(reqParams.isDebug()) {
- reqParams.getDebugInfo().debugDocuments.add(doc);
- }
- doc = null;
- if (result){
- importStatistics.docCount.incrementAndGet();
- } else {
- importStatistics.failedDocCount.incrementAndGet();
- }
- }
- }
- } catch (DataImportHandlerException e) {
- if (verboseDebug) {
- getDebugLogger().log(DIHLogLevels.ENTITY_EXCEPTION, epw.getEntity().getName(), e);
- }
- if(e.getErrCode() == DataImportHandlerException.SKIP_ROW){
- continue;
- }
- if (isRoot) {
- if (e.getErrCode() == DataImportHandlerException.SKIP) {
- importStatistics.skipDocCount.getAndIncrement();
- doc = null;
- } else {
- SolrException.log(log, "Exception while processing: "
- + epw.getEntity().getName() + " document : " + doc, e);
- }
- if (e.getErrCode() == DataImportHandlerException.SEVERE)
- throw e;
- } else
- throw e;
- } catch (Exception t) {
- if (verboseDebug) {
- getDebugLogger().log(DIHLogLevels.ENTITY_EXCEPTION, epw.getEntity().getName(), t);
- }
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE, t);
- } finally {
- if (verboseDebug) {
- getDebugLogger().log(DIHLogLevels.ROW_END, epw.getEntity().getName(), null);
- if (epw.getEntity().isDocRoot())
- getDebugLogger().log(DIHLogLevels.END_DOC, null, null);
- }
- }
- }
- } finally {
- if (verboseDebug) {
- getDebugLogger().log(DIHLogLevels.END_ENTITY, null, null);
- }
- }
- }
-
- static class DocWrapper extends SolrInputDocument {
- //final SolrInputDocument solrDocument = new SolrInputDocument();
- Map<String ,Object> session;
-
- public void setSessionAttribute(String key, Object val){
- if(session == null) session = new HashMap<>();
- session.put(key, val);
- }
-
- public Object getSessionAttribute(String key) {
- return session == null ? null : session.get(key);
- }
- }
-
- private void handleSpecialCommands(Map<String, Object> arow, DocWrapper doc) {
- Object value = arow.get(DELETE_DOC_BY_ID);
- if (value != null) {
- if (value instanceof Collection) {
- Collection collection = (Collection) value;
- for (Object o : collection) {
- writer.deleteDoc(o.toString());
- importStatistics.deletedDocCount.incrementAndGet();
- }
- } else {
- writer.deleteDoc(value);
- importStatistics.deletedDocCount.incrementAndGet();
- }
- }
- value = arow.get(DELETE_DOC_BY_QUERY);
- if (value != null) {
- if (value instanceof Collection) {
- Collection collection = (Collection) value;
- for (Object o : collection) {
- writer.deleteByQuery(o.toString());
- importStatistics.deletedDocCount.incrementAndGet();
- }
- } else {
- writer.deleteByQuery(value.toString());
- importStatistics.deletedDocCount.incrementAndGet();
- }
- }
- value = arow.get(DOC_BOOST);
- if (value != null) {
- String message = "Ignoring document boost: " + value + " as index-time boosts are not supported anymore";
- if (WARNED_ABOUT_INDEX_TIME_BOOSTS.compareAndSet(false, true)) {
- log.warn(message);
- } else {
- log.debug(message);
- }
- }
-
- value = arow.get(SKIP_DOC);
- if (value != null) {
- if (Boolean.parseBoolean(value.toString())) {
- throw new DataImportHandlerException(DataImportHandlerException.SKIP,
- "Document skipped :" + arow);
- }
- }
-
- value = arow.get(SKIP_ROW);
- if (value != null) {
- if (Boolean.parseBoolean(value.toString())) {
- throw new DataImportHandlerException(DataImportHandlerException.SKIP_ROW);
- }
- }
- }
-
- @SuppressWarnings("unchecked")
- private void addFields(Entity entity, DocWrapper doc,
- Map<String, Object> arow, VariableResolver vr) {
- for (Map.Entry<String, Object> entry : arow.entrySet()) {
- String key = entry.getKey();
- Object value = entry.getValue();
- if (value == null) continue;
- if (key.startsWith("$")) continue;
- Set<EntityField> field = entity.getColNameVsField().get(key);
- IndexSchema schema = null == reqParams.getRequest() ? null : reqParams.getRequest().getSchema();
- if (field == null && schema != null) {
- // This can be a dynamic field or a field which does not have an entry in data-config ( an implicit field)
- SchemaField sf = schema.getFieldOrNull(key);
- if (sf == null) {
- sf = config.getSchemaField(key);
- }
- if (sf != null) {
- addFieldToDoc(entry.getValue(), sf.getName(), sf.multiValued(), doc);
- }
- //else do nothing. if we add it it may fail
- } else {
- if (field != null) {
- for (EntityField f : field) {
- String name = f.getName();
- boolean multiValued = f.isMultiValued();
- boolean toWrite = f.isToWrite();
- if(f.isDynamicName()){
- name = vr.replaceTokens(name);
- SchemaField schemaField = config.getSchemaField(name);
- if(schemaField == null) {
- toWrite = false;
- } else {
- multiValued = schemaField.multiValued();
- toWrite = true;
- }
- }
- if (toWrite) {
- addFieldToDoc(entry.getValue(), name, multiValued, doc);
- }
- }
- }
- }
- }
- }
-
- private void addFieldToDoc(Object value, String name, boolean multiValued, DocWrapper doc) {
- if (value instanceof Collection) {
- Collection collection = (Collection) value;
- if (multiValued) {
- for (Object o : collection) {
- if (o != null)
- doc.addField(name, o);
- }
- } else {
- if (doc.getField(name) == null)
- for (Object o : collection) {
- if (o != null) {
- doc.addField(name, o);
- break;
- }
- }
- }
- } else if (multiValued) {
- if (value != null) {
- doc.addField(name, value);
- }
- } else {
- if (doc.getField(name) == null && value != null)
- doc.addField(name, value);
- }
- }
-
- public EntityProcessorWrapper getEntityProcessorWrapper(Entity entity) {
- EntityProcessor entityProcessor = null;
- if (entity.getProcessorName() == null) {
- entityProcessor = new SqlEntityProcessor();
- } else {
- try {
- entityProcessor = (EntityProcessor) loadClass(entity.getProcessorName(), dataImporter.getCore())
- .newInstance();
- } catch (Exception e) {
- wrapAndThrow (SEVERE,e,
- "Unable to load EntityProcessor implementation for entity:" + entity.getName());
- }
- }
- EntityProcessorWrapper epw = new EntityProcessorWrapper(entityProcessor, entity, this);
- for(Entity e1 : entity.getChildren()) {
- epw.getChildren().add(getEntityProcessorWrapper(e1));
- }
-
- return epw;
- }
-
- private String findMatchingPkColumn(String pk, Map<String, Object> row) {
- if (row.containsKey(pk)) {
- throw new IllegalArgumentException(String.format(Locale.ROOT,
- "deltaQuery returned a row with null for primary key %s", pk));
- }
- String resolvedPk = null;
- for (String columnName : row.keySet()) {
- if (columnName.endsWith("." + pk) || pk.endsWith("." + columnName)) {
- if (resolvedPk != null)
- throw new IllegalArgumentException(
- String.format(Locale.ROOT,
- "deltaQuery has more than one column (%s and %s) that might resolve to declared primary key pk='%s'",
- resolvedPk, columnName, pk));
- resolvedPk = columnName;
- }
- }
- if (resolvedPk == null) {
- throw new IllegalArgumentException(
- String
- .format(
- Locale.ROOT,
- "deltaQuery has no column to resolve to declared primary key pk='%s'",
- pk));
- }
- log.info(String.format(Locale.ROOT,
- "Resolving deltaQuery column '%s' to match entity's declared pk '%s'",
- resolvedPk, pk));
- return resolvedPk;
- }
-
- /**
- * <p> Collects unique keys of all Solr documents for whom one or more source tables have been changed since the last
- * indexed time. </p> <p> Note: In our definition, unique key of Solr document is the primary key of the top level
- * entity (unless skipped using docRoot=false) in the Solr document in data-config.xml </p>
- *
- * @return an iterator to the list of keys for which Solr documents should be updated.
- */
- @SuppressWarnings("unchecked")
- public Set<Map<String, Object>> collectDelta(EntityProcessorWrapper epw, VariableResolver resolver,
- Set<Map<String, Object>> deletedRows) {
- //someone called abort
- if (stop.get())
- return new HashSet();
-
- ContextImpl context1 = new ContextImpl(epw, resolver, null, Context.FIND_DELTA, session, null, this);
- epw.init(context1);
-
- Set<Map<String, Object>> myModifiedPks = new HashSet<>();
-
-
-
- for (EntityProcessorWrapper childEpw : epw.getChildren()) {
- //this ensures that we start from the leaf nodes
- myModifiedPks.addAll(collectDelta(childEpw, resolver, deletedRows));
- //someone called abort
- if (stop.get())
- return new HashSet();
- }
-
- // identifying the modified rows for this entity
- Map<String, Map<String, Object>> deltaSet = new HashMap<>();
- log.info("Running ModifiedRowKey() for Entity: " + epw.getEntity().getName());
- //get the modified rows in this entity
- String pk = epw.getEntity().getPk();
- while (true) {
- Map<String, Object> row = epw.nextModifiedRowKey();
-
- if (row == null)
- break;
-
- Object pkValue = row.get(pk);
- if (pkValue == null) {
- pk = findMatchingPkColumn(pk, row);
- pkValue = row.get(pk);
- }
-
- deltaSet.put(pkValue.toString(), row);
- importStatistics.rowsCount.incrementAndGet();
- // check for abort
- if (stop.get())
- return new HashSet();
- }
- //get the deleted rows for this entity
- Set<Map<String, Object>> deletedSet = new HashSet<>();
- while (true) {
- Map<String, Object> row = epw.nextDeletedRowKey();
- if (row == null)
- break;
-
- deletedSet.add(row);
-
- Object pkValue = row.get(pk);
- if (pkValue == null) {
- pk = findMatchingPkColumn(pk, row);
- pkValue = row.get(pk);
- }
-
- // Remove deleted rows from the delta rows
- String deletedRowPk = pkValue.toString();
- if (deltaSet.containsKey(deletedRowPk)) {
- deltaSet.remove(deletedRowPk);
- }
-
- importStatistics.rowsCount.incrementAndGet();
- // check for abort
- if (stop.get())
- return new HashSet();
- }
-
- log.info("Completed ModifiedRowKey for Entity: " + epw.getEntity().getName() + " rows obtained : " + deltaSet.size());
- log.info("Completed DeletedRowKey for Entity: " + epw.getEntity().getName() + " rows obtained : " + deletedSet.size());
-
- myModifiedPks.addAll(deltaSet.values());
- Set<Map<String, Object>> parentKeyList = new HashSet<>();
- //all that we have captured is useless (in a sub-entity) if no rows in the parent is modified because of these
- //propogate up the changes in the chain
- if (epw.getEntity().getParentEntity() != null) {
- // identifying deleted rows with deltas
-
- for (Map<String, Object> row : myModifiedPks) {
- resolver.addNamespace(epw.getEntity().getName(), row);
- getModifiedParentRows(resolver, epw.getEntity().getName(), epw, parentKeyList);
- // check for abort
- if (stop.get())
- return new HashSet();
- }
- // running the same for deletedrows
- for (Map<String, Object> row : deletedSet) {
- resolver.addNamespace(epw.getEntity().getName(), row);
- getModifiedParentRows(resolver, epw.getEntity().getName(), epw, parentKeyList);
- // check for abort
- if (stop.get())
- return new HashSet();
- }
- }
- log.info("Completed parentDeltaQuery for Entity: " + epw.getEntity().getName());
- if (epw.getEntity().isDocRoot())
- deletedRows.addAll(deletedSet);
-
- // Do not use entity.isDocRoot here because one of descendant entities may set rootEntity="true"
- return epw.getEntity().getParentEntity() == null ?
- myModifiedPks : new HashSet<>(parentKeyList);
- }
-
- private void getModifiedParentRows(VariableResolver resolver,
- String entity, EntityProcessor entityProcessor,
- Set<Map<String, Object>> parentKeyList) {
- try {
- while (true) {
- Map<String, Object> parentRow = entityProcessor
- .nextModifiedParentRowKey();
- if (parentRow == null)
- break;
-
- parentKeyList.add(parentRow);
- importStatistics.rowsCount.incrementAndGet();
- // check for abort
- if (stop.get())
- return;
- }
-
- } finally {
- resolver.removeNamespace(entity);
- }
- }
-
- public void abort() {
- stop.set(true);
- }
-
- private AtomicBoolean stop = new AtomicBoolean(false);
-
- public static final String TIME_ELAPSED = "Time Elapsed";
-
- static String getTimeElapsedSince(long l) {
- l = TimeUnit.MILLISECONDS.convert(System.nanoTime() - l, TimeUnit.NANOSECONDS);
- return (l / (60000 * 60)) + ":" + (l / 60000) % 60 + ":" + (l / 1000)
- % 60 + "." + l % 1000;
- }
-
- public RequestInfo getReqParams() {
- return reqParams;
- }
-
- @SuppressWarnings("unchecked")
- static Class loadClass(String name, SolrCore core) throws ClassNotFoundException {
- try {
- return core != null ?
- core.getResourceLoader().findClass(name, Object.class) :
- Class.forName(name);
- } catch (Exception e) {
- try {
- String n = DocBuilder.class.getPackage().getName() + "." + name;
- return core != null ?
- core.getResourceLoader().findClass(n, Object.class) :
- Class.forName(n);
- } catch (Exception e1) {
- throw new ClassNotFoundException("Unable to load " + name + " or " + DocBuilder.class.getPackage().getName() + "." + name, e);
- }
- }
- }
-
- public static class Statistics {
- public AtomicLong docCount = new AtomicLong();
-
- public AtomicLong deletedDocCount = new AtomicLong();
-
- public AtomicLong failedDocCount = new AtomicLong();
-
- public AtomicLong rowsCount = new AtomicLong();
-
- public AtomicLong queryCount = new AtomicLong();
-
- public AtomicLong skipDocCount = new AtomicLong();
-
- public Statistics add(Statistics stats) {
- this.docCount.addAndGet(stats.docCount.get());
- this.deletedDocCount.addAndGet(stats.deletedDocCount.get());
- this.rowsCount.addAndGet(stats.rowsCount.get());
- this.queryCount.addAndGet(stats.queryCount.get());
-
- return this;
- }
-
- public Map<String, Object> getStatsSnapshot() {
- Map<String, Object> result = new HashMap<>();
- result.put("docCount", docCount.get());
- result.put("deletedDocCount", deletedDocCount.get());
- result.put("rowCount", rowsCount.get());
- result.put("queryCount", rowsCount.get());
- result.put("skipDocCount", skipDocCount.get());
- return result;
- }
-
- }
-
- private void cleanByQuery(String delQuery, AtomicBoolean completeCleanDone) {
- delQuery = getVariableResolver().replaceTokens(delQuery);
- if (reqParams.isClean()) {
- if (delQuery == null && !completeCleanDone.get()) {
- writer.doDeleteAll();
- completeCleanDone.set(true);
- } else if (delQuery != null) {
- writer.deleteByQuery(delQuery);
- }
- }
- }
-
- public static final String LAST_INDEX_TIME = "last_index_time";
- public static final String INDEX_START_TIME = "index_start_time";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessor.java
deleted file mode 100644
index 8cfbed9..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessor.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.Map;
-
-/**
- * <p>
- * An instance of entity processor serves an entity. It is reused throughout the
- * import process.
- * </p>
- * <p>
- * Implementations of this abstract class must provide a public no-args constructor.
- * </p>
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
- * <p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 1.3
- */
-public abstract class EntityProcessor {
-
- /**
- * This method is called when it starts processing an entity. When it comes
- * back to the entity it is called again. So it can reset anything at that point.
- * For a rootmost entity this is called only once for an ingestion. For sub-entities , this
- * is called multiple once for each row from its parent entity
- *
- * @param context The current context
- */
- public abstract void init(Context context);
-
- /**
- * This method helps streaming the data for each row . The implementation
- * would fetch as many rows as needed and gives one 'row' at a time. Only this
- * method is used during a full import
- *
- * @return A 'row'. The 'key' for the map is the column name and the 'value'
- * is the value of that column. If there are no more rows to be
- * returned, return 'null'
- */
- public abstract Map<String, Object> nextRow();
-
- /**
- * This is used for delta-import. It gives the pks of the changed rows in this
- * entity
- *
- * @return the pk vs value of all changed rows
- */
- public abstract Map<String, Object> nextModifiedRowKey();
-
- /**
- * This is used during delta-import. It gives the primary keys of the rows
- * that are deleted from this entity. If this entity is the root entity, solr
- * document is deleted. If this is a sub-entity, the Solr document is
- * considered as 'changed' and will be recreated
- *
- * @return the pk vs value of all changed rows
- */
- public abstract Map<String, Object> nextDeletedRowKey();
-
- /**
- * This is used during delta-import. This gives the primary keys and their
- * values of all the rows changed in a parent entity due to changes in this
- * entity.
- *
- * @return the pk vs value of all changed rows in the parent entity
- */
- public abstract Map<String, Object> nextModifiedParentRowKey();
-
- /**
- * Invoked for each entity at the very end of the import to do any needed cleanup tasks.
- *
- */
- public abstract void destroy();
-
- /**
- * Invoked after the transformers are invoked. EntityProcessors can add, remove or modify values
- * added by Transformers in this method.
- *
- * @param r The transformed row
- * @since solr 1.4
- */
- public void postTransform(Map<String, Object> r) {
- }
-
- /**
- * Invoked when the Entity processor is destroyed towards the end of import.
- *
- * @since solr 1.4
- */
- public void close() {
- //no-op
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
deleted file mode 100644
index 8311f36..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.common.SolrException;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.invoke.MethodHandles;
-import java.util.*;
-
-/**
- * <p> Base class for all implementations of {@link EntityProcessor} </p> <p> Most implementations of {@link EntityProcessor}
- * extend this base class which provides common functionality. </p>
- * <p>
- * <b>This API is experimental and subject to change</b>
- *
- * @since solr 1.3
- */
-public class EntityProcessorBase extends EntityProcessor {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- protected boolean isFirstInit = true;
-
- protected String entityName;
-
- protected Context context;
-
- protected Iterator<Map<String, Object>> rowIterator;
-
- protected String query;
-
- protected String onError = ABORT;
-
- protected DIHCacheSupport cacheSupport = null;
-
- private Zipper zipper;
-
-
- @Override
- public void init(Context context) {
- this.context = context;
- if (isFirstInit) {
- firstInit(context);
- }
- if(zipper!=null){
- zipper.onNewParent(context);
- }else{
- if(cacheSupport!=null) {
- cacheSupport.initNewParent(context);
- }
- }
- }
-
- /**
- * first time init call. do one-time operations here
- * it's necessary to call it from the overridden method,
- * otherwise it throws NPE on accessing zipper from nextRow()
- */
- protected void firstInit(Context context) {
- entityName = context.getEntityAttribute("name");
- String s = context.getEntityAttribute(ON_ERROR);
- if (s != null) onError = s;
-
- zipper = Zipper.createOrNull(context);
-
- if(zipper==null){
- initCache(context);
- }
- isFirstInit = false;
- }
-
- protected void initCache(Context context) {
- String cacheImplName = context
- .getResolvedEntityAttribute(DIHCacheSupport.CACHE_IMPL);
-
- if (cacheImplName != null ) {
- cacheSupport = new DIHCacheSupport(context, cacheImplName);
- }
- }
-
- @Override
- public Map<String, Object> nextModifiedRowKey() {
- return null;
- }
-
- @Override
- public Map<String, Object> nextDeletedRowKey() {
- return null;
- }
-
- @Override
- public Map<String, Object> nextModifiedParentRowKey() {
- return null;
- }
-
- /**
- * For a simple implementation, this is the only method that the sub-class should implement. This is intended to
- * stream rows one-by-one. Return null to signal end of rows
- *
- * @return a row where the key is the name of the field and value can be any Object or a Collection of objects. Return
- * null to signal end of rows
- */
- @Override
- public Map<String, Object> nextRow() {
- return null;// do not do anything
- }
-
- protected Map<String, Object> getNext() {
- if(zipper!=null){
- return zipper.supplyNextChild(rowIterator);
- }else{
- if(cacheSupport==null) {
- try {
- if (rowIterator == null)
- return null;
- if (rowIterator.hasNext())
- return rowIterator.next();
- query = null;
- rowIterator = null;
- return null;
- } catch (Exception e) {
- SolrException.log(log, "getNext() failed for query '" + query + "'", e);
- query = null;
- rowIterator = null;
- wrapAndThrow(DataImportHandlerException.WARN, e);
- return null;
- }
- } else {
- return cacheSupport.getCacheData(context, query, rowIterator);
- }
- }
- }
-
-
- @Override
- public void destroy() {
- query = null;
- if(cacheSupport!=null){
- cacheSupport.destroyAll();
- }
- cacheSupport = null;
- }
-
-
-
- public static final String TRANSFORMER = "transformer";
-
- public static final String TRANSFORM_ROW = "transformRow";
-
- public static final String ON_ERROR = "onError";
-
- public static final String ABORT = "abort";
-
- public static final String CONTINUE = "continue";
-
- public static final String SKIP = "skip";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java
deleted file mode 100644
index 8a76e11..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java
+++ /dev/null
@@ -1,351 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.handler.dataimport.config.ConfigNameConstants;
-import org.apache.solr.handler.dataimport.config.Entity;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.*;
-import static org.apache.solr.handler.dataimport.EntityProcessorBase.*;
-import static org.apache.solr.handler.dataimport.EntityProcessorBase.SKIP;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.invoke.MethodHandles;
-import java.lang.reflect.Method;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A Wrapper over {@link EntityProcessor} instance which performs transforms and handles multi-row outputs correctly.
- *
- * @since solr 1.4
- */
-public class EntityProcessorWrapper extends EntityProcessor {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private EntityProcessor delegate;
- private Entity entity;
- private DataSource datasource;
- private List<EntityProcessorWrapper> children = new ArrayList<>();
- private DocBuilder docBuilder;
- private boolean initialized;
- private String onError;
- private Context context;
- private VariableResolver resolver;
- private String entityName;
-
- protected List<Transformer> transformers;
-
- protected List<Map<String, Object>> rowcache;
-
- public EntityProcessorWrapper(EntityProcessor delegate, Entity entity, DocBuilder docBuilder) {
- this.delegate = delegate;
- this.entity = entity;
- this.docBuilder = docBuilder;
- }
-
- @Override
- public void init(Context context) {
- rowcache = null;
- this.context = context;
- resolver = (VariableResolver) context.getVariableResolver();
- if (entityName == null) {
- onError = resolver.replaceTokens(context.getEntityAttribute(ON_ERROR));
- if (onError == null) onError = ABORT;
- entityName = context.getEntityAttribute(ConfigNameConstants.NAME);
- }
- delegate.init(context);
-
- }
-
- @SuppressWarnings("unchecked")
- void loadTransformers() {
- String transClasses = context.getEntityAttribute(TRANSFORMER);
-
- if (transClasses == null) {
- transformers = Collections.EMPTY_LIST;
- return;
- }
-
- String[] transArr = transClasses.split(",");
- transformers = new ArrayList<Transformer>() {
- @Override
- public boolean add(Transformer transformer) {
- if (docBuilder != null && docBuilder.verboseDebug) {
- transformer = docBuilder.getDebugLogger().wrapTransformer(transformer);
- }
- return super.add(transformer);
- }
- };
- for (String aTransArr : transArr) {
- String trans = aTransArr.trim();
- if (trans.startsWith("script:")) {
- // The script transformer is a potential vulnerability, esp. when the script is
- // provided from an untrusted source. Check and don't proceed if source is untrusted.
- checkIfTrusted(trans);
- String functionName = trans.substring("script:".length());
- ScriptTransformer scriptTransformer = new ScriptTransformer();
- scriptTransformer.setFunctionName(functionName);
- transformers.add(scriptTransformer);
- continue;
- }
- try {
- Class clazz = DocBuilder.loadClass(trans, context.getSolrCore());
- if (Transformer.class.isAssignableFrom(clazz)) {
- transformers.add((Transformer) clazz.newInstance());
- } else {
- Method meth = clazz.getMethod(TRANSFORM_ROW, Map.class);
- transformers.add(new ReflectionTransformer(meth, clazz, trans));
- }
- } catch (NoSuchMethodException nsme){
- String msg = "Transformer :"
- + trans
- + "does not implement Transformer interface or does not have a transformRow(Map<String.Object> m)method";
- log.error(msg);
- wrapAndThrow(SEVERE, nsme,msg);
- } catch (Exception e) {
- log.error("Unable to load Transformer: " + aTransArr, e);
- wrapAndThrow(SEVERE, e,"Unable to load Transformer: " + trans);
- }
- }
-
- }
-
- private void checkIfTrusted(String trans) {
- if (docBuilder != null) {
- SolrCore core = docBuilder.dataImporter.getCore();
- boolean trusted = (core != null)? core.getCoreDescriptor().isConfigSetTrusted(): true;
- if (!trusted) {
- Exception ex = new SolrException(ErrorCode.UNAUTHORIZED, "The configset for this collection was uploaded "
- + "without any authentication in place,"
- + " and this transformer is not available for collections with untrusted configsets. To use this transformer,"
- + " re-upload the configset after enabling authentication and authorization.");
- String msg = "Transformer: "
- + trans
- + ". " + ex.getMessage();
- log.error(msg);
- wrapAndThrow(SEVERE, ex, msg);
- }
- }
- }
-
- @SuppressWarnings("unchecked")
- static class ReflectionTransformer extends Transformer {
- final Method meth;
-
- final Class clazz;
-
- final String trans;
-
- final Object o;
-
- public ReflectionTransformer(Method meth, Class clazz, String trans)
- throws Exception {
- this.meth = meth;
- this.clazz = clazz;
- this.trans = trans;
- o = clazz.newInstance();
- }
-
- @Override
- public Object transformRow(Map<String, Object> aRow, Context context) {
- try {
- return meth.invoke(o, aRow);
- } catch (Exception e) {
- log.warn("method invocation failed on transformer : " + trans, e);
- throw new DataImportHandlerException(WARN, e);
- }
- }
- }
-
- protected Map<String, Object> getFromRowCache() {
- Map<String, Object> r = rowcache.remove(0);
- if (rowcache.isEmpty())
- rowcache = null;
- return r;
- }
-
- @SuppressWarnings("unchecked")
- protected Map<String, Object> applyTransformer(Map<String, Object> row) {
- if(row == null) return null;
- if (transformers == null)
- loadTransformers();
- if (transformers == Collections.EMPTY_LIST)
- return row;
- Map<String, Object> transformedRow = row;
- List<Map<String, Object>> rows = null;
- boolean stopTransform = checkStopTransform(row);
- VariableResolver resolver = (VariableResolver) context.getVariableResolver();
- for (Transformer t : transformers) {
- if (stopTransform) break;
- try {
- if (rows != null) {
- List<Map<String, Object>> tmpRows = new ArrayList<>();
- for (Map<String, Object> map : rows) {
- resolver.addNamespace(entityName, map);
- Object o = t.transformRow(map, context);
- if (o == null)
- continue;
- if (o instanceof Map) {
- Map oMap = (Map) o;
- stopTransform = checkStopTransform(oMap);
- tmpRows.add((Map) o);
- } else if (o instanceof List) {
- tmpRows.addAll((List) o);
- } else {
- log.error("Transformer must return Map<String, Object> or a List<Map<String, Object>>");
- }
- }
- rows = tmpRows;
- } else {
- resolver.addNamespace(entityName, transformedRow);
- Object o = t.transformRow(transformedRow, context);
- if (o == null)
- return null;
- if (o instanceof Map) {
- Map oMap = (Map) o;
- stopTransform = checkStopTransform(oMap);
- transformedRow = (Map) o;
- } else if (o instanceof List) {
- rows = (List) o;
- } else {
- log.error("Transformer must return Map<String, Object> or a List<Map<String, Object>>");
- }
- }
- } catch (Exception e) {
- log.warn("transformer threw error", e);
- if (ABORT.equals(onError)) {
- wrapAndThrow(SEVERE, e);
- } else if (SKIP.equals(onError)) {
- wrapAndThrow(DataImportHandlerException.SKIP, e);
- }
- // onError = continue
- }
- }
- if (rows == null) {
- return transformedRow;
- } else {
- rowcache = rows;
- return getFromRowCache();
- }
-
- }
-
- private boolean checkStopTransform(Map oMap) {
- return oMap.get("$stopTransform") != null
- && Boolean.parseBoolean(oMap.get("$stopTransform").toString());
- }
-
- @Override
- public Map<String, Object> nextRow() {
- if (rowcache != null) {
- return getFromRowCache();
- }
- while (true) {
- Map<String, Object> arow = null;
- try {
- arow = delegate.nextRow();
- } catch (Exception e) {
- if(ABORT.equals(onError)){
- wrapAndThrow(SEVERE, e);
- } else {
- //SKIP is not really possible. If this calls the nextRow() again the Entityprocessor would be in an inconisttent state
- SolrException.log(log, "Exception in entity : "+ entityName, e);
- return null;
- }
- }
- if (arow == null) {
- return null;
- } else {
- arow = applyTransformer(arow);
- if (arow != null) {
- delegate.postTransform(arow);
- return arow;
- }
- }
- }
- }
-
- @Override
- public Map<String, Object> nextModifiedRowKey() {
- Map<String, Object> row = delegate.nextModifiedRowKey();
- row = applyTransformer(row);
- rowcache = null;
- return row;
- }
-
- @Override
- public Map<String, Object> nextDeletedRowKey() {
- Map<String, Object> row = delegate.nextDeletedRowKey();
- row = applyTransformer(row);
- rowcache = null;
- return row;
- }
-
- @Override
- public Map<String, Object> nextModifiedParentRowKey() {
- return delegate.nextModifiedParentRowKey();
- }
-
- @Override
- public void destroy() {
- delegate.destroy();
- }
-
- public VariableResolver getVariableResolver() {
- return (VariableResolver) context.getVariableResolver();
- }
-
- public Context getContext() {
- return context;
- }
-
- @Override
- public void close() {
- delegate.close();
- }
-
- public Entity getEntity() {
- return entity;
- }
-
- public List<EntityProcessorWrapper> getChildren() {
- return children;
- }
-
- public DataSource getDatasource() {
- return datasource;
- }
-
- public void setDatasource(DataSource datasource) {
- this.datasource = datasource;
- }
-
- public boolean isInitialized() {
- return initialized;
- }
-
- public void setInitialized(boolean initialized) {
- this.initialized = initialized;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Evaluator.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Evaluator.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Evaluator.java
deleted file mode 100644
index 22282b9..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Evaluator.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.regex.Pattern;
-
-/**
- * <p>
- * Pluggable functions for resolving variables
- * </p>
- * <p>
- * Implementations of this abstract class must provide a public no-arg constructor.
- * </p>
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
- * <b>This API is experimental and may change in the future.</b>
- *
- * @since solr 1.3
- */
-public abstract class Evaluator {
-
- /**
- * Return a String after processing an expression and a {@link VariableResolver}
- *
- * @see VariableResolver
- * @param expression string to be evaluated
- * @param context instance
- * @return the value of the given expression evaluated using the resolver
- */
- public abstract String evaluate(String expression, Context context);
-
- /**
- * Parses a string of expression into separate params. The values are separated by commas. each value will be
- * translated into one of the following:
- * <ol>
- * <li>If it is in single quotes the value will be translated to a String</li>
- * <li>If is is not in quotes and is a number a it will be translated into a Double</li>
- * <li>else it is a variable which can be resolved and it will be put in as an instance of VariableWrapper</li>
- * </ol>
- *
- * @param expression the expression to be parsed
- * @param vr the VariableResolver instance for resolving variables
- *
- * @return a List of objects which can either be a string, number or a variable wrapper
- */
- protected List<Object> parseParams(String expression, VariableResolver vr) {
- List<Object> result = new ArrayList<>();
- expression = expression.trim();
- String[] ss = expression.split(",");
- for (int i = 0; i < ss.length; i++) {
- ss[i] = ss[i].trim();
- if (ss[i].startsWith("'")) {//a string param has started
- StringBuilder sb = new StringBuilder();
- while (true) {
- sb.append(ss[i]);
- if (ss[i].endsWith("'")) break;
- i++;
- if (i >= ss.length)
- throw new DataImportHandlerException(SEVERE, "invalid string at " + ss[i - 1] + " in function params: " + expression);
- sb.append(",");
- }
- String s = sb.substring(1, sb.length() - 1);
- s = s.replaceAll("\\\\'", "'");
- result.add(s);
- } else {
- if (Character.isDigit(ss[i].charAt(0))) {
- try {
- Double doub = Double.parseDouble(ss[i]);
- result.add(doub);
- } catch (NumberFormatException e) {
- if (vr.resolve(ss[i]) == null) {
- wrapAndThrow(
- SEVERE, e, "Invalid number :" + ss[i] +
- "in parameters " + expression);
- }
- }
- } else {
- result.add(getVariableWrapper(ss[i], vr));
- }
- }
- }
- return result;
- }
-
- protected VariableWrapper getVariableWrapper(String s, VariableResolver vr) {
- return new VariableWrapper(s,vr);
- }
-
- static protected class VariableWrapper {
- public final String varName;
- public final VariableResolver vr;
-
- public VariableWrapper(String s, VariableResolver vr) {
- this.varName = s;
- this.vr = vr;
- }
-
- public Object resolve() {
- return vr.resolve(varName);
- }
-
- @Override
- public String toString() {
- Object o = vr.resolve(varName);
- return o == null ? null : o.toString();
- }
- }
-
- static Pattern IN_SINGLE_QUOTES = Pattern.compile("^'(.*?)'$");
-
- public static final String DATE_FORMAT_EVALUATOR = "formatDate";
-
- public static final String URL_ENCODE_EVALUATOR = "encodeUrl";
-
- public static final String ESCAPE_SOLR_QUERY_CHARS = "escapeQueryChars";
-
- public static final String SQL_ESCAPE_EVALUATOR = "escapeSql";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EventListener.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EventListener.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EventListener.java
deleted file mode 100644
index 0c43a0b..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EventListener.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-/**
- * Event listener for DataImportHandler
- *
- * <b>This API is experimental and subject to change</b>
- *
- * @since solr 1.4
- */
-public interface EventListener {
-
- /**
- * Event callback
- *
- * @param ctx the Context in which this event was called
- */
- void onEvent(Context ctx);
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java
deleted file mode 100644
index 571c280..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.*;
-import java.lang.invoke.MethodHandles;
-import java.nio.charset.StandardCharsets;
-import java.sql.Blob;
-import java.sql.Clob;
-import java.sql.SQLException;
-import java.util.Properties;
-
-/**
- * This can be useful for users who have a DB field containing xml and wish to use a nested {@link XPathEntityProcessor}
- * <p>
- * The datasouce may be configured as follows
- * <p>
- * <datasource name="f1" type="FieldReaderDataSource" />
- * <p>
- * The entity which uses this datasource must keep the url value as the variable name url="field-name"
- * <p>
- * The fieldname must be resolvable from {@link VariableResolver}
- * <p>
- * This may be used with any {@link EntityProcessor} which uses a {@link DataSource}<{@link Reader}> eg: {@link XPathEntityProcessor}
- * <p>
- * Supports String, BLOB, CLOB data types and there is an extra field (in the entity) 'encoding' for BLOB types
- *
- * @since 1.4
- */
-public class FieldReaderDataSource extends DataSource<Reader> {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- protected VariableResolver vr;
- protected String dataField;
- private String encoding;
- private EntityProcessorWrapper entityProcessor;
-
- @Override
- public void init(Context context, Properties initProps) {
- dataField = context.getEntityAttribute("dataField");
- encoding = context.getEntityAttribute("encoding");
- entityProcessor = (EntityProcessorWrapper) context.getEntityProcessor();
- /*no op*/
- }
-
- @Override
- public Reader getData(String query) {
- Object o = entityProcessor.getVariableResolver().resolve(dataField);
- if (o == null) {
- throw new DataImportHandlerException (SEVERE, "No field available for name : " +dataField);
- }
- if (o instanceof String) {
- return new StringReader((String) o);
- } else if (o instanceof Clob) {
- Clob clob = (Clob) o;
- try {
- //Most of the JDBC drivers have getCharacterStream defined as public
- // so let us just check it
- return readCharStream(clob);
- } catch (Exception e) {
- log.info("Unable to get data from CLOB");
- return null;
-
- }
-
- } else if (o instanceof Blob) {
- Blob blob = (Blob) o;
- try {
- return getReader(blob);
- } catch (Exception e) {
- log.info("Unable to get data from BLOB");
- return null;
-
- }
- } else {
- return new StringReader(o.toString());
- }
-
- }
-
- static Reader readCharStream(Clob clob) {
- try {
- return clob.getCharacterStream();
- } catch (Exception e) {
- wrapAndThrow(SEVERE, e,"Unable to get reader from clob");
- return null;//unreachable
- }
- }
-
- private Reader getReader(Blob blob)
- throws SQLException, UnsupportedEncodingException {
- if (encoding == null) {
- return (new InputStreamReader(blob.getBinaryStream(), StandardCharsets.UTF_8));
- } else {
- return (new InputStreamReader(blob.getBinaryStream(), encoding));
- }
- }
-
- @Override
- public void close() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldStreamDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldStreamDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldStreamDataSource.java
deleted file mode 100644
index ba7ca5d..0000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldStreamDataSource.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.lang.invoke.MethodHandles;
-import java.sql.Blob;
-import java.sql.SQLException;
-import java.util.Properties;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * This can be useful for users who have a DB field containing BLOBs which may be Rich documents
- * <p>
- * The datasource may be configured as follows
- * <p>
- * <dataSource name="f1" type="FieldStreamDataSource" />
- * <p>
- * The entity which uses this datasource must keep and attribute dataField
- * <p>
- * The fieldname must be resolvable from {@link VariableResolver}
- * <p>
- * This may be used with any {@link EntityProcessor} which uses a {@link DataSource}<{@link InputStream}> eg: TikaEntityProcessor
- *
- * @since 3.1
- */
-public class FieldStreamDataSource extends DataSource<InputStream> {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- protected VariableResolver vr;
- protected String dataField;
- private EntityProcessorWrapper wrapper;
-
- @Override
- public void init(Context context, Properties initProps) {
- dataField = context.getEntityAttribute("dataField");
- wrapper = (EntityProcessorWrapper) context.getEntityProcessor();
- /*no op*/
- }
-
- @Override
- public InputStream getData(String query) {
- Object o = wrapper.getVariableResolver().resolve(dataField);
- if (o == null) {
- throw new DataImportHandlerException(SEVERE, "No field available for name : " + dataField);
- } else if (o instanceof Blob) {
- Blob blob = (Blob) o;
- try {
- return blob.getBinaryStream();
- } catch (SQLException sqle) {
- log.info("Unable to get data from BLOB");
- return null;
- }
- } else if (o instanceof byte[]) {
- byte[] bytes = (byte[]) o;
- return new ByteArrayInputStream(bytes);
- } else {
- throw new RuntimeException("unsupported type : " + o.getClass());
- }
-
- }
-
- @Override
- public void close() {
- }
-}
[25/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
Adding dataimporthandler-extras module
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d7c03684
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d7c03684
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d7c03684
Branch: refs/heads/jira/gradle
Commit: d7c036847ee30019176427eb2e094cae5f67b2d2
Parents: 887b808
Author: Cao Manh Dat <da...@apache.org>
Authored: Fri Nov 2 11:33:06 2018 +0000
Committer: Cao Manh Dat <da...@apache.org>
Committed: Fri Nov 2 11:33:06 2018 +0000
----------------------------------------------------------------------
settings.gradle | 4 +-
.../dataimporthandler-extras/build.gradle | 14 +
.../handler/dataimport/MailEntityProcessor.java | 888 -----------------
.../handler/dataimport/TikaEntityProcessor.java | 248 -----
.../apache/solr/handler/dataimport/package.html | 23 -
.../src/java/overview.html | 21 -
.../handler/dataimport/MailEntityProcessor.java | 888 +++++++++++++++++
.../handler/dataimport/TikaEntityProcessor.java | 248 +++++
.../apache/solr/handler/dataimport/package.html | 23 +
.../src/main/java/overview.html | 21 +
.../main/resources/solr-default-tika-config.xml | 20 +
.../src/resources/solr-default-tika-config.xml | 20 -
.../src/test-files/dihextras/bad.doc | Bin 116222 -> 0 bytes
.../src/test-files/dihextras/solr-word.pdf | Bin 21052 -> 0 bytes
.../conf/dataimport-schema-no-unique-key.xml | 205 ----
.../collection1/conf/dataimport-solrconfig.xml | 283 ------
.../src/test-files/dihextras/structured.html | 29 -
.../src/test-files/dihextras/test_jpeg.jpg | Bin 12924 -> 0 bytes
.../dihextras/test_recursive_embedded.docx | Bin 27082 -> 0 bytes
.../src/test-files/dihextras/test_vsdx.vsdx | Bin 44426 -> 0 bytes
.../dataimport/TestMailEntityProcessor.java | 200 ++++
.../dataimport/TestTikaEntityProcessor.java | 221 +++++
.../dataimport/TestMailEntityProcessor.java | 200 ----
.../dataimport/TestTikaEntityProcessor.java | 221 -----
.../src/test/resources/dihextras/bad.doc | Bin 0 -> 116222 bytes
.../src/test/resources/dihextras/solr-word.pdf | Bin 0 -> 21052 bytes
.../conf/dataimport-schema-no-unique-key.xml | 205 ++++
.../collection1/conf/dataimport-solrconfig.xml | 283 ++++++
.../test/resources/dihextras/structured.html | 29 +
.../src/test/resources/dihextras/test_jpeg.jpg | Bin 0 -> 12924 bytes
.../dihextras/test_recursive_embedded.docx | Bin 0 -> 27082 bytes
.../src/test/resources/dihextras/test_vsdx.vsdx | Bin 0 -> 44426 bytes
solr/contrib/dataimporthandler/build.gradle | 25 +
solr/contrib/dataimporthandler/derby.log | 10 +
.../dataimport/BinContentStreamDataSource.java | 70 --
.../handler/dataimport/BinFileDataSource.java | 64 --
.../handler/dataimport/BinURLDataSource.java | 104 --
.../handler/dataimport/CachePropertyUtil.java | 48 -
.../handler/dataimport/ClobTransformer.java | 84 --
.../handler/dataimport/ConfigParseUtil.java | 73 --
.../dataimport/ContentStreamDataSource.java | 69 --
.../apache/solr/handler/dataimport/Context.java | 219 ----
.../solr/handler/dataimport/ContextImpl.java | 261 -----
.../solr/handler/dataimport/DIHCache.java | 103 --
.../handler/dataimport/DIHCacheSupport.java | 279 ------
.../solr/handler/dataimport/DIHLogLevels.java | 21 -
.../solr/handler/dataimport/DIHProperties.java | 45 -
.../solr/handler/dataimport/DIHWriter.java | 99 --
.../solr/handler/dataimport/DIHWriterBase.java | 44 -
.../handler/dataimport/DataImportHandler.java | 304 ------
.../dataimport/DataImportHandlerException.java | 75 --
.../solr/handler/dataimport/DataImporter.java | 622 ------------
.../solr/handler/dataimport/DataSource.java | 65 --
.../handler/dataimport/DateFormatEvaluator.java | 180 ----
.../dataimport/DateFormatTransformer.java | 105 --
.../solr/handler/dataimport/DebugInfo.java | 66 --
.../solr/handler/dataimport/DebugLogger.java | 285 ------
.../solr/handler/dataimport/DocBuilder.java | 988 -------------------
.../handler/dataimport/EntityProcessor.java | 113 ---
.../handler/dataimport/EntityProcessorBase.java | 174 ----
.../dataimport/EntityProcessorWrapper.java | 351 -------
.../solr/handler/dataimport/Evaluator.java | 140 ---
.../solr/handler/dataimport/EventListener.java | 35 -
.../dataimport/FieldReaderDataSource.java | 122 ---
.../dataimport/FieldStreamDataSource.java | 85 --
.../solr/handler/dataimport/FileDataSource.java | 153 ---
.../dataimport/FileListEntityProcessor.java | 305 ------
.../dataimport/HTMLStripTransformer.java | 95 --
.../solr/handler/dataimport/JdbcDataSource.java | 593 -----------
.../handler/dataimport/LineEntityProcessor.java | 164 ---
.../solr/handler/dataimport/LogTransformer.java | 67 --
.../solr/handler/dataimport/MockDataSource.java | 61 --
.../dataimport/NumberFormatTransformer.java | 133 ---
.../dataimport/PlainTextEntityProcessor.java | 77 --
.../handler/dataimport/RegexTransformer.java | 201 ----
.../solr/handler/dataimport/RequestInfo.java | 177 ----
.../handler/dataimport/ScriptTransformer.java | 102 --
.../dataimport/SimplePropertiesWriter.java | 247 -----
.../handler/dataimport/SolrEntityProcessor.java | 321 ------
.../dataimport/SolrQueryEscapingEvaluator.java | 35 -
.../solr/handler/dataimport/SolrWriter.java | 175 ----
.../dataimport/SortedMapBackedCache.java | 238 -----
.../handler/dataimport/SqlEntityProcessor.java | 171 ----
.../dataimport/SqlEscapingEvaluator.java | 41 -
.../handler/dataimport/TemplateTransformer.java | 114 ---
.../solr/handler/dataimport/Transformer.java | 50 -
.../solr/handler/dataimport/URLDataSource.java | 154 ---
.../solr/handler/dataimport/UrlEvaluator.java | 46 -
.../handler/dataimport/VariableResolver.java | 211 ----
.../dataimport/XPathEntityProcessor.java | 548 ----------
.../handler/dataimport/XPathRecordReader.java | 669 -------------
.../handler/dataimport/ZKPropertiesWriter.java | 95 --
.../apache/solr/handler/dataimport/Zipper.java | 115 ---
.../dataimport/config/ConfigNameConstants.java | 60 --
.../dataimport/config/ConfigParseUtil.java | 72 --
.../dataimport/config/DIHConfiguration.java | 195 ----
.../solr/handler/dataimport/config/Entity.java | 228 -----
.../handler/dataimport/config/EntityField.java | 102 --
.../solr/handler/dataimport/config/Field.java | 108 --
.../dataimport/config/PropertyWriter.java | 38 -
.../solr/handler/dataimport/config/Script.java | 41 -
.../handler/dataimport/config/package-info.java | 24 -
.../solr/handler/dataimport/package-info.java | 25 -
.../dataimporthandler/src/java/overview.html | 21 -
.../dataimport/BinContentStreamDataSource.java | 70 ++
.../handler/dataimport/BinFileDataSource.java | 64 ++
.../handler/dataimport/BinURLDataSource.java | 104 ++
.../handler/dataimport/CachePropertyUtil.java | 48 +
.../handler/dataimport/ClobTransformer.java | 84 ++
.../handler/dataimport/ConfigParseUtil.java | 73 ++
.../dataimport/ContentStreamDataSource.java | 69 ++
.../apache/solr/handler/dataimport/Context.java | 219 ++++
.../solr/handler/dataimport/ContextImpl.java | 261 +++++
.../solr/handler/dataimport/DIHCache.java | 103 ++
.../handler/dataimport/DIHCacheSupport.java | 279 ++++++
.../solr/handler/dataimport/DIHLogLevels.java | 21 +
.../solr/handler/dataimport/DIHProperties.java | 45 +
.../solr/handler/dataimport/DIHWriter.java | 99 ++
.../solr/handler/dataimport/DIHWriterBase.java | 44 +
.../handler/dataimport/DataImportHandler.java | 304 ++++++
.../dataimport/DataImportHandlerException.java | 75 ++
.../solr/handler/dataimport/DataImporter.java | 622 ++++++++++++
.../solr/handler/dataimport/DataSource.java | 65 ++
.../handler/dataimport/DateFormatEvaluator.java | 180 ++++
.../dataimport/DateFormatTransformer.java | 105 ++
.../solr/handler/dataimport/DebugInfo.java | 66 ++
.../solr/handler/dataimport/DebugLogger.java | 285 ++++++
.../solr/handler/dataimport/DocBuilder.java | 988 +++++++++++++++++++
.../handler/dataimport/EntityProcessor.java | 113 +++
.../handler/dataimport/EntityProcessorBase.java | 174 ++++
.../dataimport/EntityProcessorWrapper.java | 351 +++++++
.../solr/handler/dataimport/Evaluator.java | 140 +++
.../solr/handler/dataimport/EventListener.java | 35 +
.../dataimport/FieldReaderDataSource.java | 122 +++
.../dataimport/FieldStreamDataSource.java | 85 ++
.../solr/handler/dataimport/FileDataSource.java | 153 +++
.../dataimport/FileListEntityProcessor.java | 305 ++++++
.../dataimport/HTMLStripTransformer.java | 95 ++
.../solr/handler/dataimport/JdbcDataSource.java | 593 +++++++++++
.../handler/dataimport/LineEntityProcessor.java | 164 +++
.../solr/handler/dataimport/LogTransformer.java | 67 ++
.../solr/handler/dataimport/MockDataSource.java | 61 ++
.../dataimport/NumberFormatTransformer.java | 133 +++
.../dataimport/PlainTextEntityProcessor.java | 77 ++
.../handler/dataimport/RegexTransformer.java | 201 ++++
.../solr/handler/dataimport/RequestInfo.java | 177 ++++
.../handler/dataimport/ScriptTransformer.java | 102 ++
.../dataimport/SimplePropertiesWriter.java | 247 +++++
.../handler/dataimport/SolrEntityProcessor.java | 321 ++++++
.../dataimport/SolrQueryEscapingEvaluator.java | 35 +
.../solr/handler/dataimport/SolrWriter.java | 175 ++++
.../dataimport/SortedMapBackedCache.java | 238 +++++
.../handler/dataimport/SqlEntityProcessor.java | 171 ++++
.../dataimport/SqlEscapingEvaluator.java | 41 +
.../handler/dataimport/TemplateTransformer.java | 114 +++
.../solr/handler/dataimport/Transformer.java | 50 +
.../solr/handler/dataimport/URLDataSource.java | 154 +++
.../solr/handler/dataimport/UrlEvaluator.java | 46 +
.../handler/dataimport/VariableResolver.java | 211 ++++
.../dataimport/XPathEntityProcessor.java | 548 ++++++++++
.../handler/dataimport/XPathRecordReader.java | 669 +++++++++++++
.../handler/dataimport/ZKPropertiesWriter.java | 95 ++
.../apache/solr/handler/dataimport/Zipper.java | 115 +++
.../dataimport/config/ConfigNameConstants.java | 60 ++
.../dataimport/config/ConfigParseUtil.java | 72 ++
.../dataimport/config/DIHConfiguration.java | 195 ++++
.../solr/handler/dataimport/config/Entity.java | 228 +++++
.../handler/dataimport/config/EntityField.java | 102 ++
.../solr/handler/dataimport/config/Field.java | 108 ++
.../dataimport/config/PropertyWriter.java | 38 +
.../solr/handler/dataimport/config/Script.java | 41 +
.../handler/dataimport/config/package-info.java | 24 +
.../solr/handler/dataimport/package-info.java | 25 +
.../src/main/java/overview.html | 21 +
.../conf/contentstream-solrconfig.xml | 293 ------
.../collection1/conf/data-config-end-to-end.xml | 41 -
.../conf/data-config-with-datasource.xml | 9 -
.../conf/data-config-with-transformer.xml | 10 -
.../conf/dataconfig-contentstream.xml | 10 -
.../conf/dataimport-nodatasource-solrconfig.xml | 285 ------
.../solr/collection1/conf/dataimport-schema.xml | 70 --
.../conf/dataimport-solr_id-schema.xml | 313 ------
.../collection1/conf/dataimport-solrconfig.xml | 293 ------
.../dih/solr/collection1/conf/protwords.txt | 20 -
.../conf/single-entity-data-config.xml | 9 -
.../dih/solr/collection1/conf/stopwords.txt | 16 -
.../dih/solr/collection1/conf/synonyms.txt | 22 -
.../src/test-files/dih/solr/solr.xml | 27 -
.../dataimport/AbstractDIHCacheTestCase.java | 235 +++++
.../dataimport/AbstractDIHJdbcTestCase.java | 198 ++++
.../AbstractDataImportHandlerTestCase.java | 379 +++++++
.../AbstractSqlEntityProcessorTestCase.java | 845 ++++++++++++++++
.../dataimport/AddAColumnTransformer.java | 31 +
.../handler/dataimport/DestroyCountCache.java | 37 +
.../dataimport/MockInitialContextFactory.java | 52 +
.../dataimport/MockSolrEntityProcessor.java | 76 ++
.../dataimport/MockStringDataSource.java | 54 +
.../dataimport/TestBuiltInEvaluators.java | 188 ++++
.../handler/dataimport/TestClobTransformer.java | 62 ++
.../dataimport/TestContentStreamDataSource.java | 193 ++++
.../handler/dataimport/TestContextImpl.java | 70 ++
.../solr/handler/dataimport/TestDataConfig.java | 76 ++
.../dataimport/TestDateFormatTransformer.java | 89 ++
.../solr/handler/dataimport/TestDocBuilder.java | 335 +++++++
.../handler/dataimport/TestDocBuilder2.java | 433 ++++++++
.../dataimport/TestEntityProcessorBase.java | 84 ++
.../handler/dataimport/TestEphemeralCache.java | 140 +++
.../handler/dataimport/TestErrorHandling.java | 209 ++++
.../handler/dataimport/TestFieldReader.java | 65 ++
.../dataimport/TestFileListEntityProcessor.java | 188 ++++
.../TestFileListWithLineEntityProcessor.java | 64 ++
.../dataimport/TestHierarchicalDocBuilder.java | 479 +++++++++
.../handler/dataimport/TestJdbcDataSource.java | 662 +++++++++++++
.../TestJdbcDataSourceConvertType.java | 76 ++
.../dataimport/TestLineEntityProcessor.java | 251 +++++
.../handler/dataimport/TestNestedChildren.java | 65 ++
.../dataimport/TestNonWritablePersistFile.java | 102 ++
.../dataimport/TestNumberFormatTransformer.java | 160 +++
.../TestPlainTextEntityProcessor.java | 179 ++++
.../dataimport/TestRegexTransformer.java | 208 ++++
.../dataimport/TestScriptTransformer.java | 149 +++
.../dataimport/TestSimplePropertiesWriter.java | 151 +++
.../TestSolrEntityProcessorEndToEnd.java | 373 +++++++
.../dataimport/TestSolrEntityProcessorUnit.java | 187 ++++
.../dataimport/TestSortedMapBackedCache.java | 192 ++++
.../dataimport/TestSqlEntityProcessor.java | 115 +++
.../dataimport/TestSqlEntityProcessorDelta.java | 209 ++++
.../dataimport/TestTemplateTransformer.java | 111 +++
.../handler/dataimport/TestURLDataSource.java | 45 +
.../dataimport/TestVariableResolver.java | 172 ++++
.../TestVariableResolverEndToEnd.java | 141 +++
.../solr/handler/dataimport/TestWriterImpl.java | 81 ++
.../dataimport/TestXPathEntityProcessor.java | 491 +++++++++
.../dataimport/TestXPathRecordReader.java | 598 +++++++++++
.../dataimport/TestZKPropertiesWriter.java | 164 +++
.../dataimport/TripleThreatTransformer.java | 75 ++
.../dataimport/AbstractDIHCacheTestCase.java | 235 -----
.../dataimport/AbstractDIHJdbcTestCase.java | 198 ----
.../AbstractDataImportHandlerTestCase.java | 379 -------
.../AbstractSqlEntityProcessorTestCase.java | 845 ----------------
.../dataimport/AddAColumnTransformer.java | 31 -
.../handler/dataimport/DestroyCountCache.java | 37 -
.../dataimport/MockInitialContextFactory.java | 52 -
.../dataimport/MockSolrEntityProcessor.java | 76 --
.../dataimport/MockStringDataSource.java | 54 -
.../dataimport/TestBuiltInEvaluators.java | 188 ----
.../handler/dataimport/TestClobTransformer.java | 62 --
.../dataimport/TestContentStreamDataSource.java | 193 ----
.../handler/dataimport/TestContextImpl.java | 70 --
.../solr/handler/dataimport/TestDataConfig.java | 76 --
.../dataimport/TestDateFormatTransformer.java | 89 --
.../solr/handler/dataimport/TestDocBuilder.java | 335 -------
.../handler/dataimport/TestDocBuilder2.java | 433 --------
.../dataimport/TestEntityProcessorBase.java | 84 --
.../handler/dataimport/TestEphemeralCache.java | 140 ---
.../handler/dataimport/TestErrorHandling.java | 209 ----
.../handler/dataimport/TestFieldReader.java | 65 --
.../dataimport/TestFileListEntityProcessor.java | 188 ----
.../TestFileListWithLineEntityProcessor.java | 64 --
.../dataimport/TestHierarchicalDocBuilder.java | 479 ---------
.../handler/dataimport/TestJdbcDataSource.java | 662 -------------
.../TestJdbcDataSourceConvertType.java | 76 --
.../dataimport/TestLineEntityProcessor.java | 251 -----
.../handler/dataimport/TestNestedChildren.java | 65 --
.../dataimport/TestNonWritablePersistFile.java | 102 --
.../dataimport/TestNumberFormatTransformer.java | 160 ---
.../TestPlainTextEntityProcessor.java | 179 ----
.../dataimport/TestRegexTransformer.java | 208 ----
.../dataimport/TestScriptTransformer.java | 149 ---
.../dataimport/TestSimplePropertiesWriter.java | 151 ---
.../TestSolrEntityProcessorEndToEnd.java | 373 -------
.../dataimport/TestSolrEntityProcessorUnit.java | 187 ----
.../dataimport/TestSortedMapBackedCache.java | 192 ----
.../dataimport/TestSqlEntityProcessor.java | 115 ---
.../dataimport/TestSqlEntityProcessorDelta.java | 209 ----
.../dataimport/TestTemplateTransformer.java | 111 ---
.../handler/dataimport/TestURLDataSource.java | 45 -
.../dataimport/TestVariableResolver.java | 172 ----
.../TestVariableResolverEndToEnd.java | 141 ---
.../solr/handler/dataimport/TestWriterImpl.java | 81 --
.../dataimport/TestXPathEntityProcessor.java | 491 ---------
.../dataimport/TestXPathRecordReader.java | 598 -----------
.../dataimport/TestZKPropertiesWriter.java | 164 ---
.../dataimport/TripleThreatTransformer.java | 75 --
.../conf/contentstream-solrconfig.xml | 293 ++++++
.../collection1/conf/data-config-end-to-end.xml | 41 +
.../conf/data-config-with-datasource.xml | 9 +
.../conf/data-config-with-transformer.xml | 10 +
.../conf/dataconfig-contentstream.xml | 10 +
.../conf/dataimport-nodatasource-solrconfig.xml | 285 ++++++
.../solr/collection1/conf/dataimport-schema.xml | 70 ++
.../conf/dataimport-solr_id-schema.xml | 313 ++++++
.../collection1/conf/dataimport-solrconfig.xml | 293 ++++++
.../dih/solr/collection1/conf/protwords.txt | 20 +
.../conf/single-entity-data-config.xml | 9 +
.../dih/solr/collection1/conf/stopwords.txt | 16 +
.../dih/solr/collection1/conf/synonyms.txt | 22 +
.../src/test/resources/dih/solr/solr.xml | 27 +
298 files changed, 24712 insertions(+), 24661 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/settings.gradle
----------------------------------------------------------------------
diff --git a/settings.gradle b/settings.gradle
index 08e9679..658b11a 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -48,4 +48,6 @@ include 'solr:test-framework'
include 'solr:example:example-DIH'
include 'solr:contrib:analysis-extras'
include 'solr:contrib:analytics'
-include 'solr:contrib:clustering'
\ No newline at end of file
+include 'solr:contrib:clustering'
+include 'solr:contrib:dataimporthandler'
+include 'solr:contrib:dataimporthandler-extras'
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/build.gradle
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/build.gradle b/solr/contrib/dataimporthandler-extras/build.gradle
new file mode 100644
index 0000000..21966ee
--- /dev/null
+++ b/solr/contrib/dataimporthandler-extras/build.gradle
@@ -0,0 +1,14 @@
+apply plugin: 'java'
+
+dependencies {
+ compile project(':solr:core')
+ compile project(':solr:contrib:dataimporthandler')
+ compile library.activation
+ compile library.javax_mail
+ compile library.gimap
+ compile library.tika_core
+ compile library.tika_parsers
+
+ testCompile project(':solr:test-framework')
+ testCompile project(path: ':solr:contrib:dataimporthandler', configuration: 'testOutput')
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java b/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java
deleted file mode 100644
index 54f5984..0000000
--- a/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java
+++ /dev/null
@@ -1,888 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import com.sun.mail.imap.IMAPMessage;
-
-import org.apache.solr.common.util.SuppressForbidden;
-import org.apache.solr.handler.dataimport.config.ConfigNameConstants;
-import org.apache.solr.util.RTimer;
-import org.apache.tika.Tika;
-import org.apache.tika.metadata.Metadata;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.mail.*;
-import javax.mail.internet.AddressException;
-import javax.mail.internet.ContentType;
-import javax.mail.internet.InternetAddress;
-import javax.mail.internet.MimeMessage;
-import javax.mail.search.*;
-
-import java.io.InputStream;
-import java.lang.invoke.MethodHandles;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.function.Supplier;
-
-import com.sun.mail.gimap.GmailFolder;
-import com.sun.mail.gimap.GmailRawSearchTerm;
-
-/**
- * An EntityProcessor instance which can index emails along with their
- * attachments from POP3 or IMAP sources. Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler"
- * >http://wiki.apache.org/solr/DataImportHandler</a> for more details. <b>This
- * API is experimental and subject to change</b>
- *
- * @since solr 1.4
- */
-public class MailEntityProcessor extends EntityProcessorBase {
-
- private static final SimpleDateFormat sinceDateParser =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT);
- private static final SimpleDateFormat afterFmt =
- new SimpleDateFormat("yyyy/MM/dd", Locale.ROOT);
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public static interface CustomFilter {
- public SearchTerm getCustomSearch(Folder folder);
- }
-
- public void init(Context context) {
- super.init(context);
- // set attributes using XXX getXXXFromContext(attribute, defualtValue);
- // applies variable resolver and return default if value is not found or null
- // REQUIRED : connection and folder info
- user = getStringFromContext("user", null);
- password = getStringFromContext("password", null);
- host = getStringFromContext("host", null);
- protocol = getStringFromContext("protocol", null);
- folderNames = getStringFromContext("folders", null);
- // validate
- if (host == null || protocol == null || user == null || password == null
- || folderNames == null) throw new DataImportHandlerException(
- DataImportHandlerException.SEVERE,
- "'user|password|protocol|host|folders' are required attributes");
-
- // OPTIONAL : have defaults and are optional
- recurse = getBoolFromContext("recurse", true);
-
- exclude.clear();
- String excludes = getStringFromContext("exclude", "");
- if (excludes != null && !excludes.trim().equals("")) {
- exclude = Arrays.asList(excludes.split(","));
- }
-
- include.clear();
- String includes = getStringFromContext("include", "");
- if (includes != null && !includes.trim().equals("")) {
- include = Arrays.asList(includes.split(","));
- }
- batchSize = getIntFromContext("batchSize", 20);
- customFilter = getStringFromContext("customFilter", "");
- if (filters != null) filters.clear();
- folderIter = null;
- msgIter = null;
-
- String lastIndexTime = null;
- String command =
- String.valueOf(context.getRequestParameters().get("command"));
- if (!DataImporter.FULL_IMPORT_CMD.equals(command))
- throw new IllegalArgumentException(this.getClass().getSimpleName()+
- " only supports "+DataImporter.FULL_IMPORT_CMD);
-
- // Read the last_index_time out of the dataimport.properties if available
- String cname = getStringFromContext("name", "mailimporter");
- String varName = ConfigNameConstants.IMPORTER_NS_SHORT + "." + cname + "."
- + DocBuilder.LAST_INDEX_TIME;
- Object varValue = context.getVariableResolver().resolve(varName);
- log.info(varName+"="+varValue);
-
- if (varValue != null && !"".equals(varValue) &&
- !"".equals(getStringFromContext("fetchMailsSince", ""))) {
-
- // need to check if varValue is the epoch, which we'll take to mean the
- // initial value, in which case means we should use fetchMailsSince instead
- Date tmp = null;
- try {
- tmp = sinceDateParser.parse((String)varValue);
- if (tmp.getTime() == 0) {
- log.info("Ignoring initial value "+varValue+" for "+varName+
- " in favor of fetchMailsSince config parameter");
- tmp = null; // don't use this value
- }
- } catch (ParseException e) {
- // probably ok to ignore this since we have other options below
- // as we're just trying to figure out if the date is 0
- log.warn("Failed to parse "+varValue+" from "+varName+" due to: "+e);
- }
-
- if (tmp == null) {
- // favor fetchMailsSince in this case because the value from
- // dataimport.properties is the default/init value
- varValue = getStringFromContext("fetchMailsSince", "");
- log.info("fetchMailsSince="+varValue);
- }
- }
-
- if (varValue == null || "".equals(varValue)) {
- varName = ConfigNameConstants.IMPORTER_NS_SHORT + "."
- + DocBuilder.LAST_INDEX_TIME;
- varValue = context.getVariableResolver().resolve(varName);
- log.info(varName+"="+varValue);
- }
-
- if (varValue != null && varValue instanceof String) {
- lastIndexTime = (String)varValue;
- if (lastIndexTime != null && lastIndexTime.length() == 0)
- lastIndexTime = null;
- }
-
- if (lastIndexTime == null)
- lastIndexTime = getStringFromContext("fetchMailsSince", "");
-
- log.info("Using lastIndexTime "+lastIndexTime+" for mail import");
-
- this.fetchMailsSince = null;
- if (lastIndexTime != null && lastIndexTime.length() > 0) {
- try {
- fetchMailsSince = sinceDateParser.parse(lastIndexTime);
- log.info("Parsed fetchMailsSince=" + lastIndexTime);
- } catch (ParseException e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Invalid value for fetchMailSince: " + lastIndexTime, e);
- }
- }
-
- fetchSize = getIntFromContext("fetchSize", 32 * 1024);
- cTimeout = getIntFromContext("connectTimeout", 30 * 1000);
- rTimeout = getIntFromContext("readTimeout", 60 * 1000);
-
- String tmp = context.getEntityAttribute("includeOtherUserFolders");
- includeOtherUserFolders = (tmp != null && Boolean.valueOf(tmp.trim()));
- tmp = context.getEntityAttribute("includeSharedFolders");
- includeSharedFolders = (tmp != null && Boolean.valueOf(tmp.trim()));
-
- setProcessAttachmentConfig();
- includeContent = getBoolFromContext("includeContent", true);
-
- logConfig();
- }
-
- private void setProcessAttachmentConfig() {
- processAttachment = true;
- String tbval = context.getEntityAttribute("processAttachments");
- if (tbval == null) {
- tbval = context.getEntityAttribute("processAttachement");
- if (tbval != null) processAttachment = Boolean.valueOf(tbval);
- } else processAttachment = Boolean.valueOf(tbval);
- }
-
- @Override
- public Map<String,Object> nextRow() {
- Message mail = null;
- Map<String,Object> row = null;
- do {
- // try till there is a valid document or folders get exhausted.
- // when mail == NULL, it means end of processing
- mail = getNextMail();
-
- if (mail != null)
- row = getDocumentFromMail(mail);
-
- if (row != null && row.get("folder") == null)
- row.put("folder", mail.getFolder().getFullName());
-
- } while (row == null && mail != null);
- return row;
- }
-
- private Message getNextMail() {
- if (!connected) {
- // this is needed to load the activation mail stuff correctly
- // otherwise, the JavaMail multipart support doesn't get configured
- // correctly, which leads to a class cast exception when processing
- // multipart messages: IMAPInputStream cannot be cast to
- // javax.mail.Multipart
- if (false == withContextClassLoader(getClass().getClassLoader(), this::connectToMailBox)) {
- return null;
- }
- connected = true;
- }
- if (folderIter == null) {
- createFilters();
- folderIter = new FolderIterator(mailbox);
- }
- // get next message from the folder
- // if folder is exhausted get next folder
- // loop till a valid mail or all folders exhausted.
- while (msgIter == null || !msgIter.hasNext()) {
- Folder next = folderIter.hasNext() ? folderIter.next() : null;
- if (next == null) return null;
-
- msgIter = new MessageIterator(next, batchSize);
- }
- return msgIter.next();
- }
-
- private Map<String,Object> getDocumentFromMail(Message mail) {
- Map<String,Object> row = new HashMap<>();
- try {
- addPartToDocument(mail, row, true);
- return row;
- } catch (Exception e) {
- log.error("Failed to convert message [" + mail.toString()
- + "] to document due to: " + e, e);
- return null;
- }
- }
-
- public void addPartToDocument(Part part, Map<String,Object> row, boolean outerMost) throws Exception {
- if (part instanceof Message) {
- addEnvelopeToDocument(part, row);
- }
-
- String ct = part.getContentType().toLowerCase(Locale.ROOT);
- ContentType ctype = new ContentType(ct);
- if (part.isMimeType("multipart/*")) {
- Object content = part.getContent();
- if (content != null && content instanceof Multipart) {
- Multipart mp = (Multipart) part.getContent();
- int count = mp.getCount();
- if (part.isMimeType("multipart/alternative")) count = 1;
- for (int i = 0; i < count; i++)
- addPartToDocument(mp.getBodyPart(i), row, false);
- } else {
- log.warn("Multipart content is a not an instance of Multipart! Content is: "
- + (content != null ? content.getClass().getName() : "null")
- + ". Typically, this is due to the Java Activation JAR being loaded by the wrong classloader.");
- }
- } else if (part.isMimeType("message/rfc822")) {
- addPartToDocument((Part) part.getContent(), row, false);
- } else {
- String disp = part.getDisposition();
- if (includeContent
- && !(disp != null && disp.equalsIgnoreCase(Part.ATTACHMENT))) {
- InputStream is = part.getInputStream();
- Metadata contentTypeHint = new Metadata();
- contentTypeHint.set(Metadata.CONTENT_TYPE, ctype.getBaseType()
- .toLowerCase(Locale.ENGLISH));
- String content = (new Tika()).parseToString(is, contentTypeHint);
- if (row.get(CONTENT) == null) row.put(CONTENT, new ArrayList<String>());
- List<String> contents = (List<String>) row.get(CONTENT);
- contents.add(content.trim());
- row.put(CONTENT, contents);
- }
- if (!processAttachment || disp == null
- || !disp.equalsIgnoreCase(Part.ATTACHMENT)) return;
- InputStream is = part.getInputStream();
- String fileName = part.getFileName();
- Metadata contentTypeHint = new Metadata();
- contentTypeHint.set(Metadata.CONTENT_TYPE, ctype.getBaseType()
- .toLowerCase(Locale.ENGLISH));
- String content = (new Tika()).parseToString(is, contentTypeHint);
- if (content == null || content.trim().length() == 0) return;
-
- if (row.get(ATTACHMENT) == null) row.put(ATTACHMENT,
- new ArrayList<String>());
- List<String> contents = (List<String>) row.get(ATTACHMENT);
- contents.add(content.trim());
- row.put(ATTACHMENT, contents);
- if (row.get(ATTACHMENT_NAMES) == null) row.put(ATTACHMENT_NAMES,
- new ArrayList<String>());
- List<String> names = (List<String>) row.get(ATTACHMENT_NAMES);
- names.add(fileName);
- row.put(ATTACHMENT_NAMES, names);
- }
- }
-
- private void addEnvelopeToDocument(Part part, Map<String,Object> row)
- throws MessagingException {
- MimeMessage mail = (MimeMessage) part;
- Address[] adresses;
- if ((adresses = mail.getFrom()) != null && adresses.length > 0) row.put(
- FROM, adresses[0].toString());
-
- List<String> to = new ArrayList<>();
- if ((adresses = mail.getRecipients(Message.RecipientType.TO)) != null) addAddressToList(
- adresses, to);
- if ((adresses = mail.getRecipients(Message.RecipientType.CC)) != null) addAddressToList(
- adresses, to);
- if ((adresses = mail.getRecipients(Message.RecipientType.BCC)) != null) addAddressToList(
- adresses, to);
- if (to.size() > 0) row.put(TO_CC_BCC, to);
-
- row.put(MESSAGE_ID, mail.getMessageID());
- row.put(SUBJECT, mail.getSubject());
-
- Date d = mail.getSentDate();
- if (d != null) {
- row.put(SENT_DATE, d);
- }
-
- List<String> flags = new ArrayList<>();
- for (Flags.Flag flag : mail.getFlags().getSystemFlags()) {
- if (flag == Flags.Flag.ANSWERED) flags.add(FLAG_ANSWERED);
- else if (flag == Flags.Flag.DELETED) flags.add(FLAG_DELETED);
- else if (flag == Flags.Flag.DRAFT) flags.add(FLAG_DRAFT);
- else if (flag == Flags.Flag.FLAGGED) flags.add(FLAG_FLAGGED);
- else if (flag == Flags.Flag.RECENT) flags.add(FLAG_RECENT);
- else if (flag == Flags.Flag.SEEN) flags.add(FLAG_SEEN);
- }
- flags.addAll(Arrays.asList(mail.getFlags().getUserFlags()));
- if (flags.size() == 0) flags.add(FLAG_NONE);
- row.put(FLAGS, flags);
-
- String[] hdrs = mail.getHeader("X-Mailer");
- if (hdrs != null) row.put(XMAILER, hdrs[0]);
- }
-
- private void addAddressToList(Address[] adresses, List<String> to)
- throws AddressException {
- for (Address address : adresses) {
- to.add(address.toString());
- InternetAddress ia = (InternetAddress) address;
- if (ia.isGroup()) {
- InternetAddress[] group = ia.getGroup(false);
- for (InternetAddress member : group)
- to.add(member.toString());
- }
- }
- }
-
- private boolean connectToMailBox() {
- try {
- Properties props = new Properties();
- if (System.getProperty("mail.debug") != null)
- props.setProperty("mail.debug", System.getProperty("mail.debug"));
-
- if (("imap".equals(protocol) || "imaps".equals(protocol))
- && "imap.gmail.com".equals(host)) {
- log.info("Consider using 'gimaps' protocol instead of '" + protocol
- + "' for enabling GMail specific extensions for " + host);
- }
-
- props.setProperty("mail.store.protocol", protocol);
-
- String imapPropPrefix = protocol.startsWith("gimap") ? "gimap" : "imap";
- props.setProperty("mail." + imapPropPrefix + ".fetchsize", "" + fetchSize);
- props.setProperty("mail." + imapPropPrefix + ".timeout", "" + rTimeout);
- props.setProperty("mail." + imapPropPrefix + ".connectiontimeout", "" + cTimeout);
-
- int port = -1;
- int colonAt = host.indexOf(":");
- if (colonAt != -1) {
- port = Integer.parseInt(host.substring(colonAt + 1));
- host = host.substring(0, colonAt);
- }
-
- Session session = Session.getDefaultInstance(props, null);
- mailbox = session.getStore(protocol);
- if (port != -1) {
- mailbox.connect(host, port, user, password);
- } else {
- mailbox.connect(host, user, password);
- }
- log.info("Connected to " + user + "'s mailbox on " + host);
-
- return true;
- } catch (MessagingException e) {
- String errMsg = String.format(Locale.ENGLISH,
- "Failed to connect to %s server %s as user %s due to: %s", protocol,
- host, user, e.toString());
- log.error(errMsg, e);
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- errMsg, e);
- }
- }
-
- private void createFilters() {
- if (fetchMailsSince != null) {
- filters.add(new MailsSinceLastCheckFilter(fetchMailsSince));
- }
- if (customFilter != null && !customFilter.equals("")) {
- try {
- Class cf = Class.forName(customFilter);
- Object obj = cf.newInstance();
- if (obj instanceof CustomFilter) {
- filters.add((CustomFilter) obj);
- }
- } catch (Exception e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Custom filter could not be created", e);
- }
- }
- }
-
- private void logConfig() {
- if (!log.isInfoEnabled()) return;
-
- String lineSep = System.getProperty("line.separator");
-
- StringBuffer config = new StringBuffer();
- config.append("user : ").append(user).append(lineSep);
- config
- .append("pwd : ")
- .append(
- password != null && password.length() > 0 ? "<non-null>" : "<null>")
- .append(lineSep);
- config.append("protocol : ").append(protocol)
- .append(lineSep);
- config.append("host : ").append(host)
- .append(lineSep);
- config.append("folders : ").append(folderNames)
- .append(lineSep);
- config.append("recurse : ").append(recurse)
- .append(lineSep);
- config.append("exclude : ").append(exclude.toString())
- .append(lineSep);
- config.append("include : ").append(include.toString())
- .append(lineSep);
- config.append("batchSize : ").append(batchSize)
- .append(lineSep);
- config.append("fetchSize : ").append(fetchSize)
- .append(lineSep);
- config.append("read timeout : ").append(rTimeout)
- .append(lineSep);
- config.append("conection timeout : ").append(cTimeout)
- .append(lineSep);
- config.append("custom filter : ").append(customFilter)
- .append(lineSep);
- config.append("fetch mail since : ").append(fetchMailsSince)
- .append(lineSep);
- config.append("includeContent : ").append(includeContent)
- .append(lineSep);
- config.append("processAttachments : ").append(processAttachment)
- .append(lineSep);
- config.append("includeOtherUserFolders : ").append(includeOtherUserFolders)
- .append(lineSep);
- config.append("includeSharedFolders : ").append(includeSharedFolders)
- .append(lineSep);
- log.info(config.toString());
- }
-
- class FolderIterator implements Iterator<Folder> {
- private Store mailbox;
- private List<String> topLevelFolders;
- private List<Folder> folders = null;
- private Folder lastFolder = null;
-
- public FolderIterator(Store mailBox) {
- this.mailbox = mailBox;
- folders = new ArrayList<>();
- getTopLevelFolders(mailBox);
- if (includeOtherUserFolders) getOtherUserFolders();
- if (includeSharedFolders) getSharedFolders();
- }
-
- public boolean hasNext() {
- return !folders.isEmpty();
- }
-
- public Folder next() {
- try {
- boolean hasMessages = false;
- Folder next;
- do {
- if (lastFolder != null) {
- lastFolder.close(false);
- lastFolder = null;
- }
- if (folders.isEmpty()) {
- mailbox.close();
- return null;
- }
- next = folders.remove(0);
- if (next != null) {
- String fullName = next.getFullName();
- if (!excludeFolder(fullName)) {
- hasMessages = (next.getType() & Folder.HOLDS_MESSAGES) != 0;
- next.open(Folder.READ_ONLY);
- lastFolder = next;
- log.info("Opened folder : " + fullName);
- }
- if (recurse && ((next.getType() & Folder.HOLDS_FOLDERS) != 0)) {
- Folder[] children = next.list();
- log.info("Added its children to list : ");
- for (int i = children.length - 1; i >= 0; i--) {
- folders.add(0, children[i]);
- log.info("child name : " + children[i].getFullName());
- }
- if (children.length == 0) log.info("NO children : ");
- }
- }
- } while (!hasMessages);
- return next;
- } catch (Exception e) {
- log.warn("Failed to read folders due to: "+e);
- // throw new
- // DataImportHandlerException(DataImportHandlerException.SEVERE,
- // "Folder open failed", e);
- }
- return null;
- }
-
- public void remove() {
- throw new UnsupportedOperationException("It's read only mode...");
- }
-
- private void getTopLevelFolders(Store mailBox) {
- if (folderNames != null) topLevelFolders = Arrays.asList(folderNames
- .split(","));
- for (int i = 0; topLevelFolders != null && i < topLevelFolders.size(); i++) {
- try {
- folders.add(mailbox.getFolder(topLevelFolders.get(i)));
- } catch (MessagingException e) {
- // skip bad ones unless it's the last one and still no good folder
- if (folders.size() == 0 && i == topLevelFolders.size() - 1) throw new DataImportHandlerException(
- DataImportHandlerException.SEVERE, "Folder retreival failed");
- }
- }
- if (topLevelFolders == null || topLevelFolders.size() == 0) {
- try {
- folders.add(mailBox.getDefaultFolder());
- } catch (MessagingException e) {
- throw new DataImportHandlerException(
- DataImportHandlerException.SEVERE, "Folder retreival failed");
- }
- }
- }
-
- private void getOtherUserFolders() {
- try {
- Folder[] ufldrs = mailbox.getUserNamespaces(null);
- if (ufldrs != null) {
- log.info("Found " + ufldrs.length + " user namespace folders");
- for (Folder ufldr : ufldrs)
- folders.add(ufldr);
- }
- } catch (MessagingException me) {
- log.warn("Messaging exception retrieving user namespaces: "
- + me.getMessage());
- }
- }
-
- private void getSharedFolders() {
- try {
- Folder[] sfldrs = mailbox.getSharedNamespaces();
- if (sfldrs != null) {
- log.info("Found " + sfldrs.length + " shared namespace folders");
- for (Folder sfldr : sfldrs)
- folders.add(sfldr);
- }
- } catch (MessagingException me) {
- log.warn("Messaging exception retrieving shared namespaces: "
- + me.getMessage());
- }
- }
-
- private boolean excludeFolder(String name) {
- for (String s : exclude) {
- if (name.matches(s)) return true;
- }
- for (String s : include) {
- if (name.matches(s)) return false;
- }
- return include.size() > 0;
- }
- }
-
- class MessageIterator extends SearchTerm implements Iterator<Message> {
- private Folder folder;
- private Message[] messagesInCurBatch = null;
- private int current = 0;
- private int currentBatch = 0;
- private int batchSize = 0;
- private int totalInFolder = 0;
- private boolean doBatching = true;
-
- public MessageIterator(Folder folder, int batchSize) {
- super();
-
- try {
- this.folder = folder;
- this.batchSize = batchSize;
- SearchTerm st = getSearchTerm();
-
- log.info("SearchTerm=" + st);
-
- if (st != null || folder instanceof GmailFolder) {
- doBatching = false;
- // Searching can still take a while even though we're only pulling
- // envelopes; unless you're using gmail server-side filter, which is
- // fast
- log.info("Searching folder " + folder.getName() + " for messages");
- final RTimer searchTimer = new RTimer();
-
- // If using GMail, speed up the envelope processing by doing a
- // server-side
- // search for messages occurring on or after the fetch date (at
- // midnight),
- // which reduces the number of envelopes we need to pull from the
- // server
- // to apply the precise DateTerm filter; GMail server-side search has
- // date
- // granularity only but the local filters are also applied
-
- if (folder instanceof GmailFolder && fetchMailsSince != null) {
- String afterCrit = "after:" + afterFmt.format(fetchMailsSince);
- log.info("Added server-side gmail filter: " + afterCrit);
- Message[] afterMessages = folder.search(new GmailRawSearchTerm(
- afterCrit));
-
- log.info("GMail server-side filter found " + afterMessages.length
- + " messages received " + afterCrit + " in folder " + folder.getName());
-
- // now pass in the server-side filtered messages to the local filter
- messagesInCurBatch = folder.search((st != null ? st : this), afterMessages);
- } else {
- messagesInCurBatch = folder.search(st);
- }
- totalInFolder = messagesInCurBatch.length;
- folder.fetch(messagesInCurBatch, fp);
- current = 0;
- log.info("Total messages : " + totalInFolder);
- log.info("Search criteria applied. Batching disabled. Took {} (ms)", searchTimer.getTime());
- } else {
- totalInFolder = folder.getMessageCount();
- log.info("Total messages : " + totalInFolder);
- getNextBatch(batchSize, folder);
- }
- } catch (MessagingException e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Message retreival failed", e);
- }
- }
-
- private void getNextBatch(int batchSize, Folder folder)
- throws MessagingException {
- // after each batch invalidate cache
- if (messagesInCurBatch != null) {
- for (Message m : messagesInCurBatch) {
- if (m instanceof IMAPMessage) ((IMAPMessage) m).invalidateHeaders();
- }
- }
- int lastMsg = (currentBatch + 1) * batchSize;
- lastMsg = lastMsg > totalInFolder ? totalInFolder : lastMsg;
- messagesInCurBatch = folder.getMessages(currentBatch * batchSize + 1,
- lastMsg);
- folder.fetch(messagesInCurBatch, fp);
- current = 0;
- currentBatch++;
- log.info("Current Batch : " + currentBatch);
- log.info("Messages in this batch : " + messagesInCurBatch.length);
- }
-
- public boolean hasNext() {
- boolean hasMore = current < messagesInCurBatch.length;
- if (!hasMore && doBatching && currentBatch * batchSize < totalInFolder) {
- // try next batch
- try {
- getNextBatch(batchSize, folder);
- hasMore = current < messagesInCurBatch.length;
- } catch (MessagingException e) {
- throw new DataImportHandlerException(
- DataImportHandlerException.SEVERE, "Message retreival failed", e);
- }
- }
- return hasMore;
- }
-
- public Message next() {
- return hasNext() ? messagesInCurBatch[current++] : null;
- }
-
- public void remove() {
- throw new UnsupportedOperationException("It's read only mode...");
- }
-
- private SearchTerm getSearchTerm() {
- if (filters.size() == 0) return null;
- if (filters.size() == 1) return filters.get(0).getCustomSearch(folder);
- SearchTerm last = filters.get(0).getCustomSearch(folder);
- for (int i = 1; i < filters.size(); i++) {
- CustomFilter filter = filters.get(i);
- SearchTerm st = filter.getCustomSearch(folder);
- if (st != null) {
- last = new AndTerm(last, st);
- }
- }
- return last;
- }
-
- public boolean match(Message message) {
- return true;
- }
- }
-
- static class MailsSinceLastCheckFilter implements CustomFilter {
-
- private Date since;
-
- public MailsSinceLastCheckFilter(Date date) {
- since = date;
- }
-
- @SuppressWarnings("serial")
- public SearchTerm getCustomSearch(final Folder folder) {
- log.info("Building mail filter for messages in " + folder.getName()
- + " that occur after " + sinceDateParser.format(since));
- return new DateTerm(ComparisonTerm.GE, since) {
- private int matched = 0;
- private int seen = 0;
-
- @Override
- public boolean match(Message msg) {
- boolean isMatch = false;
- ++seen;
- try {
- Date msgDate = msg.getReceivedDate();
- if (msgDate == null) msgDate = msg.getSentDate();
-
- if (msgDate != null && msgDate.getTime() >= since.getTime()) {
- ++matched;
- isMatch = true;
- } else {
- String msgDateStr = (msgDate != null) ? sinceDateParser.format(msgDate) : "null";
- String sinceDateStr = (since != null) ? sinceDateParser.format(since) : "null";
- log.debug("Message " + msg.getSubject() + " was received at [" + msgDateStr
- + "], since filter is [" + sinceDateStr + "]");
- }
- } catch (MessagingException e) {
- log.warn("Failed to process message due to: "+e, e);
- }
-
- if (seen % 100 == 0) {
- log.info("Matched " + matched + " of " + seen + " messages since: "
- + sinceDateParser.format(since));
- }
-
- return isMatch;
- }
- };
- }
- }
-
- // user settings stored in member variables
- private String user;
- private String password;
- private String host;
- private String protocol;
-
- private String folderNames;
- private List<String> exclude = new ArrayList<>();
- private List<String> include = new ArrayList<>();
- private boolean recurse;
-
- private int batchSize;
- private int fetchSize;
- private int cTimeout;
- private int rTimeout;
-
- private Date fetchMailsSince;
- private String customFilter;
-
- private boolean processAttachment = true;
- private boolean includeContent = true;
- private boolean includeOtherUserFolders = false;
- private boolean includeSharedFolders = false;
-
- // holds the current state
- private Store mailbox;
- private boolean connected = false;
- private FolderIterator folderIter;
- private MessageIterator msgIter;
- private List<CustomFilter> filters = new ArrayList<>();
- private static FetchProfile fp = new FetchProfile();
-
- static {
- fp.add(FetchProfile.Item.ENVELOPE);
- fp.add(FetchProfile.Item.FLAGS);
- fp.add("X-Mailer");
- }
-
- // Fields To Index
- // single valued
- private static final String MESSAGE_ID = "messageId";
- private static final String SUBJECT = "subject";
- private static final String FROM = "from";
- private static final String SENT_DATE = "sentDate";
- private static final String XMAILER = "xMailer";
- // multi valued
- private static final String TO_CC_BCC = "allTo";
- private static final String FLAGS = "flags";
- private static final String CONTENT = "content";
- private static final String ATTACHMENT = "attachment";
- private static final String ATTACHMENT_NAMES = "attachmentNames";
- // flag values
- private static final String FLAG_NONE = "none";
- private static final String FLAG_ANSWERED = "answered";
- private static final String FLAG_DELETED = "deleted";
- private static final String FLAG_DRAFT = "draft";
- private static final String FLAG_FLAGGED = "flagged";
- private static final String FLAG_RECENT = "recent";
- private static final String FLAG_SEEN = "seen";
-
- private int getIntFromContext(String prop, int ifNull) {
- int v = ifNull;
- try {
- String val = context.getEntityAttribute(prop);
- if (val != null) {
- val = context.replaceTokens(val);
- v = Integer.parseInt(val);
- }
- } catch (NumberFormatException e) {
- // do nothing
- }
- return v;
- }
-
- private boolean getBoolFromContext(String prop, boolean ifNull) {
- boolean v = ifNull;
- String val = context.getEntityAttribute(prop);
- if (val != null) {
- val = context.replaceTokens(val);
- v = Boolean.valueOf(val);
- }
- return v;
- }
-
- private String getStringFromContext(String prop, String ifNull) {
- String v = ifNull;
- String val = context.getEntityAttribute(prop);
- if (val != null) {
- val = context.replaceTokens(val);
- v = val;
- }
- return v;
- }
-
- @SuppressForbidden(reason = "Uses context class loader as a workaround to inject correct classloader to 3rd party libs")
- private static <T> T withContextClassLoader(ClassLoader loader, Supplier<T> action) {
- Thread ct = Thread.currentThread();
- ClassLoader prev = ct.getContextClassLoader();
- try {
- ct.setContextClassLoader(loader);
- return action.get();
- } finally {
- ct.setContextClassLoader(prev);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java b/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java
deleted file mode 100644
index 875032b..0000000
--- a/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.AutoDetectParser;
-import org.apache.tika.parser.EmptyParser;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
-import org.apache.tika.parser.html.HtmlMapper;
-import org.apache.tika.parser.html.IdentityHtmlMapper;
-import org.apache.tika.sax.BodyContentHandler;
-import org.apache.tika.sax.ContentHandlerDecorator;
-import org.apache.tika.sax.XHTMLContentHandler;
-import org.xml.sax.Attributes;
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-import org.xml.sax.helpers.DefaultHandler;
-
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.TransformerConfigurationException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.sax.SAXTransformerFactory;
-import javax.xml.transform.sax.TransformerHandler;
-import javax.xml.transform.stream.StreamResult;
-import java.io.File;
-import java.io.InputStream;
-import java.io.StringWriter;
-import java.io.Writer;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-import static org.apache.solr.handler.dataimport.DataImporter.COLUMN;
-import static org.apache.solr.handler.dataimport.XPathEntityProcessor.URL;
-/**
- * <p>An implementation of {@link EntityProcessor} which reads data from rich docs
- * using <a href="http://tika.apache.org/">Apache Tika</a>
- *
- * <p>To index latitude/longitude data that might
- * be extracted from a file's metadata, identify
- * the geo field for this information with this attribute:
- * <code>spatialMetadataField</code>
- *
- * @since solr 3.1
- */
-public class TikaEntityProcessor extends EntityProcessorBase {
- private static Parser EMPTY_PARSER = new EmptyParser();
- private TikaConfig tikaConfig;
- private String format = "text";
- private boolean done = false;
- private boolean extractEmbedded = false;
- private String parser;
- static final String AUTO_PARSER = "org.apache.tika.parser.AutoDetectParser";
- private String htmlMapper;
- private String spatialMetadataField;
-
- @Override
- public void init(Context context) {
- super.init(context);
- done = false;
- }
-
- @Override
- protected void firstInit(Context context) {
- super.firstInit(context);
- try {
- String tikaConfigFile = context.getResolvedEntityAttribute("tikaConfig");
- if (tikaConfigFile == null) {
- ClassLoader classLoader = context.getSolrCore().getResourceLoader().getClassLoader();
- try (InputStream is = classLoader.getResourceAsStream("solr-default-tika-config.xml")) {
- tikaConfig = new TikaConfig(is);
- }
- } else {
- File configFile = new File(tikaConfigFile);
- if (!configFile.isAbsolute()) {
- configFile = new File(context.getSolrCore().getResourceLoader().getConfigDir(), tikaConfigFile);
- }
- tikaConfig = new TikaConfig(configFile);
- }
- } catch (Exception e) {
- wrapAndThrow (SEVERE, e,"Unable to load Tika Config");
- }
-
- String extractEmbeddedString = context.getResolvedEntityAttribute("extractEmbedded");
- if ("true".equals(extractEmbeddedString)) {
- extractEmbedded = true;
- }
- format = context.getResolvedEntityAttribute("format");
- if(format == null)
- format = "text";
- if (!"html".equals(format) && !"xml".equals(format) && !"text".equals(format)&& !"none".equals(format) )
- throw new DataImportHandlerException(SEVERE, "'format' can be one of text|html|xml|none");
-
- htmlMapper = context.getResolvedEntityAttribute("htmlMapper");
- if (htmlMapper == null)
- htmlMapper = "default";
- if (!"default".equals(htmlMapper) && !"identity".equals(htmlMapper))
- throw new DataImportHandlerException(SEVERE, "'htmlMapper', if present, must be 'default' or 'identity'");
-
- parser = context.getResolvedEntityAttribute("parser");
- if(parser == null) {
- parser = AUTO_PARSER;
- }
-
- spatialMetadataField = context.getResolvedEntityAttribute("spatialMetadataField");
- }
-
- @Override
- public Map<String, Object> nextRow() {
- if(done) return null;
- Map<String, Object> row = new HashMap<>();
- DataSource<InputStream> dataSource = context.getDataSource();
- InputStream is = dataSource.getData(context.getResolvedEntityAttribute(URL));
- ContentHandler contentHandler = null;
- Metadata metadata = new Metadata();
- StringWriter sw = new StringWriter();
- try {
- if ("html".equals(format)) {
- contentHandler = getHtmlHandler(sw);
- } else if ("xml".equals(format)) {
- contentHandler = getXmlContentHandler(sw);
- } else if ("text".equals(format)) {
- contentHandler = getTextContentHandler(sw);
- } else if("none".equals(format)){
- contentHandler = new DefaultHandler();
- }
- } catch (TransformerConfigurationException e) {
- wrapAndThrow(SEVERE, e, "Unable to create content handler");
- }
- Parser tikaParser = null;
- if(parser.equals(AUTO_PARSER)){
- tikaParser = new AutoDetectParser(tikaConfig);
- } else {
- tikaParser = context.getSolrCore().getResourceLoader().newInstance(parser, Parser.class);
- }
- try {
- ParseContext context = new ParseContext();
- if ("identity".equals(htmlMapper)){
- context.set(HtmlMapper.class, IdentityHtmlMapper.INSTANCE);
- }
- if (extractEmbedded) {
- context.set(Parser.class, tikaParser);
- } else {
- context.set(Parser.class, EMPTY_PARSER);
- }
- tikaParser.parse(is, contentHandler, metadata , context);
- } catch (Exception e) {
- if(SKIP.equals(onError)) {
- throw new DataImportHandlerException(DataImportHandlerException.SKIP_ROW,
- "Document skipped :" + e.getMessage());
- }
- wrapAndThrow(SEVERE, e, "Unable to read content");
- }
- IOUtils.closeQuietly(is);
- for (Map<String, String> field : context.getAllEntityFields()) {
- if (!"true".equals(field.get("meta"))) continue;
- String col = field.get(COLUMN);
- String s = metadata.get(col);
- if (s != null) row.put(col, s);
- }
- if(!"none".equals(format) ) row.put("text", sw.toString());
- tryToAddLatLon(metadata, row);
- done = true;
- return row;
- }
-
- private void tryToAddLatLon(Metadata metadata, Map<String, Object> row) {
- if (spatialMetadataField == null) return;
- String latString = metadata.get(Metadata.LATITUDE);
- String lonString = metadata.get(Metadata.LONGITUDE);
- if (latString != null && lonString != null) {
- row.put(spatialMetadataField, String.format(Locale.ROOT, "%s,%s", latString, lonString));
- }
- }
-
- private static ContentHandler getHtmlHandler(Writer writer)
- throws TransformerConfigurationException {
- SAXTransformerFactory factory = (SAXTransformerFactory)
- TransformerFactory.newInstance();
- TransformerHandler handler = factory.newTransformerHandler();
- handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "html");
- handler.setResult(new StreamResult(writer));
- return new ContentHandlerDecorator(handler) {
- @Override
- public void startElement(
- String uri, String localName, String name, Attributes atts)
- throws SAXException {
- if (XHTMLContentHandler.XHTML.equals(uri)) {
- uri = null;
- }
- if (!"head".equals(localName)) {
- super.startElement(uri, localName, name, atts);
- }
- }
-
- @Override
- public void endElement(String uri, String localName, String name)
- throws SAXException {
- if (XHTMLContentHandler.XHTML.equals(uri)) {
- uri = null;
- }
- if (!"head".equals(localName)) {
- super.endElement(uri, localName, name);
- }
- }
-
- @Override
- public void startPrefixMapping(String prefix, String uri) {/*no op*/ }
-
- @Override
- public void endPrefixMapping(String prefix) {/*no op*/ }
- };
- }
-
- private static ContentHandler getTextContentHandler(Writer writer) {
- return new BodyContentHandler(writer);
- }
-
- private static ContentHandler getXmlContentHandler(Writer writer)
- throws TransformerConfigurationException {
- SAXTransformerFactory factory = (SAXTransformerFactory)
- TransformerFactory.newInstance();
- TransformerHandler handler = factory.newTransformerHandler();
- handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "xml");
- handler.setResult(new StreamResult(writer));
- return handler;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/package.html
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/package.html b/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/package.html
deleted file mode 100644
index 9a7f6f2..0000000
--- a/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/package.html
+++ /dev/null
@@ -1,23 +0,0 @@
-<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!-- not a package-info.java, because we already defined this package in core/ -->
-<html>
-<body>
-Plugins for <code>DataImportHandler</code> that have additional dependencies.
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler-extras/src/java/overview.html
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler-extras/src/java/overview.html b/solr/contrib/dataimporthandler-extras/src/java/overview.html
deleted file mode 100644
index a60a25e..0000000
--- a/solr/contrib/dataimporthandler-extras/src/java/overview.html
+++ /dev/null
@@ -1,21 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<html>
-<body>
-Apache Solr Search Server: DataImportHandler Extras contrib
-</body>
-</html>
[08/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java
new file mode 100644
index 0000000..91bdd00
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.Test;
+
+import java.text.DecimalFormatSymbols;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+/**
+ * <p>
+ * Test for NumberFormatTransformer
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestNumberFormatTransformer extends AbstractDataImportHandlerTestCase {
+ private char GROUPING_SEP = new DecimalFormatSymbols(Locale.ROOT).getGroupingSeparator();
+ private char DECIMAL_SEP = new DecimalFormatSymbols(Locale.ROOT).getDecimalSeparator();
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testTransformRow_SingleNumber() {
+ char GERMAN_GROUPING_SEP = new DecimalFormatSymbols(Locale.GERMANY).getGroupingSeparator();
+ List<Map<String, String>> l = new ArrayList<>();
+ l.add(createMap("column", "num",
+ NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.NUMBER));
+ l.add(createMap("column", "localizedNum",
+ NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.NUMBER, NumberFormatTransformer.LOCALE, "de-DE"));
+ Context c = getContext(null, null, null, Context.FULL_DUMP, l, null);
+ Map<String,Object> m = createMap("num", "123" + GROUPING_SEP + "567", "localizedNum", "123" + GERMAN_GROUPING_SEP + "567");
+ new NumberFormatTransformer().transformRow(m, c);
+ assertEquals(123567L, m.get("num"));
+ assertEquals(123567L, m.get("localizedNum"));
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testTransformRow_MultipleNumbers() throws Exception {
+ List<Map<String, String>> fields = new ArrayList<>();
+ fields.add(createMap(DataImporter.COLUMN, "inputs"));
+ fields.add(createMap(DataImporter.COLUMN,
+ "outputs", RegexTransformer.SRC_COL_NAME, "inputs",
+ NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.NUMBER));
+
+ List<String> inputs = new ArrayList<>();
+ inputs.add("123" + GROUPING_SEP + "567");
+ inputs.add("245" + GROUPING_SEP + "678");
+ Map<String, Object> row = createMap("inputs", inputs);
+
+ VariableResolver resolver = new VariableResolver();
+ resolver.addNamespace("e", row);
+
+ Context context = getContext(null, resolver, null, Context.FULL_DUMP, fields, null);
+ new NumberFormatTransformer().transformRow(row, context);
+
+ List<Long> output = new ArrayList<>();
+ output.add(123567L);
+ output.add(245678L);
+ Map<String, Object> outputRow = createMap("inputs", inputs, "outputs", output);
+
+ assertEquals(outputRow, row);
+ }
+
+ @Test(expected = DataImportHandlerException.class)
+ @SuppressWarnings("unchecked")
+ public void testTransformRow_InvalidInput1_Number() {
+ List<Map<String, String>> l = new ArrayList<>();
+ l.add(createMap("column", "num",
+ NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.NUMBER));
+ Context c = getContext(null, null, null, Context.FULL_DUMP, l, null);
+ Map<String, Object> m = createMap("num", "123" + GROUPING_SEP + "5a67");
+ new NumberFormatTransformer().transformRow(m, c);
+ }
+
+ @Test(expected = DataImportHandlerException.class)
+ @SuppressWarnings("unchecked")
+ public void testTransformRow_InvalidInput2_Number() {
+ List<Map<String, String>> l = new ArrayList<>();
+ l.add(createMap("column", "num",
+ NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.NUMBER));
+ Context c = getContext(null, null, null, Context.FULL_DUMP, l, null);
+ Map<String, Object> m = createMap("num", "123" + GROUPING_SEP + "567b");
+ new NumberFormatTransformer().transformRow(m, c);
+ }
+
+ @Test(expected = DataImportHandlerException.class)
+ @SuppressWarnings("unchecked")
+ public void testTransformRow_InvalidInput2_Currency() {
+ List<Map<String, String>> l = new ArrayList<>();
+ l.add(createMap("column", "num",
+ NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.CURRENCY));
+ Context c = getContext(null, null, null, Context.FULL_DUMP, l, null);
+ Map<String, Object> m = createMap("num", "123" + GROUPING_SEP + "567b");
+ new NumberFormatTransformer().transformRow(m, c);
+ }
+
+ @Test(expected = DataImportHandlerException.class)
+ @SuppressWarnings("unchecked")
+ public void testTransformRow_InvalidInput1_Percent() {
+ List<Map<String, String>> l = new ArrayList<>();
+ l.add(createMap("column", "num",
+ NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.PERCENT));
+ Context c = getContext(null, null, null, Context.FULL_DUMP, l, null);
+ Map<String, Object> m = createMap("num", "123" + GROUPING_SEP + "5a67");
+ new NumberFormatTransformer().transformRow(m, c);
+ }
+
+ @Test(expected = DataImportHandlerException.class)
+ @SuppressWarnings("unchecked")
+ public void testTransformRow_InvalidInput3_Currency() {
+ List<Map<String, String>> l = new ArrayList<>();
+ l.add(createMap("column", "num",
+ NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.CURRENCY));
+ Context c = getContext(null, null, null, Context.FULL_DUMP, l, null);
+ Map<String, Object> m = createMap("num", "123" + DECIMAL_SEP + "456" + DECIMAL_SEP + "789");
+ new NumberFormatTransformer().transformRow(m, c);
+ }
+
+ @Test(expected = DataImportHandlerException.class)
+ @SuppressWarnings("unchecked")
+ public void testTransformRow_InvalidInput3_Number() {
+ List<Map<String, String>> l = new ArrayList<>();
+ l.add(createMap("column", "num",
+ NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.NUMBER));
+ Context c = getContext(null, null, null, Context.FULL_DUMP, l, null);
+ Map<String, Object> m = createMap("num", "123" + DECIMAL_SEP + "456" + DECIMAL_SEP + "789");
+ new NumberFormatTransformer().transformRow(m, c);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testTransformRow_MalformedInput_Number() {
+ List<Map<String, String>> l = new ArrayList<>();
+ l.add(createMap("column", "num",
+ NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.NUMBER));
+ Context c = getContext(null, null, null, Context.FULL_DUMP, l, null);
+ Map<String, Object> m = createMap("num", "123" + GROUPING_SEP + GROUPING_SEP + "789");
+ new NumberFormatTransformer().transformRow(m, c);
+ assertEquals(123789L, m.get("num"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java
new file mode 100644
index 0000000..2385438
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.StringReader;
+import java.sql.Blob;
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.Properties;
+
+import org.apache.solr.common.util.Utils;
+import org.junit.Test;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * Test for PlainTextEntityProcessor
+ *
+ *
+ * @see org.apache.solr.handler.dataimport.PlainTextEntityProcessor
+ * @since solr 1.4
+ */
+public class TestPlainTextEntityProcessor extends AbstractDataImportHandlerTestCase {
+ @Test
+ public void testSimple() throws IOException {
+ DataImporter di = new DataImporter();
+ di.loadAndInit(DATA_CONFIG);
+ redirectTempProperties(di);
+
+ TestDocBuilder.SolrWriterImpl sw = new TestDocBuilder.SolrWriterImpl();
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
+ di.runCmd(rp, sw);
+ assertEquals(DS.s, sw.docs.get(0).getFieldValue("x"));
+ }
+
+ static class BlobImpl implements Blob{
+ private final byte[] bytes;
+
+ BlobImpl(byte[] bytes) {
+ this.bytes = bytes;
+ }
+
+ @Override
+ public long length() throws SQLException {
+ return 0;
+ }
+
+ @Override
+ public byte[] getBytes(long pos, int length) throws SQLException {
+ return bytes;
+ }
+
+ @Override
+ public InputStream getBinaryStream() throws SQLException {
+ return new ByteArrayInputStream(bytes);
+ }
+
+ @Override
+ public long position(byte[] pattern, long start) throws SQLException {
+ return 0;
+ }
+
+ @Override
+ public long position(Blob pattern, long start) throws SQLException {
+ return 0;
+ }
+
+ @Override
+ public int setBytes(long pos, byte[] bytes) throws SQLException {
+ return 0;
+ }
+
+ @Override
+ public int setBytes(long pos, byte[] bytes, int offset, int len) throws SQLException {
+ return 0;
+ }
+
+ @Override
+ public OutputStream setBinaryStream(long pos) throws SQLException {
+ return null;
+ }
+
+ @Override
+ public void truncate(long len) throws SQLException {
+
+ }
+
+ @Override
+ public void free() throws SQLException {
+
+ }
+
+ @Override
+ public InputStream getBinaryStream(long pos, long length) throws SQLException {
+ return new ByteArrayInputStream(bytes);
+ }
+ }
+
+ @Test
+ public void testSimple2() throws IOException {
+ DataImporter di = new DataImporter();
+ MockDataSource.setIterator("select id, name, blob_field from lw_table4", Collections.singletonList(Utils.makeMap("blob_field",new BlobImpl(DS.s.getBytes(UTF_8)) ) ).iterator());
+
+ String dc =
+
+ " <dataConfig>" +
+ "<dataSource name=\"ds1\" type=\"MockDataSource\"/>\n" +
+ " <!-- dataSource for FieldReaderDataSource -->\n" +
+ " <dataSource dataField=\"root.blob_field\" name=\"fr\" type=\"FieldReaderDataSource\"/>\n" +
+ "\n" +
+ " <document name=\"items\">\n" +
+ " <entity dataSource=\"ds1\" name=\"root\" pk=\"id\" query=\"select id, name, blob_field from lw_table4\" transformer=\"TemplateTransformer\">\n" +
+ " <field column=\"id\" name=\"id\"/>\n" +
+ "\n" +
+ " <entity dataField=\"root.blob_field\" dataSource=\"fr\" format=\"text\" name=\"n1\" processor=\"PlainTextEntityProcessor\" url=\"blob_field\">\n" +
+ " <field column=\"plainText\" name=\"plainText\"/>\n" +
+ " </entity>\n" +
+ "\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+ System.out.println(dc);
+ di.loadAndInit(dc);
+ redirectTempProperties(di);
+
+ TestDocBuilder.SolrWriterImpl sw = new TestDocBuilder.SolrWriterImpl();
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
+ di.runCmd(rp, sw);
+ assertEquals(DS.s, sw.docs.get(0).getFieldValue("plainText"));
+ }
+
+
+ public static class DS extends DataSource {
+ static String s = "hello world";
+
+ @Override
+ public void init(Context context, Properties initProps) {
+
+ }
+
+ @Override
+ public Object getData(String query) {
+
+ return new StringReader(s);
+ }
+
+ @Override
+ public void close() {
+
+ }
+ }
+
+ static String DATA_CONFIG = "<dataConfig>\n" +
+ "\t<dataSource type=\"TestPlainTextEntityProcessor$DS\" />\n" +
+ "\t<document>\n" +
+ "\t\t<entity processor=\"PlainTextEntityProcessor\" name=\"x\" query=\"x\">\n" +
+ "\t\t\t<field column=\"plainText\" name=\"x\" />\n" +
+ "\t\t</entity>\n" +
+ "\t</document>\n" +
+ "</dataConfig>";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestRegexTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestRegexTransformer.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestRegexTransformer.java
new file mode 100644
index 0000000..e792804
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestRegexTransformer.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static org.apache.solr.handler.dataimport.RegexTransformer.REGEX;
+import static org.apache.solr.handler.dataimport.RegexTransformer.GROUP_NAMES;
+import static org.apache.solr.handler.dataimport.RegexTransformer.REPLACE_WITH;
+import static org.apache.solr.handler.dataimport.DataImporter.COLUMN;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * <p> Test for RegexTransformer </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestRegexTransformer extends AbstractDataImportHandlerTestCase {
+
+ @Test
+ public void testCommaSeparated() {
+ List<Map<String, String>> fields = new ArrayList<>();
+ // <field column="col1" sourceColName="a" splitBy="," />
+ fields.add(getField("col1", "string", null, "a", ","));
+ Context context = getContext(null, null, null, Context.FULL_DUMP, fields, null);
+
+ Map<String, Object> src = new HashMap<>();
+ src.put("a", "a,bb,cc,d");
+
+ Map<String, Object> result = new RegexTransformer().transformRow(src, context);
+ assertEquals(2, result.size());
+ assertEquals(4, ((List) result.get("col1")).size());
+ }
+
+
+ @Test
+ public void testGroupNames() {
+ List<Map<String, String>> fields = new ArrayList<>();
+ // <field column="col1" regex="(\w*)(\w*) (\w*)" groupNames=",firstName,lastName"/>
+ Map<String ,String > m = new HashMap<>();
+ m.put(COLUMN,"fullName");
+ m.put(GROUP_NAMES,",firstName,lastName");
+ m.put(REGEX,"(\\w*) (\\w*) (\\w*)");
+ fields.add(m);
+ Context context = getContext(null, null, null, Context.FULL_DUMP, fields, null);
+ Map<String, Object> src = new HashMap<>();
+ src.put("fullName", "Mr Noble Paul");
+
+ Map<String, Object> result = new RegexTransformer().transformRow(src, context);
+ assertEquals("Noble", result.get("firstName"));
+ assertEquals("Paul", result.get("lastName"));
+ src= new HashMap<>();
+ List<String> l= new ArrayList();
+ l.add("Mr Noble Paul") ;
+ l.add("Mr Shalin Mangar") ;
+ src.put("fullName", l);
+ result = new RegexTransformer().transformRow(src, context);
+ List l1 = (List) result.get("firstName");
+ List l2 = (List) result.get("lastName");
+ assertEquals("Noble", l1.get(0));
+ assertEquals("Shalin", l1.get(1));
+ assertEquals("Paul", l2.get(0));
+ assertEquals("Mangar", l2.get(1));
+ }
+
+ @Test
+ public void testReplaceWith() {
+ List<Map<String, String>> fields = new ArrayList<>();
+ // <field column="name" regexp="'" replaceWith="''" />
+ Map<String, String> fld = getField("name", "string", "'", null, null);
+ fld.put(REPLACE_WITH, "''");
+ fields.add(fld);
+ Context context = getContext(null, null, null, Context.FULL_DUMP, fields, null);
+
+ Map<String, Object> src = new HashMap<>();
+ String s = "D'souza";
+ src.put("name", s);
+
+ Map<String, Object> result = new RegexTransformer().transformRow(src,
+ context);
+ assertEquals("D''souza", result.get("name"));
+
+ fld = getField("title_underscore", "string", "\\s+", "title", null);
+ fld.put(REPLACE_WITH, "_");
+ fields.clear();
+ fields.add(fld);
+ context = getContext(null, null, null, Context.FULL_DUMP, fields, null);
+ src.clear();
+ src.put("title", "value with spaces"); // a value which will match the regex
+ result = new RegexTransformer().transformRow(src, context);
+ assertEquals("value_with_spaces", result.get("title_underscore"));
+ src.clear();
+ src.put("title", "valueWithoutSpaces"); // value which will not match regex
+ result = new RegexTransformer().transformRow(src, context);
+ assertEquals("valueWithoutSpaces", result.get("title_underscore")); // value should be returned as-is
+ }
+
+ @Test
+ public void testMileage() {
+ // init a whole pile of fields
+ List<Map<String, String>> fields = getFields();
+
+ // add another regex which reuses result from previous regex again!
+ // <field column="hltCityMPG" sourceColName="rowdata" regexp="(${e.city_mileage})" />
+ Map<String, String> fld = getField("hltCityMPG", "string",
+ ".*(${e.city_mileage})", "rowdata", null);
+ fld.put(REPLACE_WITH, "*** $1 ***");
+ fields.add(fld);
+
+ // **ATTEMPTS** a match WITHOUT a replaceWith
+ // <field column="t1" sourceColName="rowdata" regexp="duff" />
+ fld = getField("t1", "string","duff", "rowdata", null);
+ fields.add(fld);
+
+ // **ATTEMPTS** a match WITH a replaceWith (should return original data)
+ // <field column="t2" sourceColName="rowdata" regexp="duff" replaceWith="60"/>
+ fld = getField("t2", "string","duff", "rowdata", null);
+ fld.put(REPLACE_WITH, "60");
+ fields.add(fld);
+
+ // regex WITH both replaceWith and groupName (groupName ignored!)
+ // <field column="t3" sourceColName="rowdata" regexp="(Range)" />
+ fld = getField("t3", "string","(Range)", "rowdata", null);
+ fld.put(REPLACE_WITH, "range");
+ fld.put(GROUP_NAMES,"t4,t5");
+ fields.add(fld);
+
+ Map<String, Object> row = new HashMap<>();
+ String s = "Fuel Economy Range: 26 mpg Hwy, 19 mpg City";
+ row.put("rowdata", s);
+
+ VariableResolver resolver = new VariableResolver();
+ resolver.addNamespace("e", row);
+ Map<String, String> eAttrs = createMap("name", "e");
+ Context context = getContext(null, resolver, null, Context.FULL_DUMP, fields, eAttrs);
+
+ Map<String, Object> result = new RegexTransformer().transformRow(row, context);
+ assertEquals(6, result.size());
+ assertEquals(s, result.get("t2"));
+ assertEquals(s, result.get("rowdata"));
+ assertEquals("26", result.get("highway_mileage"));
+ assertEquals("19", result.get("city_mileage"));
+ assertEquals("*** 19 *** mpg City", result.get("hltCityMPG"));
+ assertEquals("Fuel Economy range: 26 mpg Hwy, 19 mpg City", result.get("t3"));
+ }
+
+ @Test
+ public void testMultiValuedRegex(){
+ List<Map<String, String>> fields = new ArrayList<>();
+// <field column="participant" sourceColName="person" regex="(.*)" />
+ Map<String, String> fld = getField("participant", null, "(.*)", "person", null);
+ fields.add(fld);
+ Context context = getContext(null, null,
+ null, Context.FULL_DUMP, fields, null);
+
+ ArrayList<String> strings = new ArrayList<>();
+ strings.add("hello");
+ strings.add("world");
+ Map<String, Object> result = new RegexTransformer().transformRow(createMap("person", strings), context);
+ assertEquals(strings,result.get("participant"));
+ }
+
+ public static List<Map<String, String>> getFields() {
+ List<Map<String, String>> fields = new ArrayList<>();
+
+ // <field column="city_mileage" sourceColName="rowdata" regexp=
+ // "Fuel Economy Range:\\s*?\\d*?\\s*?mpg Hwy,\\s*?(\\d*?)\\s*?mpg City"
+ fields.add(getField("city_mileage", "sint",
+ "Fuel Economy Range:\\s*?\\d*?\\s*?mpg Hwy,\\s*?(\\d*?)\\s*?mpg City",
+ "rowdata", null));
+
+ // <field column="highway_mileage" sourceColName="rowdata" regexp=
+ // "Fuel Economy Range:\\s*?(\\d*?)\\s*?mpg Hwy,\\s*?\\d*?\\s*?mpg City"
+ fields.add(getField("highway_mileage", "sint",
+ "Fuel Economy Range:\\s*?(\\d*?)\\s*?mpg Hwy,\\s*?\\d*?\\s*?mpg City",
+ "rowdata", null));
+
+ // <field column="seating_capacity" sourceColName="rowdata" regexp="Seating capacity:(.*)"
+ fields.add(getField("seating_capacity", "sint", "Seating capacity:(.*)",
+ "rowdata", null));
+
+ // <field column="warranty" sourceColName="rowdata" regexp="Warranty:(.*)" />
+ fields.add(getField("warranty", "string", "Warranty:(.*)", "rowdata", null));
+
+ // <field column="rowdata" sourceColName="rowdata" />
+ fields.add(getField("rowdata", "string", null, "rowdata", null));
+ return fields;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestScriptTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestScriptTransformer.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestScriptTransformer.java
new file mode 100644
index 0000000..c106f8e
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestScriptTransformer.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.handler.dataimport.config.DIHConfiguration;
+import org.junit.Test;
+import org.w3c.dom.Document;
+import org.xml.sax.InputSource;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Test for ScriptTransformer
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestScriptTransformer extends AbstractDataImportHandlerTestCase {
+
+ @Test
+ public void testBasic() {
+ try {
+ String script = "function f1(row,context){"
+ + "row.put('name','Hello ' + row.get('name'));" + "return row;\n" + "}";
+ Context context = getContext("f1", script);
+ Map<String, Object> map = new HashMap<>();
+ map.put("name", "Scott");
+ EntityProcessorWrapper sep = new EntityProcessorWrapper(new SqlEntityProcessor(), null, null);
+ sep.init(context);
+ sep.applyTransformer(map);
+ assertEquals("Hello Scott", map.get("name").toString());
+ } catch (DataImportHandlerException e) {
+ assumeFalse("This JVM does not have JavaScript installed. Test Skipped.", e
+ .getMessage().startsWith("Cannot load Script Engine for language"));
+ throw e;
+ }
+ }
+
+ private Context getContext(String funcName, String script) {
+ List<Map<String, String>> fields = new ArrayList<>();
+ Map<String, String> entity = new HashMap<>();
+ entity.put("name", "hello");
+ entity.put("transformer", "script:" + funcName);
+
+ TestContext context = getContext(null, null, null,
+ Context.FULL_DUMP, fields, entity);
+ context.script = script;
+ context.scriptlang = "JavaScript";
+ return context;
+ }
+
+ @Test
+ public void testOneparam() {
+ try {
+ String script = "function f1(row){"
+ + "row.put('name','Hello ' + row.get('name'));" + "return row;\n" + "}";
+
+ Context context = getContext("f1", script);
+ Map<String, Object> map = new HashMap<>();
+ map.put("name", "Scott");
+ EntityProcessorWrapper sep = new EntityProcessorWrapper(new SqlEntityProcessor(), null, null);
+ sep.init(context);
+ sep.applyTransformer(map);
+ assertEquals("Hello Scott", map.get("name").toString());
+ } catch (DataImportHandlerException e) {
+ assumeFalse("This JVM does not have JavaScript installed. Test Skipped.", e
+ .getMessage().startsWith("Cannot load Script Engine for language"));
+ throw e;
+ }
+ }
+
+ @Test
+ public void testReadScriptTag() throws Exception {
+ try {
+ DocumentBuilder builder = DocumentBuilderFactory.newInstance()
+ .newDocumentBuilder();
+ Document document = builder.parse(new InputSource(new StringReader(xml)));
+ DataImporter di = new DataImporter();
+ DIHConfiguration dc = di.readFromXml(document);
+ assertTrue(dc.getScript().getText().indexOf("checkNextToken") > -1);
+ } catch (DataImportHandlerException e) {
+ assumeFalse("This JVM does not have JavaScript installed. Test Skipped.", e
+ .getMessage().startsWith("Cannot load Script Engine for language"));
+ throw e;
+ }
+ }
+
+ @Test
+ public void testCheckScript() throws Exception {
+ try {
+ DocumentBuilder builder = DocumentBuilderFactory.newInstance()
+ .newDocumentBuilder();
+ Document document = builder.parse(new InputSource(new StringReader(xml)));
+ DataImporter di = new DataImporter();
+ DIHConfiguration dc = di.readFromXml(document);
+ Context c = getContext("checkNextToken", dc.getScript().getText());
+
+ Map map = new HashMap();
+ map.put("nextToken", "hello");
+ EntityProcessorWrapper sep = new EntityProcessorWrapper(new SqlEntityProcessor(), null, null);
+ sep.init(c);
+ sep.applyTransformer(map);
+ assertEquals("true", map.get("$hasMore"));
+ map = new HashMap();
+ map.put("nextToken", "");
+ sep.applyTransformer(map);
+ assertNull(map.get("$hasMore"));
+ } catch (DataImportHandlerException e) {
+ assumeFalse("This JVM does not have JavaScript installed. Test Skipped.", e
+ .getMessage().startsWith("Cannot load Script Engine for language"));
+ throw e;
+ }
+ }
+
+ static String xml = "<dataConfig>\n"
+ + "<script><![CDATA[\n"
+ + "function checkNextToken(row)\t{\n"
+ + " var nt = row.get('nextToken');"
+ + " if (nt && nt !='' ){ "
+ + " row.put('$hasMore', 'true');}\n"
+ + " return row;\n"
+ + "}]]></script>\t<document>\n"
+ + "\t\t<entity name=\"mbx\" pk=\"articleNumber\" processor=\"XPathEntityProcessor\"\n"
+ + "\t\t\turl=\"?boardId=${dataimporter.defaults.boardId}&maxRecords=20&includeBody=true&startDate=${dataimporter.defaults.startDate}&guid=:autosearch001&reqId=1&transactionId=stringfortracing&listPos=${mbx.nextToken}\"\n"
+ + "\t\t\tforEach=\"/mbmessage/articles/navigation | /mbmessage/articles/article\" transformer=\"script:checkNextToken\">\n"
+ + "\n" + "\t\t\t<field column=\"nextToken\"\n"
+ + "\t\t\t\txpath=\"/mbmessage/articles/navigation/nextToken\" />\n"
+ + "\n" + "\t\t</entity>\n" + "\t</document>\n" + "</dataConfig>";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSimplePropertiesWriter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSimplePropertiesWriter.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSimplePropertiesWriter.java
new file mode 100644
index 0000000..93a26c3
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSimplePropertiesWriter.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.lang.invoke.MethodHandles;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.solr.common.util.SuppressForbidden;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class TestSimplePropertiesWriter extends AbstractDIHJdbcTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private boolean useJdbcEscapeSyntax;
+ private String dateFormat;
+ private String fileLocation;
+ private String fileName;
+
+ @Before
+ public void spwBefore() throws Exception {
+ fileLocation = createTempDir().toFile().getAbsolutePath();
+ fileName = "the.properties";
+ }
+
+ @SuppressForbidden(reason = "Needs currentTimeMillis to construct date stamps")
+ @Test
+ public void testSimplePropertiesWriter() throws Exception {
+
+ SimpleDateFormat errMsgFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSS", Locale.ROOT);
+
+ String[] d = {
+ "{'ts' ''yyyy-MM-dd HH:mm:ss.SSSSSS''}",
+ "{'ts' ''yyyy-MM-dd HH:mm:ss''}",
+ "yyyy-MM-dd HH:mm:ss",
+ "yyyy-MM-dd HH:mm:ss.SSSSSS"
+ };
+ for(int i=0 ; i<d.length ; i++) {
+ delQ("*:*");
+ commit();
+ if(i<2) {
+ useJdbcEscapeSyntax = true;
+ } else {
+ useJdbcEscapeSyntax = false;
+ }
+ dateFormat = d[i];
+ SimpleDateFormat df = new SimpleDateFormat(dateFormat, Locale.ROOT);
+ Date oneSecondAgo = new Date(System.currentTimeMillis() - 1000);
+
+ Map<String,String> init = new HashMap<>();
+ init.put("dateFormat", dateFormat);
+ init.put("filename", fileName);
+ init.put("directory", fileLocation);
+ SimplePropertiesWriter spw = new SimplePropertiesWriter();
+ spw.init(new DataImporter(), init);
+ Map<String, Object> props = new HashMap<>();
+ props.put("SomeDates.last_index_time", oneSecondAgo);
+ props.put("last_index_time", oneSecondAgo);
+ spw.persist(props);
+
+ h.query("/dataimport", generateRequest());
+ props = spw.readIndexerProperties();
+ Date entityDate = df.parse((String) props.get("SomeDates.last_index_time"));
+ Date docDate= df.parse((String) props.get("last_index_time"));
+ int year = currentYearFromDatabase();
+
+ Assert.assertTrue("This date: " + errMsgFormat.format(oneSecondAgo) + " should be prior to the document date: " + errMsgFormat.format(docDate), docDate.getTime() - oneSecondAgo.getTime() > 0);
+ Assert.assertTrue("This date: " + errMsgFormat.format(oneSecondAgo) + " should be prior to the entity date: " + errMsgFormat.format(entityDate), entityDate.getTime() - oneSecondAgo.getTime() > 0);
+ assertQ(req("*:*"), "//*[@numFound='1']", "//doc/str[@name=\"ayear_s\"]=\"" + year + "\"");
+ }
+ }
+
+ private int currentYearFromDatabase() throws Exception {
+ Connection conn = null;
+ Statement s = null;
+ ResultSet rs = null;
+ try {
+ conn = newConnection();
+ s = conn.createStatement();
+ rs = s.executeQuery("select year(current_timestamp) from sysibm.sysdummy1");
+ if (rs.next()) {
+ return rs.getInt(1);
+ }
+ Assert.fail("We should have gotten a row from the db.");
+ } catch (SQLException e) {
+ throw e;
+ } finally {
+ try {
+ rs.close();
+ } catch (Exception ex) {}
+ try {
+ s.close();
+ } catch (Exception ex) {}
+ try {
+ conn.close();
+ } catch (Exception ex) {}
+ }
+ return 0;
+ }
+
+ @Override
+ protected Database setAllowedDatabases() {
+ return Database.DERBY;
+ }
+ @Override
+ protected String generateConfig() {
+ StringBuilder sb = new StringBuilder();
+ String q = useJdbcEscapeSyntax ? "" : "'";
+ sb.append("<dataConfig> \n");
+ sb.append("<propertyWriter dateFormat=\"" + dateFormat + "\" type=\"SimplePropertiesWriter\" directory=\"" + fileLocation + "\" filename=\"" + fileName + "\" />\n");
+ sb.append("<dataSource name=\"derby\" driver=\"org.apache.derby.jdbc.EmbeddedDriver\" url=\"jdbc:derby:memory:derbyDB;territory=en_US\" /> \n");
+ sb.append("<document name=\"TestSimplePropertiesWriter\"> \n");
+ sb.append("<entity name=\"SomeDates\" processor=\"SqlEntityProcessor\" dataSource=\"derby\" ");
+ sb.append("query=\"select 1 as id, YEAR(" + q + "${dih.last_index_time}" + q + ") as AYEAR_S from sysibm.sysdummy1 \" >\n");
+ sb.append("<field column=\"AYEAR_S\" name=\"ayear_s\" /> \n");
+ sb.append("</entity>\n");
+ sb.append("</document> \n");
+ sb.append("</dataConfig> \n");
+ String config = sb.toString();
+ log.debug(config);
+ return config;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java
new file mode 100644
index 0000000..0e9cd33
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java
@@ -0,0 +1,373 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.lucene.util.IOUtils;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.common.SolrInputDocument;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+/**
+ * End-to-end test of SolrEntityProcessor. "Real" test using embedded Solr
+ */
+public class TestSolrEntityProcessorEndToEnd extends AbstractDataImportHandlerTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final String SOLR_CONFIG = "dataimport-solrconfig.xml";
+ private static final String SOLR_SCHEMA = "dataimport-schema.xml";
+ private static final String SOURCE_CONF_DIR = "dih" + File.separator + "solr" + File.separator + "collection1" + File.separator + "conf" + File.separator;
+ private static final String ROOT_DIR = "dih" + File.separator + "solr" + File.separator;
+
+ private static final String DEAD_SOLR_SERVER = "http://[ff01::114]:33332/solr";
+
+ private static final List<Map<String,Object>> DB_DOCS = new ArrayList<>();
+ private static final List<Map<String,Object>> SOLR_DOCS = new ArrayList<>();
+
+ static {
+ // dynamic fields in the destination schema
+ Map<String,Object> dbDoc = new HashMap<>();
+ dbDoc.put("dbid_s", "1");
+ dbDoc.put("dbdesc_s", "DbDescription");
+ DB_DOCS.add(dbDoc);
+
+ Map<String,Object> solrDoc = new HashMap<>();
+ solrDoc.put("id", "1");
+ solrDoc.put("desc", "SolrDescription");
+ SOLR_DOCS.add(solrDoc);
+ }
+
+
+ private SolrInstance instance = null;
+ private JettySolrRunner jetty;
+
+ private String getDihConfigTagsInnerEntity() {
+ return "<dataConfig>\r\n"
+ + " <dataSource type='MockDataSource' />\r\n"
+ + " <document>\r\n"
+ + " <entity name='db' query='select * from x'>\r\n"
+ + " <field column='dbid_s' />\r\n"
+ + " <field column='dbdesc_s' />\r\n"
+ + " <entity name='se' processor='SolrEntityProcessor' query='id:${db.dbid_s}'\n"
+ + " url='" + getSourceUrl() + "' fields='id,desc'>\r\n"
+ + " <field column='id' />\r\n"
+ + " <field column='desc' />\r\n" + " </entity>\r\n"
+ + " </entity>\r\n" + " </document>\r\n" + "</dataConfig>\r\n";
+ }
+
+ private String generateDIHConfig(String options, boolean useDeadServer) {
+ return "<dataConfig>\r\n" + " <document>\r\n"
+ + " <entity name='se' processor='SolrEntityProcessor'" + " url='"
+ + (useDeadServer ? DEAD_SOLR_SERVER : getSourceUrl()) + "' " + options + " />\r\n" + " </document>\r\n"
+ + "</dataConfig>\r\n";
+ }
+
+ private String getSourceUrl() {
+ return buildUrl(jetty.getLocalPort(), "/solr/collection1");
+ }
+
+ //TODO: fix this test to close its directories
+ static String savedFactory;
+ @BeforeClass
+ public static void beforeClass() {
+ savedFactory = System.getProperty("solr.DirectoryFactory");
+ System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
+ }
+
+ @AfterClass
+ public static void afterClass() {
+ if (savedFactory == null) {
+ System.clearProperty("solr.directoryFactory");
+ } else {
+ System.setProperty("solr.directoryFactory", savedFactory);
+ }
+ }
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ // destination solr core
+ initCore(SOLR_CONFIG, SOLR_SCHEMA);
+ // data source solr instance
+ instance = new SolrInstance();
+ instance.setUp();
+ jetty = createJetty(instance);
+ }
+
+ @Override
+ @After
+ public void tearDown() throws Exception {
+ try {
+ deleteCore();
+ } catch (Exception e) {
+ log.error("Error deleting core", e);
+ }
+ jetty.stop();
+ instance.tearDown();
+ super.tearDown();
+ }
+
+ //commented 23-AUG-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018
+ public void testFullImport() {
+ assertQ(req("*:*"), "//result[@numFound='0']");
+
+ try {
+ addDocumentsToSolr(SOLR_DOCS);
+ runFullImport(generateDIHConfig("query='*:*' rows='2' fl='id,desc' onError='skip'", false));
+ } catch (Exception e) {
+ log.error(e.getMessage(), e);
+ fail(e.getMessage());
+ }
+
+ assertQ(req("*:*"), "//result[@numFound='1']");
+ assertQ(req("id:1"), "//result/doc/str[@name='id'][.='1']",
+ "//result/doc/arr[@name='desc'][.='SolrDescription']");
+ }
+
+ public void testFullImportFqParam() {
+ assertQ(req("*:*"), "//result[@numFound='0']");
+
+ try {
+ addDocumentsToSolr(generateSolrDocuments(30));
+ Map<String,String> map = new HashMap<>();
+ map.put("rows", "50");
+ runFullImport(generateDIHConfig("query='*:*' fq='desc:Description1*,desc:Description*2' rows='2'", false), map);
+ } catch (Exception e) {
+ log.error(e.getMessage(), e);
+ fail(e.getMessage());
+ }
+
+ assertQ(req("*:*"), "//result[@numFound='1']");
+ assertQ(req("id:12"), "//result[@numFound='1']", "//result/doc/arr[@name='desc'][.='Description12']");
+ }
+
+ public void testFullImportFieldsParam() {
+ assertQ(req("*:*"), "//result[@numFound='0']");
+
+ try {
+ addDocumentsToSolr(generateSolrDocuments(7));
+ runFullImport(generateDIHConfig("query='*:*' fl='id' rows='2'"+(random().nextBoolean() ?" cursorMark='true' sort='id asc'":""), false));
+ } catch (Exception e) {
+ log.error(e.getMessage(), e);
+ fail(e.getMessage());
+ }
+
+ assertQ(req("*:*"), "//result[@numFound='7']");
+ assertQ(req("id:1"), "//result[@numFound='1']");
+ try {
+ assertQ(req("id:1"), "//result/doc/arr[@name='desc']");
+ fail("The document has a field with name desc");
+ } catch(Exception e) {
+
+ }
+
+ }
+
+ /**
+ * Receive a row from SQL (Mock) and fetch a row from Solr
+ */
+ public void testFullImportInnerEntity() {
+ assertQ(req("*:*"), "//result[@numFound='0']");
+
+ try {
+ List<Map<String,Object>> DOCS = new ArrayList<>(DB_DOCS);
+ Map<String, Object> doc = new HashMap<>();
+ doc.put("dbid_s", "2");
+ doc.put("dbdesc_s", "DbDescription2");
+ DOCS.add(doc);
+ MockDataSource.setIterator("select * from x", DOCS.iterator());
+
+ DOCS = new ArrayList<>(SOLR_DOCS);
+ Map<String,Object> solrDoc = new HashMap<>();
+ solrDoc.put("id", "2");
+ solrDoc.put("desc", "SolrDescription2");
+ DOCS.add(solrDoc);
+ addDocumentsToSolr(DOCS);
+ runFullImport(getDihConfigTagsInnerEntity());
+ } catch (Exception e) {
+ log.error(e.getMessage(), e);
+ fail(e.getMessage());
+ } finally {
+ MockDataSource.clearCache();
+ }
+
+ assertQ(req("*:*"), "//result[@numFound='2']");
+ assertQ(req("id:1"), "//result/doc/str[@name='id'][.='1']",
+ "//result/doc/str[@name='dbdesc_s'][.='DbDescription']",
+ "//result/doc/str[@name='dbid_s'][.='1']",
+ "//result/doc/arr[@name='desc'][.='SolrDescription']");
+ assertQ(req("id:2"), "//result/doc/str[@name='id'][.='2']",
+ "//result/doc/str[@name='dbdesc_s'][.='DbDescription2']",
+ "//result/doc/str[@name='dbid_s'][.='2']",
+ "//result/doc/arr[@name='desc'][.='SolrDescription2']");
+ }
+
+ public void testFullImportWrongSolrUrl() {
+ assertQ(req("*:*"), "//result[@numFound='0']");
+
+ try {
+ runFullImport(generateDIHConfig("query='*:*' rows='2' fl='id,desc' onError='skip'", true /* use dead server */));
+ } catch (Exception e) {
+ log.error(e.getMessage(), e);
+ fail(e.getMessage());
+ }
+
+ assertQ(req("*:*"), "//result[@numFound='0']");
+ }
+
+ public void testFullImportBadConfig() {
+ assertQ(req("*:*"), "//result[@numFound='0']");
+
+ try {
+ runFullImport(generateDIHConfig("query='bogus:3' rows='2' fl='id,desc' onError='"+
+ (random().nextBoolean() ? "abort" : "justtogetcoverage")+"'", false));
+ } catch (Exception e) {
+ log.error(e.getMessage(), e);
+ fail(e.getMessage());
+ }
+
+ assertQ(req("*:*"), "//result[@numFound='0']");
+ }
+
+ public void testCursorMarkNoSort() throws SolrServerException, IOException {
+ assertQ(req("*:*"), "//result[@numFound='0']");
+ addDocumentsToSolr(generateSolrDocuments(7));
+ try {
+ List<String> errors = Arrays.asList("sort='id'", //wrong sort spec
+ "", //no sort spec
+ "sort='id asc' timeout='12345'"); // sort is fine, but set timeout
+ Collections.shuffle(errors, random());
+ String attrs = "query='*:*' rows='2' fl='id,desc' cursorMark='true' "
+ + errors.get(0);
+ runFullImport(generateDIHConfig(attrs,
+ false));
+ } catch (Exception e) {
+ log.error(e.getMessage(), e);
+ fail(e.getMessage());
+ }
+
+ assertQ(req("*:*"), "//result[@numFound='0']");
+ }
+
+ private static List<Map<String,Object>> generateSolrDocuments(int num) {
+ List<Map<String,Object>> docList = new ArrayList<>();
+ for (int i = 1; i <= num; i++) {
+ Map<String,Object> map = new HashMap<>();
+ map.put("id", i);
+ map.put("desc", "Description" + i);
+ docList.add(map);
+ }
+ return docList;
+ }
+
+ private void addDocumentsToSolr(List<Map<String,Object>> docs) throws SolrServerException, IOException {
+ List<SolrInputDocument> sidl = new ArrayList<>();
+ for (Map<String,Object> doc : docs) {
+ SolrInputDocument sd = new SolrInputDocument();
+ for (Entry<String,Object> entry : doc.entrySet()) {
+ sd.addField(entry.getKey(), entry.getValue());
+ }
+ sidl.add(sd);
+ }
+
+ try (HttpSolrClient solrServer = getHttpSolrClient(getSourceUrl(), 15000, 30000)) {
+ solrServer.add(sidl);
+ solrServer.commit(true, true);
+ }
+ }
+
+ private static class SolrInstance {
+ File homeDir;
+ File confDir;
+
+ public String getHomeDir() {
+ return homeDir.toString();
+ }
+
+ public String getSchemaFile() {
+ return SOURCE_CONF_DIR + "dataimport-schema.xml";
+ }
+
+ public String getDataDir() {
+ return initCoreDataDir.toString();
+ }
+
+ public String getSolrConfigFile() {
+ return SOURCE_CONF_DIR + "dataimport-solrconfig.xml";
+ }
+
+ public String getSolrXmlFile() {
+ return ROOT_DIR + "solr.xml";
+ }
+
+ public void setUp() throws Exception {
+ homeDir = createTempDir().toFile();
+ initCoreDataDir = new File(homeDir + "/collection1", "data");
+ confDir = new File(homeDir + "/collection1", "conf");
+
+ homeDir.mkdirs();
+ initCoreDataDir.mkdirs();
+ confDir.mkdirs();
+
+ FileUtils.copyFile(getFile(getSolrXmlFile()), new File(homeDir, "solr.xml"));
+ File f = new File(confDir, "solrconfig.xml");
+ FileUtils.copyFile(getFile(getSolrConfigFile()), f);
+ f = new File(confDir, "schema.xml");
+
+ FileUtils.copyFile(getFile(getSchemaFile()), f);
+ f = new File(confDir, "data-config.xml");
+ FileUtils.copyFile(getFile(SOURCE_CONF_DIR + "dataconfig-contentstream.xml"), f);
+
+ Files.createFile(confDir.toPath().resolve("../core.properties"));
+ }
+
+ public void tearDown() throws Exception {
+ IOUtils.rm(homeDir.toPath());
+ }
+ }
+
+ private JettySolrRunner createJetty(SolrInstance instance) throws Exception {
+ Properties nodeProperties = new Properties();
+ nodeProperties.setProperty("solr.data.dir", instance.getDataDir());
+ JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), nodeProperties, buildJettyConfig("/solr"));
+ jetty.start();
+ return jetty;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java
new file mode 100644
index 0000000..a2a9fff
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.*;
+
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.CursorMarkParams;
+import org.apache.solr.handler.dataimport.SolrEntityProcessor.SolrDocumentListIterator;
+import org.junit.Test;
+
+/**
+ * Unit test of SolrEntityProcessor. A very basic test outside of the DIH.
+ */
+public class TestSolrEntityProcessorUnit extends AbstractDataImportHandlerTestCase {
+
+ private static final class NoNextMockProcessor extends SolrEntityProcessor {
+ @Override
+ protected void nextPage() {
+ }
+ }
+
+ private static final String ID = "id";
+
+ public void testQuery() {
+ List<Doc> docs = generateUniqueDocs(2);
+
+ MockSolrEntityProcessor processor = createAndInit(docs);
+ try {
+ assertExpectedDocs(docs, processor);
+ assertEquals(1, processor.getQueryCount());
+ } finally {
+ processor.destroy();
+ }
+ }
+
+ private MockSolrEntityProcessor createAndInit(List<Doc> docs) {
+ return createAndInit(docs, SolrEntityProcessor.ROWS_DEFAULT);
+ }
+
+ public void testNumDocsGreaterThanRows() {
+ List<Doc> docs = generateUniqueDocs(44);
+
+ int rowsNum = 10;
+ MockSolrEntityProcessor processor = createAndInit(docs, rowsNum);
+ try {
+ assertExpectedDocs(docs, processor);
+ assertEquals(5, processor.getQueryCount());
+ } finally {
+ processor.destroy();
+ }
+ }
+
+ private MockSolrEntityProcessor createAndInit(List<Doc> docs, int rowsNum) {
+ MockSolrEntityProcessor processor = new MockSolrEntityProcessor(docs, rowsNum);
+ HashMap<String,String> entityAttrs = new HashMap<String,String>(){{put(SolrEntityProcessor.SOLR_SERVER,"http://route:66/no");}};
+ processor.init(getContext(null, null, null, null, Collections.emptyList(),
+ entityAttrs));
+ return processor;
+ }
+
+ public void testMultiValuedFields() {
+ List<Doc> docs = new ArrayList<>();
+ List<FldType> types = new ArrayList<>();
+ types.add(new FldType(ID, ONE_ONE, new SVal('A', 'Z', 4, 4)));
+ types.add(new FldType("description", new IRange(3, 3), new SVal('a', 'c', 1, 1)));
+ Doc testDoc = createDoc(types);
+ docs.add(testDoc);
+
+ MockSolrEntityProcessor processor = createAndInit(docs);
+ try {
+ Map<String, Object> next = processor.nextRow();
+ assertNotNull(next);
+
+ @SuppressWarnings("unchecked")
+ List<Comparable> multiField = (List<Comparable>) next.get("description");
+ assertEquals(testDoc.getValues("description").size(), multiField.size());
+ assertEquals(testDoc.getValues("description"), multiField);
+ assertEquals(1, processor.getQueryCount());
+ assertNull(processor.nextRow());
+ } finally {
+ processor.destroy();
+ }
+ }
+ @Test (expected = DataImportHandlerException.class)
+ public void testNoQuery() {
+ SolrEntityProcessor processor = new SolrEntityProcessor();
+
+ HashMap<String,String> entityAttrs = new HashMap<String,String>(){{put(SolrEntityProcessor.SOLR_SERVER,"http://route:66/no");}};
+ processor.init(getContext(null, null, null, null, Collections.emptyList(),
+ entityAttrs));
+ try {
+ processor.buildIterator();
+ }finally {
+ processor.destroy();
+ }
+ }
+
+ public void testPagingQuery() {
+ SolrEntityProcessor processor = new NoNextMockProcessor() ;
+
+ HashMap<String,String> entityAttrs = new HashMap<String,String>(){{
+ put(SolrEntityProcessor.SOLR_SERVER,"http://route:66/no");
+ if (random().nextBoolean()) {
+ List<String> noCursor = Arrays.asList("","false",CursorMarkParams.CURSOR_MARK_START);//only 'true' not '*'
+ Collections.shuffle(noCursor, random());
+ put(CursorMarkParams.CURSOR_MARK_PARAM, noCursor.get(0));
+ }}};
+ processor.init(getContext(null, null, null, null, Collections.emptyList(),
+ entityAttrs));
+ try {
+ processor.buildIterator();
+ SolrQuery query = new SolrQuery();
+ ((SolrDocumentListIterator) processor.rowIterator).passNextPage(query);
+ assertEquals("0", query.get(CommonParams.START));
+ assertNull( query.get(CursorMarkParams.CURSOR_MARK_PARAM));
+ assertNotNull( query.get(CommonParams.TIME_ALLOWED));
+ }finally {
+ processor.destroy();
+ }
+ }
+
+ public void testCursorQuery() {
+ SolrEntityProcessor processor = new NoNextMockProcessor() ;
+
+ HashMap<String,String> entityAttrs = new HashMap<String,String>(){{
+ put(SolrEntityProcessor.SOLR_SERVER,"http://route:66/no");
+ put(CursorMarkParams.CURSOR_MARK_PARAM,"true");
+ }};
+ processor.init(getContext(null, null, null, null, Collections.emptyList(),
+ entityAttrs));
+ try {
+ processor.buildIterator();
+ SolrQuery query = new SolrQuery();
+ ((SolrDocumentListIterator) processor.rowIterator).passNextPage(query);
+ assertNull(query.get(CommonParams.START));
+ assertEquals(CursorMarkParams.CURSOR_MARK_START, query.get(CursorMarkParams.CURSOR_MARK_PARAM));
+ assertNull( query.get(CommonParams.TIME_ALLOWED));
+ }finally {
+ processor.destroy();
+ }
+ }
+
+ private List<Doc> generateUniqueDocs(int numDocs) {
+ List<FldType> types = new ArrayList<>();
+ types.add(new FldType(ID, ONE_ONE, new SVal('A', 'Z', 4, 40)));
+ types.add(new FldType("description", new IRange(1, 3), new SVal('a', 'c', 1, 1)));
+
+ Set<Comparable> previousIds = new HashSet<>();
+ List<Doc> docs = new ArrayList<>(numDocs);
+ for (int i = 0; i < numDocs; i++) {
+ Doc doc = createDoc(types);
+ while (previousIds.contains(doc.id)) {
+ doc = createDoc(types);
+ }
+ previousIds.add(doc.id);
+ docs.add(doc);
+ }
+ return docs;
+ }
+
+ private static void assertExpectedDocs(List<Doc> expectedDocs, SolrEntityProcessor processor) {
+ for (Doc expectedDoc : expectedDocs) {
+ Map<String, Object> next = processor.nextRow();
+ assertNotNull(next);
+ assertEquals(expectedDoc.id, next.get("id"));
+ assertEquals(expectedDoc.getValues("description"), next.get("description"));
+ }
+ assertNull(processor.nextRow());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSortedMapBackedCache.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSortedMapBackedCache.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSortedMapBackedCache.java
new file mode 100644
index 0000000..7b0e613
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSortedMapBackedCache.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.lang.invoke.MethodHandles;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestSortedMapBackedCache extends AbstractDIHCacheTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @Test
+ public void testCacheWithKeyLookup() {
+ DIHCache cache = null;
+ try {
+ cache = new SortedMapBackedCache();
+ cache.open(getContext(new HashMap<String,String>()));
+ loadData(cache, data, fieldNames, true);
+ List<ControlData> testData = extractDataByKeyLookup(cache, fieldNames);
+ compareData(data, testData);
+ } catch (Exception e) {
+ log.warn("Exception thrown: " + e.toString());
+ Assert.fail();
+ } finally {
+ try {
+ cache.destroy();
+ } catch (Exception ex) {
+ }
+ }
+ }
+
+ @Test
+ public void testCacheWithOrderedLookup() {
+ DIHCache cache = null;
+ try {
+ cache = new SortedMapBackedCache();
+ cache.open(getContext(new HashMap<String,String>()));
+ loadData(cache, data, fieldNames, true);
+ List<ControlData> testData = extractDataInKeyOrder(cache, fieldNames);
+ compareData(data, testData);
+ } catch (Exception e) {
+ log.warn("Exception thrown: " + e.toString());
+ Assert.fail();
+ } finally {
+ try {
+ cache.destroy();
+ } catch (Exception ex) {
+ }
+ }
+ }
+
+ @Test
+ public void testNullKeys() throws Exception {
+ //A null key should just be ignored, but not throw an exception
+ DIHCache cache = null;
+ try {
+ cache = new SortedMapBackedCache();
+ Map<String, String> cacheProps = new HashMap<>();
+ cacheProps.put(DIHCacheSupport.CACHE_PRIMARY_KEY, "a_id");
+ cache.open(getContext(cacheProps));
+
+ Map<String,Object> data = new HashMap<>();
+ data.put("a_id", null);
+ data.put("bogus", "data");
+ cache.add(data);
+
+ Iterator<Map<String, Object>> cacheIter = cache.iterator();
+ while (cacheIter.hasNext()) {
+ Assert.fail("cache should be empty.");
+ }
+ Assert.assertNull(cache.iterator(null));
+ cache.delete(null);
+ } catch (Exception e) {
+ throw e;
+ } finally {
+ try {
+ cache.destroy();
+ } catch (Exception ex) {
+ }
+ }
+ }
+
+ @Test
+ public void testCacheReopensWithUpdate() {
+ DIHCache cache = null;
+ try {
+ Map<String, String> cacheProps = new HashMap<>();
+ cacheProps.put(DIHCacheSupport.CACHE_PRIMARY_KEY, "a_id");
+
+ cache = new SortedMapBackedCache();
+ cache.open(getContext(cacheProps));
+ // We can let the data hit the cache with the fields out of order because
+ // we've identified the pk up-front.
+ loadData(cache, data, fieldNames, false);
+
+ // Close the cache.
+ cache.close();
+
+ List<ControlData> newControlData = new ArrayList<>();
+ Object[] newIdEqualsThree = null;
+ int j = 0;
+ for (int i = 0; i < data.size(); i++) {
+ // We'll be deleting a_id=1 so remove it from the control data.
+ if (data.get(i).data[0].equals(1)) {
+ continue;
+ }
+
+ // We'll be changing "Cookie" to "Carrot" in a_id=3 so change it in the control data.
+ if (data.get(i).data[0].equals(3)) {
+ newIdEqualsThree = new Object[data.get(i).data.length];
+ System.arraycopy(data.get(i).data, 0, newIdEqualsThree, 0, newIdEqualsThree.length);
+ newIdEqualsThree[3] = "Carrot";
+ newControlData.add(new ControlData(newIdEqualsThree));
+ }
+ // Everything else can just be copied over.
+ else {
+ newControlData.add(data.get(i));
+ }
+
+ j++;
+ }
+
+ // These new rows of data will get added to the cache, so add them to the control data too.
+ Object[] newDataRow1 = new Object[] {99, new BigDecimal(Math.PI), "Z", "Zebra", 99.99f, Feb21_2011, null };
+ Object[] newDataRow2 = new Object[] {2, new BigDecimal(Math.PI), "B", "Ballerina", 2.22f, Feb21_2011, null };
+
+ newControlData.add(new ControlData(newDataRow1));
+ newControlData.add(new ControlData(newDataRow2));
+
+ // Re-open the cache
+ cache.open(getContext(new HashMap<String,String>()));
+
+ // Delete a_id=1 from the cache.
+ cache.delete(1);
+
+ // Because the cache allows duplicates, the only way to update is to
+ // delete first then add.
+ cache.delete(3);
+ cache.add(controlDataToMap(new ControlData(newIdEqualsThree), fieldNames, false));
+
+ // Add this row with a new Primary key.
+ cache.add(controlDataToMap(new ControlData(newDataRow1), fieldNames, false));
+
+ // Add this row, creating two records in the cache with a_id=2.
+ cache.add(controlDataToMap(new ControlData(newDataRow2), fieldNames, false));
+
+ // Read the cache back and compare to the newControlData
+ List<ControlData> testData = extractDataInKeyOrder(cache, fieldNames);
+ compareData(newControlData, testData);
+
+ // Now try reading the cache read-only.
+ cache.close();
+ cache.open(getContext(new HashMap<String,String>()));
+ testData = extractDataInKeyOrder(cache, fieldNames);
+ compareData(newControlData, testData);
+
+ } catch (Exception e) {
+ log.warn("Exception thrown: " + e.toString());
+ Assert.fail();
+ } finally {
+ try {
+ cache.destroy();
+ } catch (Exception ex) {
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java
new file mode 100644
index 0000000..f1277c9
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Test with various combinations of parameters, child entities, caches, transformers.
+ */
+public class TestSqlEntityProcessor extends AbstractSqlEntityProcessorTestCase {
+
+ @Test
+ public void testSingleEntity() throws Exception {
+ singleEntity(1);
+ }
+ @Test
+ public void testWithSimpleTransformer() throws Exception {
+ simpleTransform(1);
+ }
+ @Test
+ public void testWithComplexTransformer() throws Exception {
+ complexTransform(1, 0);
+ }
+ @Test
+ public void testChildEntities() throws Exception {
+ withChildEntities(false, true);
+ }
+ @Test
+ public void testCachedChildEntities() throws Exception {
+ withChildEntities(true, true);
+ }
+
+ @Test
+ public void testSportZipperChildEntities() throws Exception {
+ sportsZipper = true;
+ withChildEntities(true, true);
+ }
+
+ @Test
+ public void testCountryZipperChildEntities() throws Exception {
+ countryZipper = true;
+ withChildEntities(true, true);
+ }
+
+ @Test
+ public void testBothZipperChildEntities() throws Exception {
+ countryZipper = true;
+ sportsZipper = true;
+ withChildEntities(true, true);
+ }
+
+ @Test(expected=RuntimeException.class /* DIH exceptions are not propagated, here we capturing assertQ exceptions */)
+ public void testSportZipperChildEntitiesWrongOrder() throws Exception {
+ if(random().nextBoolean()){
+ wrongPeopleOrder = true;
+ }else{
+ wrongSportsOrder = true;
+ }
+ testSportZipperChildEntities();
+ }
+
+ @Test(expected=RuntimeException.class )
+ public void testCountryZipperChildEntitiesWrongOrder() throws Exception {
+ if(random().nextBoolean()){
+ wrongPeopleOrder = true;
+ }else{
+ wrongCountryOrder = true;
+ }
+ testCountryZipperChildEntities();
+ }
+
+ @Test(expected=RuntimeException.class)
+ public void testBothZipperChildEntitiesWrongOrder() throws Exception {
+ if(random().nextBoolean()){
+ wrongPeopleOrder = true;
+ }else{
+ if(random().nextBoolean()){
+ wrongSportsOrder = true;
+ }else{
+ wrongCountryOrder = true;
+ }
+ }
+ testBothZipperChildEntities();
+ }
+
+ @Test
+ @Ignore("broken see SOLR-3857")
+ public void testSimpleCacheChildEntities() throws Exception {
+ simpleCacheChildEntities(true);
+ }
+
+ @Override
+ protected String deltaQueriesCountryTable() {
+ return "";
+ }
+ @Override
+ protected String deltaQueriesPersonTable() {
+ return "";
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
new file mode 100644
index 0000000..1f22a8c
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.lang.invoke.MethodHandles;
+
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test with various combinations of parameters, child entities, transformers.
+ */
+public class TestSqlEntityProcessorDelta extends AbstractSqlEntityProcessorTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private boolean delta = false;
+ private boolean useParentDeltaQueryParam = false;
+ private IntChanges personChanges = null;
+ private String[] countryChanges = null;
+
+ @Before
+ public void setupDeltaTest() {
+ delta = false;
+ personChanges = null;
+ countryChanges = null;
+ }
+ @Test
+ public void testSingleEntity() throws Exception {
+ log.debug("testSingleEntity full-import...");
+ singleEntity(1);
+ logPropertiesFile();
+ changeStuff();
+ int c = calculateDatabaseCalls();
+ log.debug("testSingleEntity delta-import (" + c + " database calls expected)...");
+ singleEntity(c);
+ validateChanges();
+ }
+
+ @Test
+ public void testDeltaImportWithoutInitialFullImport() throws Exception {
+ log.debug("testDeltaImportWithoutInitialFullImport delta-import...");
+ countryEntity = false;
+ delta = true;
+ /*
+ * We need to add 2 in total:
+ * +1 for deltaQuery i.e identifying id of items to update,
+ * +1 for deletedPkQuery i.e delete query
+ */
+ singleEntity(totalPeople() + 2);
+ validateChanges();
+ }
+
+ @Test
+ public void testWithSimpleTransformer() throws Exception {
+ log.debug("testWithSimpleTransformer full-import...");
+ simpleTransform(1);
+ logPropertiesFile();
+ changeStuff();
+ int c = calculateDatabaseCalls();
+ simpleTransform(c);
+ log.debug("testWithSimpleTransformer delta-import (" + c + " database calls expected)...");
+ validateChanges();
+ }
+ @Test
+ public void testWithComplexTransformer() throws Exception {
+ log.debug("testWithComplexTransformer full-import...");
+ complexTransform(1, 0);
+ logPropertiesFile();
+ changeStuff();
+ int c = calculateDatabaseCalls();
+ log.debug("testWithComplexTransformer delta-import (" + c + " database calls expected)...");
+ complexTransform(c, personChanges.deletedKeys.length);
+ validateChanges();
+ }
+ @Test
+ public void testChildEntities() throws Exception {
+ log.debug("testChildEntities full-import...");
+ useParentDeltaQueryParam = random().nextBoolean();
+ log.debug("using parent delta? " + useParentDeltaQueryParam);
+ withChildEntities(false, true);
+ logPropertiesFile();
+ changeStuff();
+ log.debug("testChildEntities delta-import...");
+ withChildEntities(false, false);
+ validateChanges();
+ }
+
+
+ private int calculateDatabaseCalls() {
+ //The main query generates 1
+ //Deletes generate 1
+ //Each add/mod generate 1
+ int c = 1;
+ if (countryChanges != null) {
+ c += countryChanges.length + 1;
+ }
+ if (personChanges != null) {
+ c += personChanges.addedKeys.length + personChanges.changedKeys.length + 1;
+ }
+ return c;
+ }
+ private void validateChanges() throws Exception
+ {
+ if(personChanges!=null) {
+ for(int id : personChanges.addedKeys) {
+ assertQ(req("id:" + id), "//*[@numFound='1']");
+ }
+ for(int id : personChanges.deletedKeys) {
+ assertQ(req("id:" + id), "//*[@numFound='0']");
+ }
+ for(int id : personChanges.changedKeys) {
+ assertQ(req("id:" + id), "//*[@numFound='1']", "substring(//doc/arr[@name='NAME_mult_s']/str[1], 1, 8)='MODIFIED'");
+ }
+ }
+ if(countryChanges!=null) {
+ for(String code : countryChanges) {
+ assertQ(req("COUNTRY_CODE_s:" + code), "//*[@numFound='" + numberPeopleByCountryCode(code) + "']", "substring((//doc/str[@name='COUNTRY_NAME_s'])[1], 1, 8)='MODIFIED'");
+ }
+ }
+ }
+ private void changeStuff() throws Exception {
+ if(countryEntity)
+ {
+ int n = random().nextInt(2);
+ switch(n) {
+ case 0:
+ personChanges = modifySomePeople();
+ break;
+ case 1:
+ countryChanges = modifySomeCountries();
+ break;
+ case 2:
+ personChanges = modifySomePeople();
+ countryChanges = modifySomeCountries();
+ break;
+ }
+ } else {
+ personChanges = modifySomePeople();
+ }
+ countryChangesLog();
+ personChangesLog();
+ delta = true;
+ }
+ private void countryChangesLog()
+ {
+ if(countryChanges!=null) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("country changes { ");
+ for(String s : countryChanges) {
+ sb.append(s).append(" ");
+ }
+ sb.append(" }");
+ log.debug(sb.toString());
+ }
+ }
+ private void personChangesLog()
+ {
+ if(personChanges!=null) {
+ log.debug("person changes { " + personChanges.toString() + " } ");
+ }
+ }
+ @Override
+ protected LocalSolrQueryRequest generateRequest() {
+ return lrf.makeRequest("command", (delta ? "delta-import" : "full-import"), "dataConfig", generateConfig(),
+ "clean", (delta ? "false" : "true"), "commit", "true", "synchronous", "true", "indent", "true");
+ }
+ @Override
+ protected String deltaQueriesPersonTable() {
+ return
+ "deletedPkQuery=''SELECT ID FROM PEOPLE WHERE DELETED='Y' AND last_modified >='${dih.People.last_index_time}' '' " +
+ "deltaImportQuery=''SELECT ID, NAME, COUNTRY_CODE FROM PEOPLE where ID=${dih.delta.ID} '' " +
+ "deltaQuery=''" +
+ "SELECT ID FROM PEOPLE WHERE DELETED!='Y' AND last_modified >='${dih.People.last_index_time}' " +
+ (useParentDeltaQueryParam ? "" :
+ "UNION DISTINCT " +
+ "SELECT ID FROM PEOPLE WHERE DELETED!='Y' AND COUNTRY_CODE IN (SELECT CODE FROM COUNTRIES WHERE last_modified >='${dih.People.last_index_time}') "
+ ) + "'' "
+ ;
+ }
+ @Override
+ protected String deltaQueriesCountryTable() {
+ if(useParentDeltaQueryParam) {
+ return
+ "deltaQuery=''SELECT CODE FROM COUNTRIES WHERE DELETED != 'Y' AND last_modified >='${dih.last_index_time}' '' " +
+ "parentDeltaQuery=''SELECT ID FROM PEOPLE WHERE DELETED != 'Y' AND COUNTRY_CODE='${Countries.CODE}' '' "
+ ;
+
+ }
+ return "";
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestTemplateTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestTemplateTransformer.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestTemplateTransformer.java
new file mode 100644
index 0000000..b5c3811
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestTemplateTransformer.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Arrays;
+
+/**
+ * <p>
+ * Test for TemplateTransformer
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestTemplateTransformer extends AbstractDataImportHandlerTestCase {
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testTransformRow() {
+ List fields = new ArrayList();
+ fields.add(createMap("column", "firstName"));
+ fields.add(createMap("column", "lastName"));
+ fields.add(createMap("column", "middleName"));
+ fields.add(createMap("column", "name",
+ TemplateTransformer.TEMPLATE,
+ "${e.lastName}, ${e.firstName} ${e.middleName}"));
+ fields.add(createMap("column", "emails",
+ TemplateTransformer.TEMPLATE,
+ "${e.mail}"));
+
+ // test reuse of template output in another template
+ fields.add(createMap("column", "mrname",
+ TemplateTransformer.TEMPLATE,"Mr ${e.name}"));
+
+ List<String> mails = Arrays.asList("a@b.com", "c@d.com");
+ Map row = createMap(
+ "firstName", "Shalin",
+ "middleName", "Shekhar",
+ "lastName", "Mangar",
+ "mail", mails);
+
+ VariableResolver resolver = new VariableResolver();
+ resolver.addNamespace("e", row);
+ Map<String, String> entityAttrs = createMap("name", "e");
+
+ Context context = getContext(null, resolver,
+ null, Context.FULL_DUMP, fields, entityAttrs);
+ new TemplateTransformer().transformRow(row, context);
+ assertEquals("Mangar, Shalin Shekhar", row.get("name"));
+ assertEquals("Mr Mangar, Shalin Shekhar", row.get("mrname"));
+ assertEquals(mails,row.get("emails"));
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testTransformRowMultiValue() {
+ List fields = new ArrayList();
+ fields.add(createMap("column", "year"));
+ fields.add(createMap("column", "month"));
+ fields.add(createMap("column", "day"));
+
+ // create three variations of date format
+ fields.add(createMap( "column", "date",
+ TemplateTransformer.TEMPLATE,
+ "${e.day} ${e.month}, ${e.year}" ));
+ fields.add(createMap( "column", "date",
+ TemplateTransformer.TEMPLATE,
+ "${e.month} ${e.day}, ${e.year}" ));
+ fields.add(createMap("column", "date",
+ TemplateTransformer.TEMPLATE,
+ "${e.year}-${e.month}-${e.day}" ));
+
+ Map row = createMap( "year", "2016",
+ "month", "Apr",
+ "day", "30" );
+ VariableResolver resolver = new VariableResolver();
+ resolver.addNamespace("e", row);
+ Map<String, String> entityAttrs = createMap("date", "e");
+
+ Context context = getContext(null, resolver,
+ null, Context.FULL_DUMP, fields, entityAttrs);
+ new TemplateTransformer().transformRow(row, context);
+ assertTrue( row.get( "date" ) instanceof List );
+
+ List<Object> dates = (List<Object>)row.get( "date" );
+ assertEquals( dates.size(), 3 );
+ assertEquals( dates.get(0).toString(), "30 Apr, 2016" );
+ assertEquals( dates.get(1).toString(), "Apr 30, 2016" );
+ assertEquals( dates.get(2).toString(), "2016-Apr-30" );
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestURLDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestURLDataSource.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestURLDataSource.java
new file mode 100644
index 0000000..c1acc54
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestURLDataSource.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.junit.Test;
+
+public class TestURLDataSource extends AbstractDataImportHandlerTestCase {
+ private List<Map<String, String>> fields = new ArrayList<>();
+ private URLDataSource dataSource = new URLDataSource();
+ private VariableResolver variableResolver = new VariableResolver();
+ private Context context = AbstractDataImportHandlerTestCase.getContext(null, variableResolver,
+ dataSource, Context.FULL_DUMP, fields, null);
+ private Properties initProps = new Properties();
+
+ @Test
+ public void substitutionsOnBaseUrl() throws Exception {
+ String url = "http://example.com/";
+
+ variableResolver.addNamespace("dataimporter.request", Collections.<String,Object>singletonMap("baseurl", url));
+
+ initProps.setProperty(URLDataSource.BASE_URL, "${dataimporter.request.baseurl}");
+ dataSource.init(context, initProps);
+ assertEquals(url, dataSource.getBaseUrl());
+ }
+}