You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2017/03/27 06:05:21 UTC
[27/39] lucene-solr:feature/autoscaling: SOLR-9221: Remove Solr
contribs: map-reduce, morphlines-core and morphlines-cell
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/FileUtils.java
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/FileUtils.java b/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/FileUtils.java
deleted file mode 100644
index e979d37..0000000
--- a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/FileUtils.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.morphlines.solr;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.file.Files;
-
-
-class FileUtils {
-
- //-----------------------------------------------------------------------
- /**
- * Deletes a directory recursively.
- *
- * @param directory directory to delete
- * @throws IOException in case deletion is unsuccessful
- */
- public static void deleteDirectory(File directory) throws IOException {
- if (!directory.exists()) {
- return;
- }
-
- if (!isSymlink(directory)) {
- cleanDirectory(directory);
- }
-
- Files.delete(directory.toPath());
- }
-
- /**
- * Determines whether the specified file is a Symbolic Link rather than an actual file.
- * <p>
- * Will not return true if there is a Symbolic Link anywhere in the path,
- * only if the specific file is.
- *
- * @param file the file to check
- * @return true if the file is a Symbolic Link
- * @throws IOException if an IO error occurs while checking the file
- * @since Commons IO 2.0
- */
- public static boolean isSymlink(File file) throws IOException {
- if (file == null) {
- throw new NullPointerException("File must not be null");
- }
-// if (FilenameUtils.isSystemWindows()) {
- if (File.separatorChar == '\\') {
- return false;
- }
- File fileInCanonicalDir = null;
- if (file.getParent() == null) {
- fileInCanonicalDir = file;
- } else {
- File canonicalDir = file.getParentFile().getCanonicalFile();
- fileInCanonicalDir = new File(canonicalDir, file.getName());
- }
-
- if (fileInCanonicalDir.getCanonicalFile().equals(fileInCanonicalDir.getAbsoluteFile())) {
- return false;
- } else {
- return true;
- }
- }
-
- /**
- * Cleans a directory without deleting it.
- *
- * @param directory directory to clean
- * @throws IOException in case cleaning is unsuccessful
- */
- public static void cleanDirectory(File directory) throws IOException {
- if (!directory.exists()) {
- String message = directory + " does not exist";
- throw new IllegalArgumentException(message);
- }
-
- if (!directory.isDirectory()) {
- String message = directory + " is not a directory";
- throw new IllegalArgumentException(message);
- }
-
- File[] files = directory.listFiles();
- if (files == null) { // null if security restricted
- throw new IOException("Failed to list contents of " + directory);
- }
-
- IOException exception = null;
- for (File file : files) {
- try {
- forceDelete(file);
- } catch (IOException ioe) {
- exception = ioe;
- }
- }
-
- if (null != exception) {
- throw exception;
- }
- }
-
- //-----------------------------------------------------------------------
- /**
- * Deletes a file. If file is a directory, delete it and all sub-directories.
- * <p>
- * The difference between File.delete() and this method are:
- * <ul>
- * <li>A directory to be deleted does not have to be empty.</li>
- * <li>You get exceptions when a file or directory cannot be deleted.
- * (java.io.File methods returns a boolean)</li>
- * </ul>
- *
- * @param file file or directory to delete, must not be <code>null</code>
- * @throws NullPointerException if the directory is <code>null</code>
- * @throws FileNotFoundException if the file was not found
- * @throws IOException in case deletion is unsuccessful
- */
- public static void forceDelete(File file) throws IOException {
- if (file.isDirectory()) {
- deleteDirectory(file);
- } else {
- Files.delete(file.toPath());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/GenerateSolrSequenceKeyBuilder.java
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/GenerateSolrSequenceKeyBuilder.java b/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/GenerateSolrSequenceKeyBuilder.java
deleted file mode 100644
index be002ef..0000000
--- a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/GenerateSolrSequenceKeyBuilder.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.morphlines.solr;
-
-import java.security.SecureRandom;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Random;
-
-import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.schema.SchemaField;
-import org.kitesdk.morphline.api.Command;
-import org.kitesdk.morphline.api.CommandBuilder;
-import org.kitesdk.morphline.api.MorphlineContext;
-import org.kitesdk.morphline.api.MorphlineRuntimeException;
-import org.kitesdk.morphline.api.Record;
-import org.kitesdk.morphline.base.AbstractCommand;
-import org.kitesdk.morphline.base.Fields;
-import org.kitesdk.morphline.base.Notifications;
-
-import com.typesafe.config.Config;
-
-/**
- * A command that assigns a record unique key that is the concatenation of the given
- * <code>baseIdField</code> record field, followed by a running count of the record number within
- * the current session. The count is reset to zero whenever a "startSession" notification is
- * received.
- * <p>
- * For example, assume a CSV file containing multiple records but no unique ids, and the
- * <code>baseIdField</code> field is the filesystem path of the file. Now this command can be used
- * to assign the following record values to Solr's unique key field:
- * <code>$path#0, $path#1, ... $path#N</code>.
- * <p>
- * The name of the unique key field is fetched from Solr's schema.xml file, as directed by the
- * <code>solrLocator</code> configuration parameter.
- */
-public final class GenerateSolrSequenceKeyBuilder implements CommandBuilder {
-
- @Override
- public Collection<String> getNames() {
- return Arrays.asList(
- "generateSolrSequenceKey",
- "sanitizeUniqueSolrKey" // old name (retained for backwards compatibility)
- );
- }
-
- @Override
- public Command build(Config config, Command parent, Command child, MorphlineContext context) {
- return new GenerateSolrSequenceKey(this, config, parent, child, context);
- }
-
-
- ///////////////////////////////////////////////////////////////////////////////
- // Nested classes:
- ///////////////////////////////////////////////////////////////////////////////
- private static final class GenerateSolrSequenceKey extends AbstractCommand {
-
- private final boolean preserveExisting;
- private final String baseIdFieldName;
- private final String uniqueKeyName;
- private long recordCounter = 0;
-
- private final String idPrefix; // for load testing only; enables adding same document many times with a different unique key
- private final Random randomIdPrefix; // for load testing only; enables adding same document many times with a different unique key
-
- public GenerateSolrSequenceKey(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
- super(builder, config, parent, child, context);
- this.baseIdFieldName = getConfigs().getString(config, "baseIdField", Fields.BASE_ID);
- this.preserveExisting = getConfigs().getBoolean(config, "preserveExisting", true);
-
- Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator");
- SolrLocator locator = new SolrLocator(solrLocatorConfig, context);
- LOG.debug("solrLocator: {}", locator);
- IndexSchema schema = locator.getIndexSchema();
- SchemaField uniqueKey = schema.getUniqueKeyField();
- uniqueKeyName = uniqueKey == null ? null : uniqueKey.getName();
-
- String tmpIdPrefix = getConfigs().getString(config, "idPrefix", null); // for load testing only
- Random tmpRandomIdPrefx = null;
- if ("random".equals(tmpIdPrefix)) { // for load testing only
- tmpRandomIdPrefx = new Random(new SecureRandom().nextLong());
- tmpIdPrefix = null;
- }
- idPrefix = tmpIdPrefix;
- randomIdPrefix = tmpRandomIdPrefx;
- validateArguments();
- }
-
- @Override
- protected boolean doProcess(Record doc) {
- long num = recordCounter++;
- // LOG.debug("record #{} id before sanitizing doc: {}", num, doc);
- if (uniqueKeyName == null || (preserveExisting && doc.getFields().containsKey(uniqueKeyName))) {
- ; // we must preserve the existing id
- } else {
- Object baseId = doc.getFirstValue(baseIdFieldName);
- if (baseId == null) {
- throw new MorphlineRuntimeException("Record field " + baseIdFieldName
- + " must not be null as it is needed as a basis for a unique key for solr doc: " + doc);
- }
- doc.replaceValues(uniqueKeyName, baseId.toString() + "#" + num);
- }
-
- // for load testing only; enables adding same document many times with a different unique key
- if (idPrefix != null) {
- String id = doc.getFirstValue(uniqueKeyName).toString();
- id = idPrefix + id;
- doc.replaceValues(uniqueKeyName, id);
- } else if (randomIdPrefix != null) {
- String id = doc.getFirstValue(uniqueKeyName).toString();
- id = String.valueOf(Math.abs(randomIdPrefix.nextInt())) + "#" + id;
- doc.replaceValues(uniqueKeyName, id);
- }
-
- LOG.debug("record #{} unique key sanitized to this: {}", num, doc);
-
- return super.doProcess(doc);
- }
-
- @Override
- protected void doNotify(Record notification) {
- if (Notifications.containsLifecycleEvent(notification, Notifications.LifecycleEvent.START_SESSION)) {
- recordCounter = 0; // reset
- }
- super.doNotify(notification);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/LoadSolrBuilder.java
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/LoadSolrBuilder.java b/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/LoadSolrBuilder.java
deleted file mode 100644
index a3af6e1..0000000
--- a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/LoadSolrBuilder.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.morphlines.solr;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.common.SolrInputDocument;
-
-import org.kitesdk.morphline.api.Command;
-import org.kitesdk.morphline.api.CommandBuilder;
-import org.kitesdk.morphline.api.MorphlineContext;
-import org.kitesdk.morphline.api.MorphlineRuntimeException;
-import org.kitesdk.morphline.api.Record;
-import org.kitesdk.morphline.base.AbstractCommand;
-import org.kitesdk.morphline.base.Configs;
-import org.kitesdk.morphline.base.Metrics;
-import org.kitesdk.morphline.base.Notifications;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.codahale.metrics.Timer;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-
-/**
- * A command that loads a record into a SolrServer or MapReduce SolrOutputFormat.
- */
-public final class LoadSolrBuilder implements CommandBuilder {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private static final AtomicBoolean WARNED_ABOUT_INDEX_TIME_BOOSTS = new AtomicBoolean();
-
- @Override
- public Collection<String> getNames() {
- return Collections.singletonList("loadSolr");
- }
-
- @Override
- public Command build(Config config, Command parent, Command child, MorphlineContext context) {
- return new LoadSolr(this, config, parent, child, context);
- }
-
-
- ///////////////////////////////////////////////////////////////////////////////
- // Nested classes:
- ///////////////////////////////////////////////////////////////////////////////
- private static final class LoadSolr extends AbstractCommand {
-
- private final DocumentLoader loader;
- private final Timer elapsedTime;
-
- public LoadSolr(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
- super(builder, config, parent, child, context);
- Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator");
- SolrLocator locator = new SolrLocator(solrLocatorConfig, context);
- LOG.debug("solrLocator: {}", locator);
- this.loader = locator.getLoader();
- Config boostsConfig = getConfigs().getConfig(config, "boosts", ConfigFactory.empty());
- if (new Configs().getEntrySet(boostsConfig).isEmpty() == false) {
- String message = "Ignoring field boosts: as index-time boosts are not supported anymore";
- if (WARNED_ABOUT_INDEX_TIME_BOOSTS.compareAndSet(false, true)) {
- log.warn(message);
- } else {
- log.debug(message);
- }
- }
- validateArguments();
- this.elapsedTime = getTimer(Metrics.ELAPSED_TIME);
- }
-
- @Override
- protected void doNotify(Record notification) {
- for (Object event : Notifications.getLifecycleEvents(notification)) {
- if (event == Notifications.LifecycleEvent.BEGIN_TRANSACTION) {
- try {
- loader.beginTransaction();
- } catch (SolrServerException | IOException e) {
- throw new MorphlineRuntimeException(e);
- }
- } else if (event == Notifications.LifecycleEvent.COMMIT_TRANSACTION) {
- try {
- loader.commitTransaction();
- } catch (SolrServerException | IOException e) {
- throw new MorphlineRuntimeException(e);
- }
- }
- else if (event == Notifications.LifecycleEvent.ROLLBACK_TRANSACTION) {
- try {
- loader.rollbackTransaction();
- } catch (SolrServerException | IOException e) {
- throw new MorphlineRuntimeException(e);
- }
- }
- else if (event == Notifications.LifecycleEvent.SHUTDOWN) {
- try {
- loader.shutdown();
- } catch (SolrServerException | IOException e) {
- throw new MorphlineRuntimeException(e);
- }
- }
- }
- super.doNotify(notification);
- }
-
- @Override
- protected boolean doProcess(Record record) {
- Timer.Context timerContext = elapsedTime.time();
- SolrInputDocument doc = convert(record);
- try {
- loader.load(doc);
- } catch (IOException | SolrServerException e) {
- throw new MorphlineRuntimeException(e);
- } finally {
- timerContext.stop();
- }
-
- // pass record to next command in chain:
- return super.doProcess(record);
- }
-
- private SolrInputDocument convert(Record record) {
- Map<String, Collection<Object>> map = record.getFields().asMap();
- SolrInputDocument doc = new SolrInputDocument(new HashMap(2 * map.size()));
- for (Map.Entry<String, Collection<Object>> entry : map.entrySet()) {
- String key = entry.getKey();
- doc.setField(key, entry.getValue());
- }
- return doc;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SafeConcurrentUpdateSolrClient.java
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SafeConcurrentUpdateSolrClient.java b/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SafeConcurrentUpdateSolrClient.java
deleted file mode 100644
index a5fb929..0000000
--- a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SafeConcurrentUpdateSolrClient.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.morphlines.solr;
-
-import java.lang.invoke.MethodHandles;
-
-import org.apache.http.client.HttpClient;
-import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * ConcurrentUpdateSolrServer that propagates exceptions up to the submitter of
- * requests on blockUntilFinished()
- */
-final class SafeConcurrentUpdateSolrClient extends ConcurrentUpdateSolrClient {
-
- private Throwable currentException = null;
- private final Object myLock = new Object();
-
- private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public SafeConcurrentUpdateSolrClient(String solrServerUrl, int queueSize, int threadCount) {
- this(solrServerUrl, null, queueSize, threadCount);
- }
-
- public SafeConcurrentUpdateSolrClient(String solrServerUrl, HttpClient client, int queueSize, int threadCount) {
- super(solrServerUrl, client, queueSize, threadCount, null, false);
- }
-
- @Override
- public void handleError(Throwable ex) {
- assert ex != null;
- synchronized (myLock) {
- currentException = ex;
- }
- LOGGER.error("handleError", ex);
- }
-
- @Override
- public void blockUntilFinished() {
- super.blockUntilFinished();
- synchronized (myLock) {
- if (currentException != null) {
- throw new RuntimeException(currentException);
- }
- }
- }
-
- public void clearException() {
- synchronized (myLock) {
- currentException = null;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SanitizeUnknownSolrFieldsBuilder.java
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SanitizeUnknownSolrFieldsBuilder.java b/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SanitizeUnknownSolrFieldsBuilder.java
deleted file mode 100644
index 9ede714..0000000
--- a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SanitizeUnknownSolrFieldsBuilder.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.morphlines.solr;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
-import org.apache.solr.schema.IndexSchema;
-
-import org.kitesdk.morphline.api.Command;
-import org.kitesdk.morphline.api.CommandBuilder;
-import org.kitesdk.morphline.api.MorphlineContext;
-import org.kitesdk.morphline.api.Record;
-import org.kitesdk.morphline.base.AbstractCommand;
-import com.typesafe.config.Config;
-
-/**
- * Command that sanitizes record fields that are unknown to Solr schema.xml by either deleting them
- * (renameToPrefix is absent or a zero length string), or by moving them to a field prefixed with
- * the given renameToPrefix (e.g. renameToPrefix = "ignored_" to use typical dynamic Solr fields).
- * <p>
- * Recall that Solr throws an exception on any attempt to load a document that contains a field that
- * isn't specified in schema.xml.
- */
-public final class SanitizeUnknownSolrFieldsBuilder implements CommandBuilder {
-
- @Override
- public Collection<String> getNames() {
- return Collections.singletonList("sanitizeUnknownSolrFields");
- }
-
- @Override
- public Command build(Config config, Command parent, Command child, MorphlineContext context) {
- return new SanitizeUnknownSolrFields(this, config, parent, child, context);
- }
-
-
- ///////////////////////////////////////////////////////////////////////////////
- // Nested classes:
- ///////////////////////////////////////////////////////////////////////////////
- private static final class SanitizeUnknownSolrFields extends AbstractCommand {
-
- private final IndexSchema schema;
- private final String renameToPrefix;
-
- public SanitizeUnknownSolrFields(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
- super(builder, config, parent, child, context);
-
- Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator");
- SolrLocator locator = new SolrLocator(solrLocatorConfig, context);
- LOG.debug("solrLocator: {}", locator);
- this.schema = Objects.requireNonNull(locator.getIndexSchema());
- if (LOG.isTraceEnabled()) {
- LOG.trace("Solr schema: \n" +
- schema.getFields().entrySet().stream().sorted(Map.Entry.comparingByKey())
- .map(Map.Entry::getValue).map(Object::toString).collect(Collectors.joining("\n"))
- );
- }
-
- String str = getConfigs().getString(config, "renameToPrefix", "").trim();
- this.renameToPrefix = str.length() > 0 ? str : null;
- validateArguments();
- }
-
- @Override
- protected boolean doProcess(Record record) {
- Collection<Map.Entry> entries = new ArrayList<Map.Entry>(record.getFields().asMap().entrySet());
- for (Map.Entry<String, Collection<Object>> entry : entries) {
- String key = entry.getKey();
- if (schema.getFieldOrNull(key) == null) {
- LOG.debug("Sanitizing unknown Solr field: {}", key);
- Collection values = entry.getValue();
- if (renameToPrefix != null) {
- record.getFields().putAll(renameToPrefix + key, values);
- }
- values.clear(); // implicitly removes key from record
- }
- }
- return super.doProcess(record);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SolrClientDocumentLoader.java
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SolrClientDocumentLoader.java b/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SolrClientDocumentLoader.java
deleted file mode 100644
index ef9ea84..0000000
--- a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SolrClientDocumentLoader.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.morphlines.solr;
-
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
-import org.apache.solr.client.solrj.response.SolrPingResponse;
-import org.apache.solr.client.solrj.response.UpdateResponse;
-import org.apache.solr.common.SolrInputDocument;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * A vehicle to load a list of Solr documents into a local or remote {@link org.apache.solr.client.solrj.SolrClient}.
- */
-public class SolrClientDocumentLoader implements DocumentLoader {
-
- private final SolrClient client; // proxy to local or remote solr server
- private long numLoadedDocs = 0; // number of documents loaded in the current transaction
- private final int batchSize;
- private final List<SolrInputDocument> batch = new ArrayList();
-
- private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public SolrClientDocumentLoader(SolrClient client, int batchSize) {
- if (client == null) {
- throw new IllegalArgumentException("solr server must not be null");
- }
- this.client = client;
- if (batchSize <= 0) {
- throw new IllegalArgumentException("batchSize must be a positive number: " + batchSize);
- }
- this.batchSize = batchSize;
- }
-
- @Override
- public void beginTransaction() {
- LOGGER.trace("beginTransaction");
- batch.clear();
- numLoadedDocs = 0;
- if (client instanceof SafeConcurrentUpdateSolrClient) {
- ((SafeConcurrentUpdateSolrClient) client).clearException();
- }
- }
-
- @Override
- public void load(SolrInputDocument doc) throws IOException, SolrServerException {
- LOGGER.trace("load doc: {}", doc);
- batch.add(doc);
- if (batch.size() >= batchSize) {
- loadBatch();
- }
- }
-
- @Override
- public void commitTransaction() throws SolrServerException, IOException {
- LOGGER.trace("commitTransaction");
- if (batch.size() > 0) {
- loadBatch();
- }
- if (numLoadedDocs > 0) {
- if (client instanceof ConcurrentUpdateSolrClient) {
- ((ConcurrentUpdateSolrClient) client).blockUntilFinished();
- }
- }
- }
-
- private void loadBatch() throws SolrServerException, IOException {
- numLoadedDocs += batch.size();
- try {
- UpdateResponse rsp = client.add(batch);
- } finally {
- batch.clear();
- }
- }
-
- @Override
- public UpdateResponse rollbackTransaction() throws SolrServerException, IOException {
- LOGGER.trace("rollback");
- if (!(client instanceof CloudSolrClient)) {
- return client.rollback();
- } else {
- return new UpdateResponse();
- }
- }
-
- @Override
- public void shutdown() throws IOException {
- LOGGER.trace("shutdown");
- client.close();
- }
-
- @Override
- public SolrPingResponse ping() throws SolrServerException, IOException {
- LOGGER.trace("ping");
- return client.ping();
- }
-
- public SolrClient getSolrClient() {
- return client;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SolrLocator.java
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SolrLocator.java b/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SolrLocator.java
deleted file mode 100644
index 1d177a6..0000000
--- a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SolrLocator.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.morphlines.solr;
-
-import javax.xml.parsers.ParserConfigurationException;
-import java.io.File;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.nio.file.Paths;
-import java.util.Objects;
-
-import com.google.common.io.Files;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigRenderOptions;
-import com.typesafe.config.ConfigUtil;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder;
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.core.SolrConfig;
-import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.util.SystemIdResolver;
-import org.apache.zookeeper.KeeperException;
-import org.kitesdk.morphline.api.MorphlineCompilationException;
-import org.kitesdk.morphline.api.MorphlineContext;
-import org.kitesdk.morphline.api.MorphlineRuntimeException;
-import org.kitesdk.morphline.base.Configs;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.xml.sax.InputSource;
-import org.xml.sax.SAXException;
-
-/**
- * Set of configuration parameters that identify the location and schema of a Solr server or
- * SolrCloud; Based on this information this class can return the schema and a corresponding
- * {@link DocumentLoader}.
- */
-public class SolrLocator {
-
- private Config config;
- private MorphlineContext context;
- private String collectionName;
- private String zkHost;
- private String solrUrl;
- private String solrHomeDir;
- private int batchSize = 1000;
-
- private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- protected SolrLocator(MorphlineContext context) {
- this.context = Objects.requireNonNull(context);
- }
-
- public SolrLocator(Config config, MorphlineContext context) {
- this(context);
- this.config = config;
- Configs configs = new Configs();
- collectionName = configs.getString(config, "collection", null);
- zkHost = configs.getString(config, "zkHost", null);
- solrHomeDir = configs.getString(config, "solrHomeDir", null);
- solrUrl = configs.getString(config, "solrUrl", null);
- batchSize = configs.getInt(config, "batchSize", batchSize);
- LOG.trace("Constructed solrLocator: {}", this);
- configs.validateArguments(config);
- }
-
- public DocumentLoader getLoader() {
- if (context instanceof SolrMorphlineContext) {
- DocumentLoader loader = ((SolrMorphlineContext)context).getDocumentLoader();
- if (loader != null) {
- return loader;
- }
- }
-
- if (zkHost != null && zkHost.length() > 0) {
- if (collectionName == null || collectionName.length() == 0) {
- throw new MorphlineCompilationException("Parameter 'zkHost' requires that you also pass parameter 'collection'", config);
- }
- CloudSolrClient cloudSolrClient = new Builder()
- .withZkHost(zkHost)
- .build();
- cloudSolrClient.setDefaultCollection(collectionName);
- cloudSolrClient.connect();
- return new SolrClientDocumentLoader(cloudSolrClient, batchSize);
- } else {
- if (solrUrl == null || solrUrl.length() == 0) {
- throw new MorphlineCompilationException("Missing parameter 'solrUrl'", config);
- }
- int solrServerNumThreads = 2;
- int solrServerQueueLength = solrServerNumThreads;
- SolrClient server = new SafeConcurrentUpdateSolrClient(solrUrl, solrServerQueueLength, solrServerNumThreads);
- // SolrServer server = new HttpSolrServer(solrServerUrl);
- // SolrServer server = new ConcurrentUpdateSolrServer(solrServerUrl, solrServerQueueLength, solrServerNumThreads);
- // server.setParser(new XMLResponseParser()); // binary parser is used by default
- return new SolrClientDocumentLoader(server, batchSize);
- }
- }
-
- public IndexSchema getIndexSchema() {
- if (context instanceof SolrMorphlineContext) {
- IndexSchema schema = ((SolrMorphlineContext)context).getIndexSchema();
- if (schema != null) {
- validateSchema(schema);
- return schema;
- }
- }
-
- File downloadedSolrHomeDir = null;
- try {
- // If solrHomeDir isn't defined and zkHost and collectionName are defined
- // then download schema.xml and solrconfig.xml, etc from zk and use that as solrHomeDir
- String mySolrHomeDir = solrHomeDir;
- if (solrHomeDir == null || solrHomeDir.length() == 0) {
- if (zkHost == null || zkHost.length() == 0) {
- // TODO: implement download from solrUrl if specified
- throw new MorphlineCompilationException(
- "Downloading a Solr schema requires either parameter 'solrHomeDir' or parameters 'zkHost' and 'collection'",
- config);
- }
- if (collectionName == null || collectionName.length() == 0) {
- throw new MorphlineCompilationException(
- "Parameter 'zkHost' requires that you also pass parameter 'collection'", config);
- }
- ZooKeeperDownloader zki = new ZooKeeperDownloader();
- SolrZkClient zkClient = zki.getZkClient(zkHost);
- try {
- String configName = zki.readConfigName(zkClient, collectionName);
- downloadedSolrHomeDir = Files.createTempDir();
- downloadedSolrHomeDir = zki.downloadConfigDir(zkClient, configName, downloadedSolrHomeDir);
- mySolrHomeDir = downloadedSolrHomeDir.getAbsolutePath();
- } catch (KeeperException | InterruptedException | IOException e) {
- throw new MorphlineCompilationException("Cannot download schema.xml from ZooKeeper", config, e);
- } finally {
- zkClient.close();
- }
- }
-
- LOG.debug("SolrLocator loading IndexSchema from dir {}", mySolrHomeDir);
- try {
- SolrResourceLoader loader = new SolrResourceLoader(Paths.get(mySolrHomeDir));
- SolrConfig solrConfig = new SolrConfig(loader, "solrconfig.xml", null);
- InputSource is = new InputSource(loader.openSchema("schema.xml"));
- is.setSystemId(SystemIdResolver.createSystemIdFromResourceName("schema.xml"));
-
- IndexSchema schema = new IndexSchema(solrConfig, "schema.xml", is);
- validateSchema(schema);
- return schema;
- } catch (ParserConfigurationException | IOException | SAXException e) {
- throw new MorphlineRuntimeException(e);
- }
- } finally {
- if (downloadedSolrHomeDir != null) {
- try {
- FileUtils.deleteDirectory(downloadedSolrHomeDir);
- } catch (IOException e) {
- LOG.warn("Cannot delete tmp directory", e);
- }
- }
- }
- }
-
- private void validateSchema(IndexSchema schema) {
- if (schema.getUniqueKeyField() == null) {
- throw new MorphlineCompilationException("Solr schema.xml is missing unique key field", config);
- }
- if (!schema.getUniqueKeyField().isRequired()) {
- throw new MorphlineCompilationException("Solr schema.xml must contain a required unique key field", config);
- }
- }
-
- @Override
- public String toString() {
- return toConfig(null).root().render(ConfigRenderOptions.concise());
- }
-
- public Config toConfig(String key) {
- String json = "";
- if (key != null) {
- json = toJson(key) + " : ";
- }
- json +=
- "{" +
- " collection : " + toJson(collectionName) + ", " +
- " zkHost : " + toJson(zkHost) + ", " +
- " solrUrl : " + toJson(solrUrl) + ", " +
- " solrHomeDir : " + toJson(solrHomeDir) + ", " +
- " batchSize : " + toJson(batchSize) + " " +
- "}";
- return ConfigFactory.parseString(json);
- }
-
- private String toJson(Object key) {
- String str = key == null ? "" : key.toString();
- str = ConfigUtil.quoteString(str);
- return str;
- }
-
- public String getCollectionName() {
- return this.collectionName;
- }
-
- public void setCollectionName(String collectionName) {
- this.collectionName = collectionName;
- }
-
- public String getZkHost() {
- return this.zkHost;
- }
-
- public void setZkHost(String zkHost) {
- this.zkHost = zkHost;
- }
-
- public String getSolrHomeDir() {
- return this.solrHomeDir;
- }
-
- public void setSolrHomeDir(String solrHomeDir) {
- this.solrHomeDir = solrHomeDir;
- }
-
- public String getServerUrl() {
- return this.solrUrl;
- }
-
- public void setServerUrl(String solrUrl) {
- this.solrUrl = solrUrl;
- }
-
- public int getBatchSize() {
- return this.batchSize;
- }
-
- public void setBatchSize(int batchSize) {
- this.batchSize = batchSize;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SolrMorphlineContext.java
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SolrMorphlineContext.java b/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SolrMorphlineContext.java
deleted file mode 100644
index b3cd301..0000000
--- a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SolrMorphlineContext.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.morphlines.solr;
-
-import org.apache.solr.schema.IndexSchema;
-
-import org.kitesdk.morphline.api.MorphlineContext;
-
-/**
- * A context that is specific to Solr.
- */
-public class SolrMorphlineContext extends MorphlineContext {
-
- private DocumentLoader loader;
- private IndexSchema schema;
-
- /** For public access use {@link Builder#build()} instead */
- protected SolrMorphlineContext() {}
-
- public DocumentLoader getDocumentLoader() {
- return loader;
- }
-
- public IndexSchema getIndexSchema() {
- return schema;
- }
-
-
- ///////////////////////////////////////////////////////////////////////////////
- // Nested classes:
- ///////////////////////////////////////////////////////////////////////////////
- /**
- * Helper to construct a {@link SolrMorphlineContext} instance.
- */
- public static class Builder extends MorphlineContext.Builder {
-
- private DocumentLoader loader;
- private IndexSchema schema;
-
- public Builder() {}
-
- public Builder setDocumentLoader(DocumentLoader loader) {
- this.loader = loader;
- return this;
- }
-
- public Builder setIndexSchema(IndexSchema schema) {
- this.schema = schema;
- return this;
- }
-
- @Override
- public SolrMorphlineContext build() {
- ((SolrMorphlineContext)context).loader = loader;
- ((SolrMorphlineContext)context).schema = schema;
- return (SolrMorphlineContext) super.build();
- }
-
- @Override
- protected SolrMorphlineContext create() {
- return new SolrMorphlineContext();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/TokenizeTextBuilder.java
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/TokenizeTextBuilder.java b/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/TokenizeTextBuilder.java
deleted file mode 100644
index 7c96f3f..0000000
--- a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/TokenizeTextBuilder.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.morphlines.solr;
-
-import java.io.IOException;
-import java.io.Reader;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.solr.schema.FieldType;
-import org.apache.solr.schema.IndexSchema;
-
-import org.kitesdk.morphline.api.Command;
-import org.kitesdk.morphline.api.CommandBuilder;
-import org.kitesdk.morphline.api.MorphlineCompilationException;
-import org.kitesdk.morphline.api.MorphlineContext;
-import org.kitesdk.morphline.api.MorphlineRuntimeException;
-import org.kitesdk.morphline.api.Record;
-import org.kitesdk.morphline.base.AbstractCommand;
-import com.typesafe.config.Config;
-
-/**
- * A command that uses the embedded Solr/Lucene Analyzer library to generate tokens from a text
- * string, without sending data to a Solr server.
- */
-public final class TokenizeTextBuilder implements CommandBuilder {
-
- @Override
- public Collection<String> getNames() {
- return Collections.singletonList("tokenizeText");
- }
-
- @Override
- public Command build(Config config, Command parent, Command child, MorphlineContext context) {
- return new TokenizeText(this, config, parent, child, context);
- }
-
-
- ///////////////////////////////////////////////////////////////////////////////
- // Nested classes:
- ///////////////////////////////////////////////////////////////////////////////
- private static final class TokenizeText extends AbstractCommand {
-
- private final String inputFieldName;
- private final String outputFieldName;
- private final Analyzer analyzer;
- private final CharTermAttribute token; // cached
- private final ReusableStringReader reader = new ReusableStringReader(); // cached
-
- public TokenizeText(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
- super(builder, config, parent, child, context);
- this.inputFieldName = getConfigs().getString(config, "inputField");
- this.outputFieldName = getConfigs().getString(config, "outputField");
- String solrFieldType = getConfigs().getString(config, "solrFieldType");
- Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator");
- SolrLocator locator = new SolrLocator(solrLocatorConfig, context);
- LOG.debug("solrLocator: {}", locator);
- IndexSchema schema = locator.getIndexSchema();
- FieldType fieldType = schema.getFieldTypeByName(solrFieldType);
- if (fieldType == null) {
- throw new MorphlineCompilationException("Missing Solr field type in schema.xml for name: " + solrFieldType, config);
- }
- this.analyzer = Objects.requireNonNull(fieldType.getIndexAnalyzer());
- // register CharTermAttribute for later (implicit) reuse
- this.token = Objects.requireNonNull(analyzer.tokenStream("content", reader).addAttribute(CharTermAttribute.class));
- validateArguments();
- }
-
- @Override
- protected boolean doProcess(Record record) {
- try {
- List outputValues = record.get(outputFieldName);
- for (Object value : record.get(inputFieldName)) {
- reader.setValue(value.toString());
- TokenStream tokenStream = analyzer.tokenStream("content", reader);
- tokenStream.reset();
- while (tokenStream.incrementToken()) {
- if (token.length() > 0) { // incrementToken() updates the token!
- String tokenStr = new String(token.buffer(), 0, token.length());
- outputValues.add(tokenStr);
- }
- }
- tokenStream.end();
- tokenStream.close();
- }
- } catch (IOException e) {
- throw new MorphlineRuntimeException(e);
- }
-
- // pass record to next command in chain:
- return super.doProcess(record);
- }
-
- }
-
- private static final class ReusableStringReader extends Reader {
- private int pos = 0, size = 0;
- private String s = null;
-
- void setValue(String s) {
- this.s = s;
- this.size = s.length();
- this.pos = 0;
- }
-
- @Override
- public int read() {
- if (pos < size) {
- return s.charAt(pos++);
- } else {
- s = null;
- return -1;
- }
- }
-
- @Override
- public int read(char[] c, int off, int len) {
- if (pos < size) {
- len = Math.min(len, size-pos);
- s.getChars(pos, pos+len, c, off);
- pos += len;
- return len;
- } else {
- s = null;
- return -1;
- }
- }
-
- @Override
- public void close() {
- pos = size; // this prevents NPE when reading after close!
- s = null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/ZooKeeperDownloader.java
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/ZooKeeperDownloader.java b/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/ZooKeeperDownloader.java
deleted file mode 100644
index 7185531..0000000
--- a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/ZooKeeperDownloader.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.morphlines.solr;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.List;
-
-import org.apache.solr.cloud.ZkController;
-import org.apache.solr.common.cloud.Aliases;
-import org.apache.solr.common.cloud.ClusterState;
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.cloud.ZkConfigManager;
-import org.apache.solr.common.cloud.ZkNodeProps;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.util.StrUtils;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-import com.google.common.io.Files;
-
-/**
- * Downloads SolrCloud information from ZooKeeper.
- */
-final class ZooKeeperDownloader {
-
- private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public SolrZkClient getZkClient(String zkHost) {
- if (zkHost == null) {
- throw new IllegalArgumentException("zkHost must not be null");
- }
-
- SolrZkClient zkClient;
- try {
- zkClient = new SolrZkClient(zkHost, 30000);
- } catch (Exception e) {
- throw new IllegalArgumentException("Cannot connect to ZooKeeper: " + zkHost, e);
- }
- return zkClient;
- }
-
- /**
- * Returns config value given collection name
- * Borrowed heavily from Solr's ZKController.
- */
- public String readConfigName(SolrZkClient zkClient, String collection)
- throws KeeperException, InterruptedException {
- if (collection == null) {
- throw new IllegalArgumentException("collection must not be null");
- }
- String configName = null;
-
- // first check for alias
- byte[] aliasData = zkClient.getData(ZkStateReader.ALIASES, null, null, true);
- Aliases aliases = ClusterState.load(aliasData);
- String alias = aliases.getCollectionAlias(collection);
- if (alias != null) {
- List<String> aliasList = StrUtils.splitSmart(alias, ",", true);
- if (aliasList.size() > 1) {
- throw new IllegalArgumentException("collection cannot be an alias that maps to multiple collections");
- }
- collection = aliasList.get(0);
- }
-
- String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection;
- if (LOG.isInfoEnabled()) {
- LOG.info("Load collection config from:" + path);
- }
- byte[] data = zkClient.getData(path, null, null, true);
-
- if(data != null) {
- ZkNodeProps props = ZkNodeProps.load(data);
- configName = props.getStr(ZkController.CONFIGNAME_PROP);
- }
-
- if (configName != null && !zkClient.exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + configName, true)) {
- LOG.error("Specified config does not exist in ZooKeeper:" + configName);
- throw new IllegalArgumentException("Specified config does not exist in ZooKeeper:"
- + configName);
- }
-
- return configName;
- }
-
- /**
- * Download and return the config directory from ZK
- */
- public File downloadConfigDir(SolrZkClient zkClient, String configName, File dir)
- throws IOException, InterruptedException, KeeperException {
- Preconditions.checkArgument(dir.exists());
- Preconditions.checkArgument(dir.isDirectory());
- ZkConfigManager manager = new ZkConfigManager(zkClient);
- manager.downloadConfigDir(configName, dir.toPath());
- File confDir = new File(dir, "conf");
- if (!confDir.isDirectory()) {
- // create a temporary directory with "conf" subdir and mv the config in there. This is
- // necessary because of CDH-11188; solrctl does not generate nor accept directories with e.g.
- // conf/solrconfig.xml which is necessary for proper solr operation. This should work
- // even if solrctl changes.
- confDir = new File(Files.createTempDir().getAbsolutePath(), "conf");
- confDir.getParentFile().deleteOnExit();
- Files.move(dir, confDir);
- dir = confDir.getParentFile();
- }
- verifyConfigDir(confDir);
- return dir;
- }
-
- private void verifyConfigDir(File confDir) throws IOException {
- File solrConfigFile = new File(confDir, "solrconfig.xml");
- if (!solrConfigFile.exists()) {
- throw new IOException("Detected invalid Solr config dir in ZooKeeper - Reason: File not found: "
- + solrConfigFile.getName());
- }
- if (!solrConfigFile.isFile()) {
- throw new IOException("Detected invalid Solr config dir in ZooKeeper - Reason: Not a file: "
- + solrConfigFile.getName());
- }
- if (!solrConfigFile.canRead()) {
- throw new IOException("Insufficient permissions to read file: " + solrConfigFile);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/package-info.java
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/package-info.java b/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/package-info.java
deleted file mode 100644
index f4b91ec..0000000
--- a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/package-info.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Morphlines Solr related code.
- */
-package org.apache.solr.morphlines.solr;
-
-
-
-
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/java/overview.html
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/java/overview.html b/solr/contrib/morphlines-core/src/java/overview.html
deleted file mode 100644
index 7f8ad13..0000000
--- a/solr/contrib/morphlines-core/src/java/overview.html
+++ /dev/null
@@ -1,21 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<html>
-<body>
-Apache Solr Search Server: Solr Core Morphline Commands
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/test-files/custom-mimetypes.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/test-files/custom-mimetypes.xml b/solr/contrib/morphlines-core/src/test-files/custom-mimetypes.xml
deleted file mode 100644
index 6891e42..0000000
--- a/solr/contrib/morphlines-core/src/test-files/custom-mimetypes.xml
+++ /dev/null
@@ -1,38 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
- license agreements. See the NOTICE file distributed with this work for additional
- information regarding copyright ownership. The ASF licenses this file to
- You under the Apache License, Version 2.0 (the "License"); you may not use
- this file except in compliance with the License. You may obtain a copy of
- the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
- by applicable law or agreed to in writing, software distributed under the
- License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
- OF ANY KIND, either express or implied. See the License for the specific
- language governing permissions and limitations under the License. -->
-
-<mime-info>
-
- <mime-type type="text/space-separated-values">
- <glob pattern="*.ssv"/>
- </mime-type>
-
- <mime-type type="avro/binary">
- <magic priority="50">
- <match value="0x4f626a01" type="string" offset="0"/>
- </magic>
- <glob pattern="*.avro"/>
- </mime-type>
-
- <mime-type type="mytwittertest/json+delimited+length">
- <magic priority="50">
- <match value="[0-9]+(\r)?\n\\{"" type="regex" offset="0:16"/>
- </magic>
- </mime-type>
-
- <mime-type type="application/hadoop-sequence-file">
- <magic priority="50">
- <match value="SEQ[\0-\6]" type="regex" offset="0"/>
- </magic>
- </mime-type>
-
-</mime-info>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/test-files/log4j.properties
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/test-files/log4j.properties b/solr/contrib/morphlines-core/src/test-files/log4j.properties
deleted file mode 100644
index 40fc92b..0000000
--- a/solr/contrib/morphlines-core/src/test-files/log4j.properties
+++ /dev/null
@@ -1,7 +0,0 @@
-# Logging level
-log4j.rootLogger=INFO, CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.Target=System.err
-log4j.appender.CONSOLE.layout=org.apache.log4j.EnhancedPatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%-4r %-5p (%t) [%X{node_name} %X{collection} %X{shard} %X{replica} %X{core}] %c{1.} %m%n
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/test-files/morphlines-core.marker
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/test-files/morphlines-core.marker b/solr/contrib/morphlines-core/src/test-files/morphlines-core.marker
deleted file mode 100644
index f4ed7be..0000000
--- a/solr/contrib/morphlines-core/src/test-files/morphlines-core.marker
+++ /dev/null
@@ -1 +0,0 @@
-# Marker file, so we can lookup this file from classpath to get the resources folder for morphlines-core.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/currency.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/currency.xml b/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/currency.xml
deleted file mode 100644
index 3a9c58a..0000000
--- a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/currency.xml
+++ /dev/null
@@ -1,67 +0,0 @@
-<?xml version="1.0" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!-- Example exchange rates file for CurrencyField type named "currency" in example schema -->
-
-<currencyConfig version="1.0">
- <rates>
- <!-- Updated from http://www.exchangerate.com/ at 2011-09-27 -->
- <rate from="USD" to="ARS" rate="4.333871" comment="ARGENTINA Peso" />
- <rate from="USD" to="AUD" rate="1.025768" comment="AUSTRALIA Dollar" />
- <rate from="USD" to="EUR" rate="0.743676" comment="European Euro" />
- <rate from="USD" to="BRL" rate="1.881093" comment="BRAZIL Real" />
- <rate from="USD" to="CAD" rate="1.030815" comment="CANADA Dollar" />
- <rate from="USD" to="CLP" rate="519.0996" comment="CHILE Peso" />
- <rate from="USD" to="CNY" rate="6.387310" comment="CHINA Yuan" />
- <rate from="USD" to="CZK" rate="18.47134" comment="CZECH REP. Koruna" />
- <rate from="USD" to="DKK" rate="5.515436" comment="DENMARK Krone" />
- <rate from="USD" to="HKD" rate="7.801922" comment="HONG KONG Dollar" />
- <rate from="USD" to="HUF" rate="215.6169" comment="HUNGARY Forint" />
- <rate from="USD" to="ISK" rate="118.1280" comment="ICELAND Krona" />
- <rate from="USD" to="INR" rate="49.49088" comment="INDIA Rupee" />
- <rate from="USD" to="XDR" rate="0.641358" comment="INTNL MON. FUND SDR" />
- <rate from="USD" to="ILS" rate="3.709739" comment="ISRAEL Sheqel" />
- <rate from="USD" to="JPY" rate="76.32419" comment="JAPAN Yen" />
- <rate from="USD" to="KRW" rate="1169.173" comment="KOREA (SOUTH) Won" />
- <rate from="USD" to="KWD" rate="0.275142" comment="KUWAIT Dinar" />
- <rate from="USD" to="MXN" rate="13.85895" comment="MEXICO Peso" />
- <rate from="USD" to="NZD" rate="1.285159" comment="NEW ZEALAND Dollar" />
- <rate from="USD" to="NOK" rate="5.859035" comment="NORWAY Krone" />
- <rate from="USD" to="PKR" rate="87.57007" comment="PAKISTAN Rupee" />
- <rate from="USD" to="PEN" rate="2.730683" comment="PERU Sol" />
- <rate from="USD" to="PHP" rate="43.62039" comment="PHILIPPINES Peso" />
- <rate from="USD" to="PLN" rate="3.310139" comment="POLAND Zloty" />
- <rate from="USD" to="RON" rate="3.100932" comment="ROMANIA Leu" />
- <rate from="USD" to="RUB" rate="32.14663" comment="RUSSIA Ruble" />
- <rate from="USD" to="SAR" rate="3.750465" comment="SAUDI ARABIA Riyal" />
- <rate from="USD" to="SGD" rate="1.299352" comment="SINGAPORE Dollar" />
- <rate from="USD" to="ZAR" rate="8.329761" comment="SOUTH AFRICA Rand" />
- <rate from="USD" to="SEK" rate="6.883442" comment="SWEDEN Krona" />
- <rate from="USD" to="CHF" rate="0.906035" comment="SWITZERLAND Franc" />
- <rate from="USD" to="TWD" rate="30.40283" comment="TAIWAN Dollar" />
- <rate from="USD" to="THB" rate="30.89487" comment="THAILAND Baht" />
- <rate from="USD" to="AED" rate="3.672955" comment="U.A.E. Dirham" />
- <rate from="USD" to="UAH" rate="7.988582" comment="UKRAINE Hryvnia" />
- <rate from="USD" to="GBP" rate="0.647910" comment="UNITED KINGDOM Pound" />
-
- <!-- Cross-rates for some common currencies -->
- <rate from="EUR" to="GBP" rate="0.869914" />
- <rate from="EUR" to="NOK" rate="7.800095" />
- <rate from="GBP" to="NOK" rate="8.966508" />
- </rates>
-</currencyConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/elevate.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/elevate.xml b/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/elevate.xml
deleted file mode 100644
index 2c09ebe..0000000
--- a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/elevate.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!-- If this file is found in the config directory, it will only be
- loaded once at startup. If it is found in Solr's data
- directory, it will be re-loaded every commit.
-
- See http://wiki.apache.org/solr/QueryElevationComponent for more info
-
--->
-<elevate>
- <!-- Query elevation examples
- <query text="foo bar">
- <doc id="1" />
- <doc id="2" />
- <doc id="3" />
- </query>
-
-for use with techproducts example
-
- <query text="ipod">
- <doc id="MA147LL/A" /> put the actual ipod at the top
- <doc id="IW-02" exclude="true" /> exclude this cable
- </query>
--->
-
-</elevate>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/contractions_ca.txt
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/contractions_ca.txt b/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/contractions_ca.txt
deleted file mode 100644
index 307a85f..0000000
--- a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/contractions_ca.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# Set of Catalan contractions for ElisionFilter
-# TODO: load this as a resource from the analyzer and sync it in build.xml
-d
-l
-m
-n
-s
-t
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/contractions_fr.txt
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/contractions_fr.txt b/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/contractions_fr.txt
deleted file mode 100644
index 722db58..0000000
--- a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/contractions_fr.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-# Set of French contractions for ElisionFilter
-# TODO: load this as a resource from the analyzer and sync it in build.xml
-l
-m
-t
-qu
-n
-s
-j
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/contractions_ga.txt
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/contractions_ga.txt b/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/contractions_ga.txt
deleted file mode 100644
index 9ebe7fa..0000000
--- a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/contractions_ga.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-# Set of Irish contractions for ElisionFilter
-# TODO: load this as a resource from the analyzer and sync it in build.xml
-d
-m
-b
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/contractions_it.txt
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/contractions_it.txt b/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/contractions_it.txt
deleted file mode 100644
index cac0409..0000000
--- a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/contractions_it.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-# Set of Italian contractions for ElisionFilter
-# TODO: load this as a resource from the analyzer and sync it in build.xml
-c
-l
-all
-dall
-dell
-nell
-sull
-coll
-pell
-gl
-agl
-dagl
-degl
-negl
-sugl
-un
-m
-t
-s
-v
-d
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/hyphenations_ga.txt
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/hyphenations_ga.txt b/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/hyphenations_ga.txt
deleted file mode 100644
index 4d2642c..0000000
--- a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/hyphenations_ga.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-# Set of Irish hyphenations for StopFilter
-# TODO: load this as a resource from the analyzer and sync it in build.xml
-h
-n
-t
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1a574df/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/stemdict_nl.txt
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/stemdict_nl.txt b/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/stemdict_nl.txt
deleted file mode 100644
index 4410729..0000000
--- a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/lang/stemdict_nl.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-# Set of overrides for the dutch stemmer
-# TODO: load this as a resource from the analyzer and sync it in build.xml
-fiets fiets
-bromfiets bromfiets
-ei eier
-kind kinder