You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2019/10/15 05:19:51 UTC
[lucene-solr] 03/03: Revert "SOLR-13821: Package Store for storing
package artifacts (#929)"
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch branch_8_3
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 6bb41793e5bebb8e7a930cec1a1200fa70183137
Author: noble <no...@apache.org>
AuthorDate: Tue Oct 15 15:58:49 2019 +1100
Revert "SOLR-13821: Package Store for storing package artifacts (#929)"
This reverts commit f19aaa8097e756991afbf4e5dbdea773e87a872d.
This feature not to be released in 8.3
---
solr/CHANGES.txt | 2 -
.../java/org/apache/solr/core/BlobRepository.java | 2 +-
.../java/org/apache/solr/core/CoreContainer.java | 8 -
.../apache/solr/filestore/DistribPackageStore.java | 495 ---------------------
.../org/apache/solr/filestore/PackageStore.java | 122 -----
.../org/apache/solr/filestore/PackageStoreAPI.java | 273 ------------
.../solr/security/PermissionNameProvider.java | 2 -
.../apache/solr/servlet/SolrRequestParsers.java | 4 +-
.../src/java/org/apache/solr/util/CryptoKeys.java | 47 +-
.../core/src/test-files/cryptokeys/priv_key512.pem | 9 -
solr/core/src/test-files/cryptokeys/pub_key512.der | Bin 94 -> 0 bytes
.../test-files/runtimecode/runtimelibs_v3.jar.bin | Bin 7337 -> 0 bytes
solr/core/src/test-files/runtimecode/sig.txt | 105 -----
.../solr/filestore/TestDistribPackageStore.java | 250 -----------
.../org/apache/solr/client/solrj/SolrResponse.java | 9 +-
.../solr/client/solrj/request/V2Request.java | 39 +-
.../apache/solr/common/params/CommonParams.java | 3 -
.../java/org/apache/solr/common/util/StrUtils.java | 207 ++++-----
.../java/org/apache/solr/common/util/Utils.java | 144 ++----
.../org/apache/solr/cloud/SolrCloudTestCase.java | 4 +-
20 files changed, 135 insertions(+), 1590 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index b4a0a83..c32fc4c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -79,8 +79,6 @@ New Features
* SOLR-8241: Add CaffeineCache, an efficient implementation of SolrCache.(Ben Manes, Shawn Heisey, David Smiley, Andrzej Bialecki)
-* SOLR-13821: A Package store to store and load package artefacts (noble, Ishan Chattopadhyaya)
-
Improvements
----------------------
diff --git a/solr/core/src/java/org/apache/solr/core/BlobRepository.java b/solr/core/src/java/org/apache/solr/core/BlobRepository.java
index 59bd795..24bb88e 100644
--- a/solr/core/src/java/org/apache/solr/core/BlobRepository.java
+++ b/solr/core/src/java/org/apache/solr/core/BlobRepository.java
@@ -62,7 +62,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
public class BlobRepository {
private static final long MAX_JAR_SIZE = Long.parseLong(System.getProperty("runtme.lib.size", String.valueOf(5 * 1024 * 1024)));
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- public static final Random RANDOM;
+ static final Random RANDOM;
static final Pattern BLOB_KEY_PATTERN_CHECKER = Pattern.compile(".*/\\d+");
static {
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 69aa976..7040610 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -47,7 +47,6 @@ import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.store.Directory;
-import org.apache.solr.api.AnnotatedApi;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
@@ -77,7 +76,6 @@ import org.apache.solr.common.util.Utils;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.core.backup.repository.BackupRepositoryFactory;
-import org.apache.solr.filestore.PackageStoreAPI;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.handler.SnapShooter;
import org.apache.solr.handler.admin.AutoscalingHistoryHandler;
@@ -220,8 +218,6 @@ public class CoreContainer {
protected volatile AutoscalingHistoryHandler autoscalingHistoryHandler;
- private PackageStoreAPI packageStoreAPI;
-
// Bits for the state variable.
public final static long LOAD_COMPLETE = 0x1L;
@@ -604,10 +600,6 @@ public class CoreContainer {
}
}
- packageStoreAPI = new PackageStoreAPI(this);
- containerHandlers.getApiBag().register(new AnnotatedApi(packageStoreAPI.readAPI), Collections.EMPTY_MAP);
- containerHandlers.getApiBag().register(new AnnotatedApi(packageStoreAPI.writeAPI), Collections.EMPTY_MAP);
-
metricManager = new SolrMetricManager(loader, cfg.getMetricsConfig());
coreContainerWorkExecutor = MetricUtils.instrumentedExecutorService(
diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
deleted file mode 100644
index 910f29b..0000000
--- a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
+++ /dev/null
@@ -1,495 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.filestore;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.lang.invoke.MethodHandles;
-import java.nio.ByteBuffer;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.function.Consumer;
-import java.util.function.Predicate;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.lucene.util.IOUtils;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.BlobRepository;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.filestore.PackageStoreAPI.MetaData;
-import org.apache.zookeeper.server.ByteBufferInputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
-import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
-
-
-public class DistribPackageStore implements PackageStore {
- static final long MAX_PKG_SIZE = Long.parseLong(System.getProperty("max.file.store.size", String.valueOf(100 * 1024 * 1024)));
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private final CoreContainer coreContainer;
- private Map<String, FileInfo> tmpFiles = new ConcurrentHashMap<>();
- public DistribPackageStore(CoreContainer coreContainer) {
- this.coreContainer = coreContainer;
- ensurePackageStoreDir(coreContainer.getResourceLoader().getInstancePath());
-
- }
-
- private String myNode() {
- return coreContainer.getZkController().getNodeName();
- }
-
-
- /**
- * get a list of nodes randomly shuffled
- * * @lucene.internal
- */
- public ArrayList<String> shuffledNodes() {
- Set<String> liveNodes = coreContainer.getZkController().getZkStateReader().getClusterState().getLiveNodes();
- ArrayList<String> l = new ArrayList(liveNodes);
- l.remove(myNode());
- Collections.shuffle(l, BlobRepository.RANDOM);
- return l;
- }
-
-
- @Override
- public Path getRealpath(String path) {
- if (File.separatorChar == '\\') {
- path = path.replaceAll("/", File.separator);
- }
- if (path.charAt(0) != File.separatorChar) {
- path = File.separator + path;
- }
- return new File(this.coreContainer.getResourceLoader().getInstancePath() +
- "/" + PackageStoreAPI.PACKAGESTORE_DIRECTORY + path).toPath();
- }
-
- class FileInfo {
- final String path;
- String metaPath;
- ByteBuffer fileData, metaData;
-
-
- FileInfo(String path) {
- this.path = path;
- }
-
- public String getMetaPath() {
- if (metaPath == null) {
- int idx = path.lastIndexOf('/');
- metaPath = path.substring(0, idx + 1) + "." + path.substring(idx + 1) + ".json";
- }
- return metaPath;
- }
-
-
- private void persistToFile(ByteBuffer data, ByteBuffer meta) throws IOException {
- synchronized (DistribPackageStore.this) {
- this.metaData = meta;
- this.fileData = data;
- Path realpath = getRealpath(path);
- File file = realpath.toFile();
- File parent = file.getParentFile();
- if (!parent.exists()) {
- parent.mkdirs();
- }
- Map m = (Map) Utils.fromJSON(meta.array());
- if (m == null || m.isEmpty()) {
- throw new SolrException(SERVER_ERROR, "invalid metadata , discarding : " + path);
- }
-
-
- File metdataFile = getRealpath(getMetaPath()).toFile();
-
- try (FileOutputStream fos = new FileOutputStream(metdataFile)) {
- fos.write(meta.array(), 0, meta.limit());
- }
- IOUtils.fsync(metdataFile.toPath(), false);
-
- try (FileOutputStream fos = new FileOutputStream(file)) {
- fos.write(data.array(), 0, data.limit());
- }
- log.info("persisted a file {} and metadata. sizes {} {}", path, data.limit(), meta.limit());
- IOUtils.fsync(file.toPath(), false);
- }
- }
-
-
- public boolean exists(boolean validateContent, boolean fetchMissing) throws IOException {
- File file = getRealpath(path).toFile();
- if (!file.exists()) {
- if (fetchMissing) {
- return fetchFromAnyNode();
- } else {
- return false;
- }
- }
-
- if (validateContent) {
- MetaData metaData = readMetaData();
- if (metaData == null) return false;
- try (InputStream is = new FileInputStream(getRealpath(path).toFile())) {
- if (!Objects.equals(DigestUtils.sha512Hex(is), metaData.sha512)) {
- deleteFile();
- } else {
- return true;
- }
- } catch (Exception e) {
- throw new SolrException(SERVER_ERROR, "unable to parse metadata json file");
- }
- } else {
- return true;
- }
-
- return false;
- }
-
- private void deleteFile() {
- try {
- IOUtils.deleteFilesIfExist(getRealpath(path), getRealpath(getMetaPath()));
- } catch (IOException e) {
- log.error("Unable to delete files: "+path);
- }
-
- }
-
- private boolean fetchFileFromNodeAndPersist(String fromNode) {
- log.info("fetching a file {} from {} ", path, fromNode);
- String url = coreContainer.getZkController().getZkStateReader().getBaseUrlForNodeName(fromNode);
- if (url == null) throw new SolrException(BAD_REQUEST, "No such node");
- String baseUrl = url.replace("/solr", "/api");
-
- ByteBuffer metadata = null;
- Map m = null;
- try {
- metadata = Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(),
- baseUrl + "/node/files" + getMetaPath(),
- Utils.newBytesConsumer((int) MAX_PKG_SIZE));
- m = (Map) Utils.fromJSON(metadata.array());
- } catch (SolrException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error fetching metadata", e);
- }
-
- try {
- ByteBuffer filedata = Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(),
- baseUrl + "/node/files" + path,
- Utils.newBytesConsumer((int) MAX_PKG_SIZE));
- String sha512 = DigestUtils.sha512Hex(new ByteBufferInputStream(filedata));
- String expected = (String) m.get("sha512");
- if (!sha512.equals(expected)) {
- throw new SolrException(SERVER_ERROR, "sha512 mismatch downloading : " + path + " from node : " + fromNode);
- }
- persistToFile(filedata, metadata);
- return true;
- } catch (SolrException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error fetching data", e);
- } catch (IOException ioe) {
- throw new SolrException(SERVER_ERROR, "Error persisting file", ioe);
- }
-
- }
-
- boolean fetchFromAnyNode() {
-
- ArrayList<String> l = shuffledNodes();
- ZkStateReader stateReader = coreContainer.getZkController().getZkStateReader();
- for (String liveNode : l) {
- try {
- String baseurl = stateReader.getBaseUrlForNodeName(liveNode);
- String url = baseurl.replace("/solr", "/api");
- String reqUrl = url + "/node/files" + path +
- "?meta=true&wt=javabin&omitHeader=true";
- boolean nodeHasBlob = false;
- Object nl = Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(), reqUrl, Utils.JAVABINCONSUMER);
- if (Utils.getObjectByPath(nl, false, Arrays.asList("files", getMetaPath())) != null) {
- nodeHasBlob = true;
- }
-
- if (nodeHasBlob) {
- boolean success = fetchFileFromNodeAndPersist(liveNode);
- if (success) return true;
- }
- } catch (Exception e) {
- //it's OK for some nodes to fail
- }
- }
-
- return false;
- }
-
- String getSimpleName() {
- int idx = path.lastIndexOf("/");
- if (idx == -1) return path;
- return path.substring(idx + 1);
- }
-
- public Path realPath() {
- return getRealpath(path);
- }
-
- MetaData readMetaData() throws IOException {
- File file = getRealpath(getMetaPath()).toFile();
- if (file.exists()) {
- try (InputStream fis = new FileInputStream(file)) {
- return new MetaData((Map) Utils.fromJSON(fis));
- }
- }
- return null;
-
- }
-
-
-
-
- public FileDetails getDetails() {
- FileType type = getType(path);
-
- return new FileDetails() {
- @Override
- public MetaData getMetaData() {
- try {
- return readMetaData();
- } catch (Exception e){
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public Date getTimeStamp() {
- return new Date(realPath().toFile().lastModified());
- }
-
- @Override
- public boolean isDir() {
- return type == FileType.DIRECTORY;
- }
-
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- MetaData metaData = readMetaData();
- ew.put(CommonParams.NAME, getSimpleName());
- if (type == FileType.DIRECTORY) {
- ew.put("dir", true);
- return;
- }
- ew.put("timestamp", getTimeStamp());
- metaData.writeMap(ew);
-
- }
- };
-
-
- }
-
- public void readData(Consumer<FileEntry> consumer) throws IOException {
- MetaData meta = readMetaData();
- try (InputStream is = new FileInputStream(realPath().toFile())) {
- consumer.accept(new FileEntry(null, meta,path ){
- @Override
- public InputStream getInputStream() {
- return is;
- }
- });
- }
- }
- }
-
-
- @Override
- public void put(FileEntry entry) throws IOException {
- FileInfo info = new FileInfo(entry.path);
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- Utils.writeJson(entry.getMetaData(), baos, true);
- byte[] bytes = baos.toByteArray();
- info.persistToFile(entry.buf, ByteBuffer.wrap(bytes, 0, bytes.length));
- tmpFiles.put(entry.getPath(), info);
- List<String> nodes = shuffledNodes();
- int i = 0;
- int FETCHFROM_SRC = 50;
- String myNodeName = myNode();
- try {
- for (String node : nodes) {
- String baseUrl = coreContainer.getZkController().getZkStateReader().getBaseUrlForNodeName(node);
- String url = baseUrl.replace("/solr", "/api") + "/node/files" + entry.getPath() + "?getFrom=";
- if (i < FETCHFROM_SRC) {
- // this is to protect very large clusters from overwhelming a single node
- // the first FETCHFROM_SRC nodes will be asked to fetch from this node.
- // it's there in the memory now. So , it must be served fast
- url += myNodeName;
- } else {
- if (i == FETCHFROM_SRC) {
- // This is just an optimization
- // at this point a bunch of nodes are already downloading from me
- // I'll wait for them to finish before asking other nodes to download from each other
- try {
- Thread.sleep(2 * 1000);
- } catch (Exception e) {
- }
- }
- // trying to avoid the thundering herd problem when there are a very large no:of nodes
- // others should try to fetch it from any node where it is available. By now,
- // almost FETCHFROM_SRC other nodes may have it
- url += "*";
- }
- try {
- //fire and forget
- Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(), url, null);
- } catch (Exception e) {
- log.info("Node: " + node +
- " failed to respond for blob notification", e);
- //ignore the exception
- // some nodes may be down or not responding
- }
- i++;
- }
- } finally {
- new Thread(() -> {
- try {
- // keep the jar in memory for 10 secs , so that
- //every node can download it from memory without the file system
- Thread.sleep(10 * 1000);
- } catch (Exception e) {
- //don't care
- } finally {
- tmpFiles.remove(entry.getPath());
- }
- }).start();
-
-
- }
-
- }
-
- @Override
- public synchronized boolean fetch(String path, String from) {
- if (path == null || path.isEmpty()) return false;
- FileInfo f = new FileInfo(path);
- try {
- if(f.exists(true, false)){
- return true;
- }
- } catch (IOException e) {
- log.error("Error fetching file ", e);
- return false;
-
- }
-
- if (from == null || "*".equals(from)) {
- f.fetchFromAnyNode();
-
- } else {
- f.fetchFileFromNodeAndPersist(from);
- }
-
- return false;
- }
-
- @Override
- public synchronized void get(String path, Consumer<FileEntry> consumer) throws IOException {
- File file = getRealpath(path).toFile();
- String simpleName = file.getName();
- if (isMetaDataFile(simpleName)) {
- try (InputStream is = new FileInputStream(file)) {
- consumer.accept(new FileEntry(null, null, path) {
- //no metadata for metadata file
- @Override
- public InputStream getInputStream() {
- return is;
- }
- });
- }
- return;
- }
-
- new FileInfo(path).readData(consumer);
- }
-
-
- @Override
- public synchronized List list(String path, Predicate<String> predicate) {
- File file = getRealpath(path).toFile();
- List<FileDetails> fileDetails = new ArrayList<>();
- FileType type = getType(path);
- if (type == FileType.DIRECTORY) {
- file.list((dir, name) -> {
- if (predicate == null || predicate.test(name)) {
- if (!isMetaDataFile(name)) {
- fileDetails.add(new FileInfo(path + "/" + name).getDetails());
- }
- }
- return false;
- });
-
- } else if (type == FileType.FILE) {
-
- fileDetails.add(new FileInfo(path).getDetails());
- }
-
- return fileDetails;
- }
-
-
- @Override
- public synchronized FileType getType(String path) {
- File file = getRealpath(path).toFile();
- if (!file.exists()) return FileType.NOFILE;
- if (file.isDirectory()) return FileType.DIRECTORY;
- return isMetaDataFile(file.getName()) ? FileType.METADATA : FileType.FILE;
- }
-
- private boolean isMetaDataFile(String file) {
- return file.charAt(0) == '.' && file.endsWith(".json");
- }
-
- private void ensurePackageStoreDir(Path solrHome) {
- final File packageStoreDir = getPackageStoreDirPath(solrHome).toFile();
- if (!packageStoreDir.exists()) {
- try {
- final boolean created = packageStoreDir.mkdirs();
- if (!created) {
- log.warn("Unable to create [{}] directory in SOLR_HOME [{}]. Features requiring this directory may fail.", packageStoreDir, solrHome);
- }
- } catch (Exception e) {
- log.warn("Unable to create [" + packageStoreDir + "] directory in SOLR_HOME [" + solrHome + "]. Features requiring this directory may fail.", e);
- }
- }
- }
-
- public static Path getPackageStoreDirPath(Path solrHome) {
- return Paths.get(solrHome.toAbsolutePath().toString(), PackageStoreAPI.PACKAGESTORE_DIRECTORY).toAbsolutePath();
- }
-}
diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStore.java b/solr/core/src/java/org/apache/solr/filestore/PackageStore.java
deleted file mode 100644
index b9be691..0000000
--- a/solr/core/src/java/org/apache/solr/filestore/PackageStore.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.filestore;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.nio.file.Path;
-import java.util.Date;
-import java.util.List;
-import java.util.function.Consumer;
-import java.util.function.Predicate;
-
-import org.apache.solr.common.MapWriter;
-import org.apache.solr.filestore.PackageStoreAPI.MetaData;
-import org.apache.zookeeper.server.ByteBufferInputStream;
-
-/**
- * The interface to be implemented by any package store provider
- * * @lucene.experimental
- */
-public interface PackageStore {
-
- /**
- * Store a file into the filestore. This should ensure that it is replicated
- * across all nodes in the cluster
- */
- void put(FileEntry fileEntry) throws IOException;
-
- /**
- * read file content from a given path
- */
- void get(String path, Consumer<FileEntry> filecontent) throws IOException;
-
- /**
- * Fetch a resource from another node
- * internal
- */
- boolean fetch(String path, String from);
-
- List<FileDetails> list(String path, Predicate<String> predicate);
-
- /**
- * get the real path on filesystem
- */
- Path getRealpath(String path);
-
- /**
- * The type of the resource
- */
- FileType getType(String path);
-
- public class FileEntry {
- final ByteBuffer buf;
- final MetaData meta;
- final String path;
-
- FileEntry(ByteBuffer buf, MetaData meta, String path) {
- this.buf = buf;
- this.meta = meta;
- this.path = path;
- }
-
- public String getPath() {
- return path;
- }
-
-
- public InputStream getInputStream() {
- if (buf != null) return new ByteBufferInputStream(buf);
- return null;
-
- }
-
- /**
- * For very large files , only a stream would be available
- * This method would return null;
- */
- public ByteBuffer getBuffer() {
- return buf;
-
- }
-
- public MetaData getMetaData() {
- return meta;
- }
-
-
- }
-
- enum FileType {
- FILE, DIRECTORY, NOFILE, METADATA
- }
-
- interface FileDetails extends MapWriter {
-
- MetaData getMetaData();
-
- Date getTimeStamp();
-
- boolean isDir();
-
-
- }
-
-
-}
diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
deleted file mode 100644
index 71ee9d8..0000000
--- a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
+++ /dev/null
@@ -1,273 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.filestore;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.solr.api.Command;
-import org.apache.solr.api.EndPoint;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.cloud.CloudUtil;
-import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.ContentStream;
-import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.security.PermissionNameProvider;
-import org.apache.solr.util.CryptoKeys;
-import org.apache.solr.util.SimplePostTool;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.server.ByteBufferInputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
-
-
-public class PackageStoreAPI {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- public static final String PACKAGESTORE_DIRECTORY = "filestore";
-
-
- private final CoreContainer coreContainer;
- PackageStore packageStore;
- public final FSRead readAPI = new FSRead();
- public final FSWrite writeAPI = new FSWrite();
-
- public PackageStoreAPI(CoreContainer coreContainer) {
- this.coreContainer = coreContainer;
- packageStore = new DistribPackageStore(coreContainer);
- }
-
- public PackageStore getPackageStore() {
- return packageStore;
- }
-
- @EndPoint(
- path = "/cluster/files/*",
- method = SolrRequest.METHOD.PUT,
- permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
- public class FSWrite {
-
- static final String TMP_ZK_NODE = "/packageStoreWriteInProgress";
-
- @Command
- public void upload(SolrQueryRequest req, SolrQueryResponse rsp) {
- try {
- coreContainer.getZkController().getZkClient().create(TMP_ZK_NODE, "true".getBytes(UTF_8),
- CreateMode.EPHEMERAL, true);
-
- Iterable<ContentStream> streams = req.getContentStreams();
- if (streams == null) throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no payload");
- String path = req.getPathTemplateValues().get("*");
- if (path == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No path");
- }
- validateName(path);
- ContentStream stream = streams.iterator().next();
- try {
- ByteBuffer buf = SimplePostTool.inputStreamToByteArray(stream.getStream());
- String sha512 = DigestUtils.sha512Hex(new ByteBufferInputStream(buf));
- List<String> signatures = readSignatures(req, buf);
- Map<String, Object> vals = new HashMap<>();
- vals.put(MetaData.SHA512, sha512);
- if (signatures != null) {
- vals.put("sig", signatures);
- }
- packageStore.put(new PackageStore.FileEntry(buf, new MetaData(vals), path));
- rsp.add(CommonParams.FILE, path);
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
- }
- } catch (InterruptedException e) {
- log.error("Unexpected error", e);
- } catch (KeeperException.NodeExistsException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "A write is already in process , try later");
- } catch (KeeperException e) {
- log.error("Unexpected error", e);
- } finally {
- try {
- coreContainer.getZkController().getZkClient().delete(TMP_ZK_NODE, -1, true);
- } catch (Exception e) {
- log.error("Unexpected error ", e);
- }
- }
- }
-
- private List<String> readSignatures(SolrQueryRequest req, ByteBuffer buf)
- throws SolrException {
- String[] signatures = req.getParams().getParams("sig");
- if (signatures == null || signatures.length == 0) return null;
- List<String> sigs = Arrays.asList(signatures);
- validate(sigs, buf);
- return sigs;
- }
-
- public void validate(List<String> sigs,
- ByteBuffer buf) throws SolrException {
- Map<String, byte[]> keys = CloudUtil.getTrustedKeys(
- coreContainer.getZkController().getZkClient(), "exe");
- if (keys == null || keys.isEmpty()) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "ZK does not have any keys");
- }
- CryptoKeys cryptoKeys = null;
- try {
- cryptoKeys = new CryptoKeys(keys);
- } catch (Exception e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Error parsing public keyts in ZooKeeper");
- }
- for (String sig : sigs) {
- if (cryptoKeys.verify(sig, buf) == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Signature does not match any public key : " + sig);
- }
-
- }
- }
-
- }
-
- @EndPoint(
- path = "/node/files/*",
- method = SolrRequest.METHOD.GET,
- permission = PermissionNameProvider.Name.FILESTORE_READ_PERM)
- public class FSRead {
- @Command
- public void read(SolrQueryRequest req, SolrQueryResponse rsp) {
- String path = req.getPathTemplateValues().get("*");
- String pathCopy = path;
- String getFrom = req.getParams().get("getFrom");
- if (getFrom != null) {
- coreContainer.getUpdateShardHandler().getUpdateExecutor().submit(() -> {
- log.debug("Downloading file {}", pathCopy);
- try {
- packageStore.fetch(pathCopy, getFrom);
- } catch (Exception e) {
- log.error("Failed to download file: " + pathCopy, e);
- }
- log.info("downloaded file: {}", pathCopy);
- });
- return;
-
- }
- if (path == null) {
- path = "";
- }
-
- PackageStore.FileType typ = packageStore.getType(path);
- if (typ == PackageStore.FileType.NOFILE) {
- rsp.add("files", Collections.singletonMap(path, null));
- return;
- }
- if (typ == PackageStore.FileType.DIRECTORY) {
- rsp.add("files", Collections.singletonMap(path, packageStore.list(path, null)));
- return;
- }
- if (req.getParams().getBool("meta", false)) {
- if (typ == PackageStore.FileType.FILE) {
- int idx = path.lastIndexOf('/');
- String fileName = path.substring(idx + 1);
- String parentPath = path.substring(0, path.lastIndexOf('/'));
- List l = packageStore.list(parentPath, s -> s.equals(fileName));
- rsp.add("files", Collections.singletonMap(path, l.isEmpty() ? null : l.get(0)));
- return;
- }
- } else {
- writeRawFile(req, rsp, path);
- }
- }
-
- private void writeRawFile(SolrQueryRequest req, SolrQueryResponse rsp, String path) {
- ModifiableSolrParams solrParams = new ModifiableSolrParams();
- solrParams.add(CommonParams.WT, FILE_STREAM);
- req.setParams(SolrParams.wrapDefaults(solrParams, req.getParams()));
- rsp.add(FILE_STREAM, (SolrCore.RawWriter) os -> {
- packageStore.get(path, (it) -> {
- try {
- org.apache.commons.io.IOUtils.copy(it.getInputStream(), os);
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reading file" + path);
- }
- });
-
- });
- }
-
- }
-
- static class MetaData implements MapWriter {
- public static final String SHA512 = "sha512";
- String sha512;
- List<String> signatures;
- Map<String, Object> otherAttribs;
-
- public MetaData(Map m) {
- m = Utils.getDeepCopy(m, 3);
- this.sha512 = (String) m.remove(SHA512);
- this.signatures = (List<String>) m.remove("sig");
- this.otherAttribs = m;
- }
-
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- ew.putIfNotNull("sha512", sha512);
- ew.putIfNotNull("sig", signatures);
- if (!otherAttribs.isEmpty()) {
- otherAttribs.forEach(ew.getBiConsumer());
- }
- }
- }
-
- static final String INVALIDCHARS = " /\\#&*\n\t%@~`=+^$><?{}[]|:;!";
-
- public static void validateName(String path) {
- if (path == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "empty path");
- }
- List<String> parts = StrUtils.splitSmart(path, '/', true);
- for (String part : parts) {
- if (part.charAt(0) == '.') {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "cannot start with period");
- }
- for (int i = 0; i < part.length(); i++) {
- for (int j = 0; j < INVALIDCHARS.length(); j++) {
- if (part.charAt(i) == INVALIDCHARS.charAt(j))
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unsupported char in file name: " + part);
- }
- }
- }
- }
-}
diff --git a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
index a4c7c0d..79b4d29 100644
--- a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
+++ b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
@@ -51,8 +51,6 @@ public interface PermissionNameProvider {
AUTOSCALING_WRITE_PERM("autoscaling-write", null),
AUTOSCALING_HISTORY_READ_PERM("autoscaling-history-read", null),
METRICS_HISTORY_READ_PERM("metrics-history-read", null),
- FILESTORE_READ_PERM("filestore-read", null),
- FILESTORE_WRITE_PERM("filestore-write", null),
ALL("all", unmodifiableSet(new HashSet<>(asList("*", null))))
;
final String name;
diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
index 7c21ad1..19a4a30 100644
--- a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
@@ -63,7 +63,6 @@ import org.apache.solr.util.RTimerTree;
import org.apache.solr.util.SolrFileCleaningTracker;
import org.apache.solr.util.tracing.GlobalTracer;
-import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE;
import static org.apache.solr.common.params.CommonParams.PATH;
@@ -733,7 +732,6 @@ public class SolrRequestParsers
String contentType = req.getContentType();
String method = req.getMethod(); // No need to uppercase... HTTP verbs are case sensitive
String uri = req.getRequestURI();
- boolean isRawPut = "PUT".equals(method) && BINARY_CONTENT_TYPE.equals(contentType);
boolean isPost = "POST".equals(method);
// SOLR-6787 changed the behavior of a POST without content type. Previously it would throw an exception,
@@ -749,7 +747,7 @@ public class SolrRequestParsers
// POST was handled normally, but other methods (PUT/DELETE)
// were handled by restlet if the URI contained /schema or /config
// "handled by restlet" means that we don't attempt to handle any request body here.
- if (!isPost && !isRawPut) {
+ if (!isPost) {
if (contentType == null) {
return parseQueryString(req.getQueryString());
}
diff --git a/solr/core/src/java/org/apache/solr/util/CryptoKeys.java b/solr/core/src/java/org/apache/solr/util/CryptoKeys.java
index cb368e7..faf67fd 100644
--- a/solr/core/src/java/org/apache/solr/util/CryptoKeys.java
+++ b/solr/core/src/java/org/apache/solr/util/CryptoKeys.java
@@ -21,8 +21,7 @@ import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
-import java.io.IOException;
-import java.io.InputStream;
+
import java.lang.invoke.MethodHandles;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
@@ -43,7 +42,6 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
-import com.google.common.collect.ImmutableMap;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.Base64;
import org.slf4j.Logger;
@@ -63,7 +61,7 @@ public final class CryptoKeys {
m.put(e.getKey(), getX509PublicKey(e.getValue()));
}
- this.keys = ImmutableMap.copyOf(m);
+ this.keys = m;
}
/**
@@ -75,11 +73,11 @@ public final class CryptoKeys {
boolean verified;
try {
verified = CryptoKeys.verify(entry.getValue(), Base64.base64ToByteArray(sig), data);
- log.debug("verified {} ", verified);
+ log.info("verified {} ", verified);
if (verified) return entry.getKey();
} catch (Exception e) {
exception = e;
- log.debug("NOT verified ");
+ log.info("NOT verified ");
}
}
@@ -106,43 +104,24 @@ public final class CryptoKeys {
* @param data The data tha is signed
*/
public static boolean verify(PublicKey publicKey, byte[] sig, ByteBuffer data) throws InvalidKeyException, SignatureException {
- data = ByteBuffer.wrap(data.array(), data.arrayOffset(), data.limit());
+ int oldPos = data.position();
+ Signature signature = null;
try {
- Signature signature = Signature.getInstance("SHA1withRSA");
+ signature = Signature.getInstance("SHA1withRSA");
signature.initVerify(publicKey);
signature.update(data);
- return signature.verify(sig);
- } catch (NoSuchAlgorithmException e) {
- //wil not happen
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- }
+ boolean verify = signature.verify(sig);
+ return verify;
- }
-
- public static boolean verify(PublicKey publicKey, byte[] sig, InputStream is)
- throws InvalidKeyException, SignatureException, IOException {
- try {
- Signature signature = Signature.getInstance("SHA1withRSA");
- signature.initVerify(publicKey);
- byte[] buf = new byte[1024];
- while (true) {
- int sz = is.read(buf);
- if (sz == -1) break;
- signature.update(buf, 0, sz);
- }
- try {
- return signature.verify(sig);
- } catch (SignatureException e) {
- return false;
- }
} catch (NoSuchAlgorithmException e) {
//will not happen
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ } finally {
+ //Signature.update resets the position. set it back to old
+ data.position(oldPos);
}
-
+ return false;
}
-
private static byte[][] evpBytesTokey(int key_len, int iv_len, MessageDigest md,
byte[] salt, byte[] data, int count) {
byte[][] both = new byte[2][];
diff --git a/solr/core/src/test-files/cryptokeys/priv_key512.pem b/solr/core/src/test-files/cryptokeys/priv_key512.pem
deleted file mode 100644
index 53c032c..0000000
--- a/solr/core/src/test-files/cryptokeys/priv_key512.pem
+++ /dev/null
@@ -1,9 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIBOQIBAAJBAMgmSVfl+E2Nj/oKAP2TceWh17pk6Ugsw5A5nLH+OeMB/WeWJZg/
-NEDda8SXfQDEVRmw5P+2IZypPASzfCrc6yECAwEAAQJAbZFwEztky+fUSrhRRIAE
-GQaZV4PIpWdEA99WJaabv+YsWN5UUd7y+Evu50mhH3RQIxQd+R6SYs1ke9OlHlV2
-cQIhAP8367gybVEu2A+Cg1fE9vbHfnHrurpDQrh9r0ZKooTtAiEAyMMxvlHlSh6Q
-2cUTSxuyUEaQfN+W4efehgfIWBVlzIUCIEHBMZ0qeNnCvO36DUbuu0ZHjb9iIaDd
-tXH9B8yPbCHdAiAaV3o0ZZx3MDGDUVdpuHWaENguekva0kihP24rGIul3QIgNqZS
-EzA2aoQdNPl5oDfkhqAGjs5pb7qLgtmXJvVhi/Q=
------END RSA PRIVATE KEY-----
diff --git a/solr/core/src/test-files/cryptokeys/pub_key512.der b/solr/core/src/test-files/cryptokeys/pub_key512.der
deleted file mode 100644
index 4c926dd..0000000
Binary files a/solr/core/src/test-files/cryptokeys/pub_key512.der and /dev/null differ
diff --git a/solr/core/src/test-files/runtimecode/runtimelibs_v3.jar.bin b/solr/core/src/test-files/runtimecode/runtimelibs_v3.jar.bin
deleted file mode 100644
index 6b5bad6..0000000
Binary files a/solr/core/src/test-files/runtimecode/runtimelibs_v3.jar.bin and /dev/null differ
diff --git a/solr/core/src/test-files/runtimecode/sig.txt b/solr/core/src/test-files/runtimecode/sig.txt
deleted file mode 100644
index 4ef8e9c..0000000
--- a/solr/core/src/test-files/runtimecode/sig.txt
+++ /dev/null
@@ -1,105 +0,0 @@
-================priv_key2048.pem===================
-
-openssl dgst -sha1 -sign ../cryptokeys/priv_key2048.pem runtimelibs.jar.bin | openssl enc -base64
-
-NaTm3+i99/ZhS8YRsLc3NLz2Y6VuwEbu7DihY8GAWwWIGm+jpXgn1JiuaenfxFCc
-fNKCC9WgZmEgbTZTzmV/OZMVn90u642YJbF3vTnzelW1pHB43ZRAJ1iesH0anM37
-w03n3es+vFWQtuxc+2Go888fJoMkUX2C6Zk6Jn116KE45DWjeyPM4mp3vvGzwGvd
-RxP5K9Q3suA+iuI/ULXM7m9mV4ruvs/MZvL+ELm5Jnmk1bBtixVJhQwJP2z++8tQ
-KJghhyBxPIC/2fkAHobQpkhZrXu56JjP+v33ul3Ku4bbvfVMY/LVwCAEnxlvhk+C
-6uRCKCeFMrzQ/k5inasXLw==
-
-
-openssl dgst -sha1 -sign ../cryptokeys/priv_key2048.pem runtimelibs_v2.jar.bin | openssl enc -base64
-
-jsPpNMs74ogRbx9M4n/OH3j3s85KOq9dOtgGJkUf6O5D8T9d9zU2lDwxnTYjQCaW
-cRTLGH3Z8vpc0wyT3g4aXepgLUTSnrepbPffSFhQtFrCNxurPOLzbp6ERhwjZ0RL
-GvZrlbbjR2SxqZ3BpHiGxslj0tPCkdevNCEy1glLhl8RWG5xsLCrRL1mrEtLg97A
-53oCCrfGAHLEvW+olGeB1r7jqUaSrbfAUfDMSIvZfOIV+xdlvabkNiuzvsAc+B6Q
-pXWm+Em2f5TO/bkOh2m/UInGXcNHCa0oqRMGKP1H252Cv9eXm/d0h3Dqxv+f80Gz
-LfyA6/OKQ9FfskY4pltCsQ==
-
-openssl dgst -sha1 -sign ../cryptokeys/priv_key2048.pem runtimelibs_v3.jar.bin | openssl enc -base64
-
-
-YxFr6SpYrDwG85miDfRWHTjU9UltjtIWQZEhcV55C2rczRUVowCYBxmsDv5mAM8j
-0CTv854xpI1DtBT86wpoTdbF95LQuP9FJId4TS1j8bZ9cxHP5Cqyz1uBHFfUUNUr
-npzTHQkVTp02O9NAjh3c2W41bL4U7j6jQ32+4CW2M+x00TDG0y0H75rQDR8zbLt3
-1oWCz+sBOdZ3rGKJgAvdoGm/wVCTmsabZN+xoz4JaDeBXF16O9Uk9SSq4G0dz5YX
-FuLxHK7ciB5t0+q6pXlF/tdlDqF76Abze0R3d2/0MhXBzyNp3UxJmj6DiprgysfB
-0TbQtJG0XGfdSmx0VChvcA==
-
-YxFr6SpYrDwG85miDfRWHTjU9UltjtIWQZEhcV55C2rczRUVowCYBxmsDv5mAM8j0CTv854xpI1DtBT86wpoTdbF95LQuP9FJId4TS1j8bZ9cxHP5Cqyz1uBHFfUUNUrnpzTHQkVTp02O9NAjh3c2W41bL4U7j6jQ32+4CW2M+x00TDG0y0H75rQDR8zbLt31oWCz+sBOdZ3rGKJgAvdoGm/wVCTmsabZN+xoz4JaDeBXF16O9Uk9SSq4G0dz5YXFuLxHK7ciB5t0+q6pXlF/tdlDqF76Abze0R3d2/0MhXBzyNp3UxJmj6DiprgysfB0TbQtJG0XGfdSmx0VChvcA==
-
-=====================priv_key512.pem=====================
-openssl dgst -sha1 -sign ../cryptokeys/priv_key512.pem runtimelibs.jar.bin | openssl enc -base64
-
-L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1
-f/U3bOlMPINlSOM6LK3JpQ==
-
-openssl dgst -sha1 -sign ../cryptokeys/priv_key512.pem runtimelibs_v2.jar.bin | openssl enc -base64
-
-j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJu
-XbHkVLj638yZ0Lp1ssnoYA==
-
-openssl dgst -sha1 -sign ../cryptokeys/priv_key512.pem runtimelibs_v3.jar.bin | openssl enc -base64
-
-a400n4T7FT+2gM0SC6+MfSOExjud8MkhTSFylhvwNjtWwUgKdPFn434Wv7Qc4QEq
-DVLhQoL3WqYtQmLPti0G4Q==
-
-openssl dgst -sha1 -sign ../cryptokeys/priv_key512.pem cache.jar.bin | openssl enc -base64
-
-A2CDnReirpII005KRN1C3pvt4NM4kItsagQPNaa3ljj/5R3LKVgiPuNvqBsffU8n
-81LOAfr5VMyGFcb4QMHpyg==
-
-openssl dgst -sha1 -sign ../cryptokeys/priv_key512.pem cache_v2.jar.bin | openssl enc -base64
-
-SOrekHt+uup+z2z+nZU5indk2huRRfmbM+W+vQ0variHrcZEG9EXt5LuPFl8Ki9A
-hr6klMHdVP8nj4wuQhu/Hg==
-
-====================sha512====================
-
-openssl dgst -sha512 runtimelibs.jar.bin
-
-d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420
-
-
-openssl dgst -sha512 runtimelibs_v2.jar.bin
-
-bc5ce45ad281b6a08fb7e529b1eb475040076834816570902acb6ebdd809410e31006efdeaa7f78a6c35574f3504963f5f7e4d92247d0eb4db3fc9abdda5d417
-
-openssl dgst -sha512 runtimelibs_v3.jar.bin
-
-60ec88c2a2e9b409f7afc309273383810a0d07a078b482434eda9674f7e25b8adafa8a67c9913c996cbfb78a7f6ad2b9db26dbd4fe0ca4068f248d5db563f922
-
-openssl dgst -sha512 cache.jar.bin
-
-8946650ba88919cea2f81e4771c418411f61837b2a276088c2f2c86ef2d730f152ccf5975fa8a2c7035a1f00de1994a7788676d95dc7ccea6aaf28c7fff1f46b
-
-openssl dgst -sha512 cache_v2.jar.bin
-
-873337e67b90b7ff99df012b2e9093c63079c37a564643d34861a88c4cbaf0698ebb096905929d69cdbde3b4d29d55e31db24ee05c01b39c0b75a16e54eb4335
-
-=============sha256============================
-
-openssl dgst -sha256 runtimelibs.jar.bin
-
-e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc
-
-openssl dgst -sha512 runtimelibs_v2.jar.bin
-
-79298d7d5c3e60d91154efe7d72f4536eac46698edfa22ab894b85492d562ed4
-
-openssl dgst -sha256 runtimelibs_v3.jar.bin
-
-20e0bfaec71b2e93c4da9f2ed3745dda04dc3fc915b66cc0275863982e73b2a3
-
-openssl dgst -sha256 cache.jar.bin
-
-32e8b5b2a95ea306538b52017f0954aa1b0f8a8b2d0acbc498fd0e66a223f7bd
-
-openssl dgst -sha256 cache_v2.jar.bin
-
-0f670f6dcc2b00f9a448a7ebd457d4ff985ab702c85cdb3608dcae9889e8d702
-
-
diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java
deleted file mode 100644
index a99028a..0000000
--- a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.filestore;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.function.Predicate;
-
-import com.google.common.collect.ImmutableSet;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.BaseHttpSolrClient.RemoteExecutionException;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.request.V2Request;
-import org.apache.solr.client.solrj.response.V2Response;
-import org.apache.solr.cloud.MiniSolrCloudCluster;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.NavigableObject;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.util.LogLevel;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.server.ByteBufferInputStream;
-
-import static org.apache.solr.common.util.Utils.JAVABINCONSUMER;
-import static org.apache.solr.core.TestDynamicLoading.getFileContent;
-
-@LogLevel("org.apache.solr.core.PackageStoreAPI=DEBUG;org.apache.solr.core.DistribPackageStore=DEBUG")
-public class TestDistribPackageStore extends SolrCloudTestCase {
-
- public void testPackageStoreManagement() throws Exception {
- MiniSolrCloudCluster cluster =
- configureCluster(4)
- .withJettyConfig(jetty -> jetty.enableV2(true))
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
- try {
-
- byte[] derFile = readFile("cryptokeys/pub_key512.der");
- cluster.getZkClient().makePath("/keys/exe", true);
- cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
-
- try {
- postFile(cluster.getSolrClient(), getFileContent("runtimecode/runtimelibs.jar.bin"),
- "/package/mypkg/v1.0/runtimelibs.jar",
- "j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA=="
- );
- fail("should have failed because of wrong signature ");
- } catch (RemoteExecutionException e) {
- assertTrue(e.getMessage().contains("Signature does not match"));
- }
-
- postFile(cluster.getSolrClient(), getFileContent("runtimecode/runtimelibs.jar.bin"),
- "/package/mypkg/v1.0/runtimelibs.jar",
- "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="
- );
-
- assertResponseValues(10,
- cluster.getSolrClient(),
- new V2Request.Builder("/node/files/package/mypkg/v1.0")
- .withMethod(SolrRequest.METHOD.GET)
- .build(),
- Utils.makeMap(
- ":files:/package/mypkg/v1.0[0]:name", "runtimelibs.jar",
- ":files:/package/mypkg/v1.0[0]:sha512", "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420",
- ":files:/package/mypkg/v1.0[0]:sig[0]", "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="
- )
- );
-
- assertResponseValues(10,
- cluster.getSolrClient(),
- new V2Request.Builder("/node/files/package/mypkg")
- .withMethod(SolrRequest.METHOD.GET)
- .build(),
- Utils.makeMap(
- ":files:/package/mypkg[0]:name", "v1.0",
- ":files:/package/mypkg[0]:dir", "true"
- )
- );
-
- class Fetcher implements Callable {
- String url;
- JettySolrRunner jetty;
- Fetcher(String s, JettySolrRunner jettySolrRunner){
- this.url = s;
- this.jetty = jettySolrRunner;
- }
- @Override
- public NavigableObject call() throws Exception {
- try (HttpSolrClient solrClient = (HttpSolrClient) jetty.newClient()) {
- return (NavigableObject) Utils.executeGET(solrClient.getHttpClient(), this.url, JAVABINCONSUMER);
- }
- }
-
- @Override
- public String toString() {
- return url;
- }
-
- }
-
- Map expected = Utils.makeMap(
- ":files:/package/mypkg/v1.0/runtimelibs.jar:name", "runtimelibs.jar",
- ":files:/package/mypkg/v1.0[0]:sha512", "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420"
-
- );
- for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
- String baseUrl = jettySolrRunner.getBaseUrl().toString().replace("/solr", "/api");
- String url = baseUrl + "/node/files/package/mypkg/v1.0/runtimelibs.jar?wt=javabin&meta=true";
-
- assertResponseValues(10, new Fetcher(url, jettySolrRunner), expected);
-
- try (HttpSolrClient solrClient = (HttpSolrClient) jettySolrRunner.newClient()) {
- ByteBuffer buf = Utils.executeGET(solrClient.getHttpClient(), baseUrl + "/node/files/package/mypkg/v1.0/runtimelibs.jar",
- Utils.newBytesConsumer(Integer.MAX_VALUE));
- assertEquals(
- "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420",
- DigestUtils.sha512Hex(new ByteBufferInputStream(buf))
- );
-
- }
-
- }
-
- postFile(cluster.getSolrClient(), getFileContent("runtimecode/runtimelibs_v2.jar.bin"),
- "/package/mypkg/v1.0/runtimelibs_v2.jar",
- null
- );
-
- expected = Utils.makeMap(
- ":files:/package/mypkg/v1.0", (Predicate<Object>) o -> {
- List l = (List) o;
- assertEquals(2, l.size());
- Set expectedKeys = ImmutableSet.of("runtimelibs_v2.jar", "runtimelibs.jar");
- for (Object file : l) {
- if(! expectedKeys.contains(Utils.getObjectByPath(file, true, "name"))) return false;
- }
-
- return true;
- }
- );
- for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
- String baseUrl = jettySolrRunner.getBaseUrl().toString().replace("/solr", "/api");
- String url = baseUrl + "/node/files/package/mypkg/v1.0?wt=javabin";
-
- assertResponseValues(10, new Fetcher(url, jettySolrRunner), expected);
-
- }
-
-
-
- } finally {
- cluster.shutdown();
- }
- }
-
- public static NavigableObject assertResponseValues(int repeats, SolrClient client, SolrRequest req, Map vals) throws Exception {
- Callable<NavigableObject> callable = () -> req.process(client);
-
- return assertResponseValues(repeats, callable,vals);
- }
-
- public static NavigableObject assertResponseValues(int repeats, Callable<NavigableObject> callable,Map vals) throws Exception {
- NavigableObject rsp = null;
-
- for (int i = 0; i < repeats; i++) {
- if (i > 0) {
- Thread.sleep(100);
- }
- try {
- rsp = callable.call();
- } catch (Exception e) {
- if (i >= repeats - 1) throw e;
- continue;
- }
- for (Object e : vals.entrySet()) {
- Map.Entry entry = (Map.Entry) e;
- String k = (String) entry.getKey();
- List<String> key = StrUtils.split(k, '/');
-
- Object val = entry.getValue();
- Predicate p = val instanceof Predicate ? (Predicate) val : o -> {
- String v = o == null ? null : String.valueOf(o);
- return Objects.equals(val, o);
- };
- boolean isPass = p.test(rsp._get(key, null));
- if (isPass) return rsp;
- else if (i >= repeats - 1) {
- fail("req: " + callable.toString() +" . attempt: " + i + " Mismatch for value : '" + key + "' in response , " + Utils.toJSONString(rsp));
- }
-
- }
-
- }
- return rsp;
- }
-
-
-
- private void postFile(SolrClient client, ByteBuffer buffer, String name, String sig)
- throws SolrServerException, IOException {
- String resource = "/cluster/files" + name;
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.add("sig", sig);
- V2Response rsp = new V2Request.Builder(resource)
- .withMethod(SolrRequest.METHOD.PUT)
- .withPayload(buffer)
- .forceV2(true)
- .withMimeType("application/octet-stream")
- .withParams(params)
- .build()
- .process(client);
- assertEquals(name, rsp.getResponse().get(CommonParams.FILE));
- }
-
- public static byte[] readFile(String fname) throws IOException {
- byte[] buf = null;
- try (FileInputStream fis = new FileInputStream(getFile(fname))) {
- buf = new byte[fis.available()];
- fis.read(buf);
- }
- return buf;
- }
-}
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/SolrResponse.java b/solr/solrj/src/java/org/apache/solr/client/solrj/SolrResponse.java
index 73eb863..ef52eb2 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/SolrResponse.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/SolrResponse.java
@@ -18,12 +18,10 @@ package org.apache.solr.client.solrj;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
-import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
-import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.NamedList;
@@ -34,7 +32,7 @@ import org.apache.solr.common.util.NamedList;
*
* @since solr 1.3
*/
-public abstract class SolrResponse implements Serializable, MapWriter {
+public abstract class SolrResponse implements Serializable {
/** Elapsed time in milliseconds for the request as seen from the client. */
public abstract long getElapsedTime();
@@ -45,11 +43,6 @@ public abstract class SolrResponse implements Serializable, MapWriter {
public abstract NamedList<Object> getResponse();
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- getResponse().writeMap(ew);
- }
-
public Exception getException() {
NamedList exp = (NamedList) getResponse().get("exception");
if (exp == null) {
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/V2Request.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/V2Request.java
index 5334edd..4236177 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/V2Request.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/V2Request.java
@@ -18,15 +18,11 @@
package org.apache.solr.client.solrj.request;
import java.io.IOException;
-import java.io.InputStream;
import java.io.OutputStream;
-import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.commons.io.IOUtils;
-import org.apache.solr.client.solrj.ResponseParser;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.response.V2Response;
@@ -46,10 +42,8 @@ public class V2Request extends SolrRequest<V2Response> implements MapWriter {
private SolrParams solrParams;
public final boolean useBinary;
private String collection;
- private String mimeType;
private boolean forceV2 = false;
private boolean isPerCollectionRequest = false;
- private ResponseParser parser;
private V2Request(METHOD m, String resource, boolean useBinary) {
super(m, resource);
@@ -62,7 +56,7 @@ public class V2Request extends SolrRequest<V2Response> implements MapWriter {
}
- public boolean isForceV2() {
+ public boolean isForceV2(){
return forceV2;
}
@@ -81,15 +75,6 @@ public class V2Request extends SolrRequest<V2Response> implements MapWriter {
return new RequestWriter.ContentWriter() {
@Override
public void write(OutputStream os) throws IOException {
- if (payload instanceof ByteBuffer) {
- ByteBuffer b = (ByteBuffer) payload;
- os.write(b.array(), b.arrayOffset(), b.limit());
- return;
- }
- if (payload instanceof InputStream) {
- IOUtils.copy((InputStream) payload, os);
- return;
- }
if (useBinary) {
new JavaBinCodec().marshal(payload, os);
} else {
@@ -99,7 +84,6 @@ public class V2Request extends SolrRequest<V2Response> implements MapWriter {
@Override
public String getContentType() {
- if (mimeType != null) return mimeType;
return useBinary ? JAVABIN_MIME : JSON_MIME;
}
};
@@ -127,12 +111,6 @@ public class V2Request extends SolrRequest<V2Response> implements MapWriter {
ew.putIfNotNull("command", payload);
}
- @Override
- public ResponseParser getResponseParser() {
- if (parser != null) return parser;
- return super.getResponseParser();
- }
-
public static class Builder {
private String resource;
private METHOD method = METHOD.GET;
@@ -141,8 +119,6 @@ public class V2Request extends SolrRequest<V2Response> implements MapWriter {
private boolean useBinary = false;
private boolean forceV2EndPoint = false;
- private ResponseParser parser;
- private String mimeType;
/**
* Create a Builder object based on the provided resource.
@@ -197,24 +173,11 @@ public class V2Request extends SolrRequest<V2Response> implements MapWriter {
return this;
}
- public Builder withResponseParser(ResponseParser parser) {
- this.parser = parser;
- return this;
- }
-
- public Builder withMimeType(String mimeType) {
- this.mimeType = mimeType;
- return this;
-
- }
-
public V2Request build() {
V2Request v2Request = new V2Request(method, resource, useBinary);
v2Request.solrParams = params;
v2Request.payload = payload;
v2Request.forceV2 = forceV2EndPoint;
- v2Request.mimeType = mimeType;
- v2Request.parser = parser;
return v2Request;
}
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
index 6eda49b..39a0242 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
@@ -296,8 +296,5 @@ public interface CommonParams {
String JAVABIN_MIME = "application/javabin";
- String FILE = "file";
- String FILES = "files";
-
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java b/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java
index d153657..c0b19f5 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java
@@ -30,8 +30,8 @@ import org.apache.solr.common.SolrException;
*
*/
public class StrUtils {
- public static final char[] HEX_DIGITS = {'0', '1', '2', '3', '4', '5', '6',
- '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
+ public static final char[] HEX_DIGITS = { '0', '1', '2', '3', '4', '5', '6',
+ '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };
public static List<String> splitSmart(String s, char separator) {
ArrayList<String> lst = new ArrayList<>(4);
@@ -40,19 +40,9 @@ public class StrUtils {
}
- static final String DELIM_CHARS = "/:;.,%#";
-
- public static List<String> split(String s, char sep) {
- if (DELIM_CHARS.indexOf(s.charAt(0)) > -1) {
- sep = s.charAt(0);
- }
- return splitSmart(s, sep, true);
-
- }
-
public static List<String> splitSmart(String s, char separator, boolean trimEmpty) {
List<String> l = splitSmart(s, separator);
- if (trimEmpty) {
+ if(trimEmpty){
if (l.size() > 0 && l.get(0).isEmpty()) l.remove(0);
}
return l;
@@ -64,88 +54,77 @@ public class StrUtils {
* outside strings.
*/
public static void splitSmart(String s, char separator, List<String> lst) {
- int pos = 0, start = 0, end = s.length();
- char inString = 0;
- char ch = 0;
+ int pos=0, start=0, end=s.length();
+ char inString=0;
+ char ch=0;
while (pos < end) {
- char prevChar = ch;
+ char prevChar=ch;
ch = s.charAt(pos++);
- if (ch == '\\') { // skip escaped chars
+ if (ch=='\\') { // skip escaped chars
pos++;
- } else if (inString != 0 && ch == inString) {
- inString = 0;
- } else if (ch == '\'' || ch == '"') {
+ } else if (inString != 0 && ch==inString) {
+ inString=0;
+ } else if (ch=='\'' || ch=='"') {
// If char is directly preceeded by a number or letter
// then don't treat it as the start of a string.
// Examples: 50" TV, or can't
if (!Character.isLetterOrDigit(prevChar)) {
- inString = ch;
+ inString=ch;
}
- } else if (ch == separator && inString == 0) {
- lst.add(s.substring(start, pos - 1));
- start = pos;
+ } else if (ch==separator && inString==0) {
+ lst.add(s.substring(start,pos-1));
+ start=pos;
}
}
if (start < end) {
- lst.add(s.substring(start, end));
+ lst.add(s.substring(start,end));
}
/***
- if (SolrCore.log.isLoggable(Level.FINEST)) {
- SolrCore.log.trace("splitCommand=" + lst);
- }
- ***/
+ if (SolrCore.log.isLoggable(Level.FINEST)) {
+ SolrCore.log.trace("splitCommand=" + lst);
+ }
+ ***/
}
- /**
- * Splits a backslash escaped string on the separator.
+ /** Splits a backslash escaped string on the separator.
* <p>
* Current backslash escaping supported:
* <br> \n \t \r \b \f are escaped the same as a Java String
* <br> Other characters following a backslash are produced verbatim (\c => c)
*
- * @param s the string to split
+ * @param s the string to split
* @param separator the separator to split on
- * @param decode decode backslash escaping
+ * @param decode decode backslash escaping
* @return not null
*/
public static List<String> splitSmart(String s, String separator, boolean decode) {
ArrayList<String> lst = new ArrayList<>(2);
StringBuilder sb = new StringBuilder();
- int pos = 0, end = s.length();
+ int pos=0, end=s.length();
while (pos < end) {
- if (s.startsWith(separator, pos)) {
+ if (s.startsWith(separator,pos)) {
if (sb.length() > 0) {
lst.add(sb.toString());
- sb = new StringBuilder();
+ sb=new StringBuilder();
}
- pos += separator.length();
+ pos+=separator.length();
continue;
}
char ch = s.charAt(pos++);
- if (ch == '\\') {
+ if (ch=='\\') {
if (!decode) sb.append(ch);
- if (pos >= end) break; // ERROR, or let it go?
+ if (pos>=end) break; // ERROR, or let it go?
ch = s.charAt(pos++);
if (decode) {
- switch (ch) {
- case 'n':
- ch = '\n';
- break;
- case 't':
- ch = '\t';
- break;
- case 'r':
- ch = '\r';
- break;
- case 'b':
- ch = '\b';
- break;
- case 'f':
- ch = '\f';
- break;
+ switch(ch) {
+ case 'n' : ch='\n'; break;
+ case 't' : ch='\t'; break;
+ case 'r' : ch='\r'; break;
+ case 'b' : ch='\b'; break;
+ case 'f' : ch='\f'; break;
}
}
}
@@ -179,15 +158,14 @@ public class StrUtils {
return result;
}
- /**
- * Creates a backslash escaped string, joining all the items.
- *
+ /**
+ * Creates a backslash escaped string, joining all the items.
* @see #escapeTextWithSeparator
*/
public static String join(Collection<?> items, char separator) {
if (items == null) return "";
StringBuilder sb = new StringBuilder(items.size() << 3);
- boolean first = true;
+ boolean first=true;
for (Object o : items) {
String item = String.valueOf(o);
if (first) {
@@ -201,41 +179,32 @@ public class StrUtils {
}
+
public static List<String> splitWS(String s, boolean decode) {
ArrayList<String> lst = new ArrayList<>(2);
StringBuilder sb = new StringBuilder();
- int pos = 0, end = s.length();
+ int pos=0, end=s.length();
while (pos < end) {
char ch = s.charAt(pos++);
if (Character.isWhitespace(ch)) {
if (sb.length() > 0) {
lst.add(sb.toString());
- sb = new StringBuilder();
+ sb=new StringBuilder();
}
continue;
}
- if (ch == '\\') {
+ if (ch=='\\') {
if (!decode) sb.append(ch);
- if (pos >= end) break; // ERROR, or let it go?
+ if (pos>=end) break; // ERROR, or let it go?
ch = s.charAt(pos++);
if (decode) {
- switch (ch) {
- case 'n':
- ch = '\n';
- break;
- case 't':
- ch = '\t';
- break;
- case 'r':
- ch = '\r';
- break;
- case 'b':
- ch = '\b';
- break;
- case 'f':
- ch = '\f';
- break;
+ switch(ch) {
+ case 'n' : ch='\n'; break;
+ case 't' : ch='\t'; break;
+ case 'r' : ch='\r'; break;
+ case 'b' : ch='\b'; break;
+ case 'f' : ch='\f'; break;
}
}
}
@@ -259,48 +228,46 @@ public class StrUtils {
}
- /**
- * Return if a string starts with '1', 't', or 'T'
- * and return false otherwise.
+
+ /** Return if a string starts with '1', 't', or 'T'
+ * and return false otherwise.
*/
public static boolean parseBoolean(String s) {
- char ch = s.length() > 0 ? s.charAt(0) : 0;
- return (ch == '1' || ch == 't' || ch == 'T');
+ char ch = s.length()>0 ? s.charAt(0) : 0;
+ return (ch=='1' || ch=='t' || ch=='T');
}
-
- /**
- * how to transform a String into a boolean... more flexible than
+
+ /** how to transform a String into a boolean... more flexible than
* Boolean.parseBoolean() to enable easier integration with html forms.
*/
public static boolean parseBool(String s) {
- if (s != null) {
- if (s.startsWith("true") || s.startsWith("on") || s.startsWith("yes")) {
+ if( s != null ) {
+ if( s.startsWith("true") || s.startsWith("on") || s.startsWith("yes") ) {
return true;
}
- if (s.startsWith("false") || s.startsWith("off") || s.equals("no")) {
+ if( s.startsWith("false") || s.startsWith("off") || s.equals("no") ) {
return false;
}
}
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "invalid boolean value: " + s);
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "invalid boolean value: "+s );
}
/**
* {@link NullPointerException} and {@link SolrException} free version of {@link #parseBool(String)}
- *
* @return parsed boolean value (or def, if s is null or invalid)
*/
public static boolean parseBool(String s, boolean def) {
- if (s != null) {
- if (s.startsWith("true") || s.startsWith("on") || s.startsWith("yes")) {
+ if( s != null ) {
+ if( s.startsWith("true") || s.startsWith("on") || s.startsWith("yes") ) {
return true;
}
- if (s.startsWith("false") || s.startsWith("off") || s.equals("no")) {
+ if( s.startsWith("false") || s.startsWith("off") || s.equals("no") ) {
return false;
}
}
return def;
}
-
+
/**
* URLEncodes a value, replacing only enough chars so that
* the URL may be unambiguously pasted back into a browser.
@@ -309,7 +276,7 @@ public class StrUtils {
* &,=,%,+,space are encoded.
*/
public static void partialURLEncodeVal(Appendable dest, String val) throws IOException {
- for (int i = 0; i < val.length(); i++) {
+ for (int i=0; i<val.length(); i++) {
char ch = val.charAt(i);
if (ch < 32) {
dest.append('%');
@@ -317,60 +284,46 @@ public class StrUtils {
dest.append(Integer.toHexString(ch));
} else {
switch (ch) {
- case ' ':
- dest.append('+');
- break;
- case '&':
- dest.append("%26");
- break;
- case '%':
- dest.append("%25");
- break;
- case '=':
- dest.append("%3D");
- break;
- case '+':
- dest.append("%2B");
- break;
- default:
- dest.append(ch);
- break;
+ case ' ': dest.append('+'); break;
+ case '&': dest.append("%26"); break;
+ case '%': dest.append("%25"); break;
+ case '=': dest.append("%3D"); break;
+ case '+': dest.append("%2B"); break;
+ default : dest.append(ch); break;
}
}
}
}
- /**
+ /**
* Creates a new copy of the string with the separator backslash escaped.
- *
* @see #join
*/
public static String escapeTextWithSeparator(String item, char separator) {
StringBuilder sb = new StringBuilder(item.length() * 2);
appendEscapedTextToBuilder(sb, item, separator);
return sb.toString();
- }
+ }
/**
- * writes chars from item to out, backslash escaping as needed based on separator --
+ * writes chars from item to out, backslash escaping as needed based on separator --
* but does not append the separator itself
*/
- public static void appendEscapedTextToBuilder(StringBuilder out,
- String item,
- char separator) {
+ public static void appendEscapedTextToBuilder(StringBuilder out,
+ String item,
+ char separator) {
for (int i = 0; i < item.length(); i++) {
char ch = item.charAt(i);
- if (ch == '\\' || ch == separator) {
+ if (ch == '\\' || ch == separator) {
out.append('\\');
}
out.append(ch);
}
}
- /**
- * Format using MesssageFormat but with the ROOT locale
+ /**Format using MesssageFormat but with the ROOT locale
*/
- public static String formatString(String pattern, Object... args) {
+ public static String formatString(String pattern, Object... args) {
return new MessageFormat(pattern, Locale.ROOT).format(args);
}
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
index db6ef37..4a8d987 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
@@ -28,7 +28,6 @@ import java.io.Writer;
import java.lang.invoke.MethodHandles;
import java.net.URL;
import java.net.URLDecoder;
-import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.AbstractMap;
@@ -54,9 +53,6 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
import org.apache.solr.client.solrj.cloud.DistribStateManager;
import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData;
@@ -95,7 +91,7 @@ public class Utils {
public static final Function NEW_SYNCHRONIZED_ARRAYLIST_FUN = o -> Collections.synchronizedList(new ArrayList<>());
public static final Function NEW_HASHSET_FUN = o -> new HashSet<>();
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
+
public static Map getDeepCopy(Map map, int maxDepth) {
return getDeepCopy(map, maxDepth, true, false);
}
@@ -105,17 +101,17 @@ public class Utils {
}
public static Map getDeepCopy(Map map, int maxDepth, boolean mutable, boolean sorted) {
- if (map == null) return null;
+ if(map == null) return null;
if (maxDepth < 1) return map;
Map copy;
if (sorted) {
copy = new TreeMap();
} else {
- copy = map instanceof LinkedHashMap ? new LinkedHashMap(map.size()) : new HashMap(map.size());
+ copy = map instanceof LinkedHashMap? new LinkedHashMap(map.size()): new HashMap(map.size());
}
for (Object o : map.entrySet()) {
Map.Entry e = (Map.Entry) o;
- copy.put(e.getKey(), makeDeepCopy(e.getValue(), maxDepth, mutable, sorted));
+ copy.put(e.getKey(), makeDeepCopy(e.getValue(),maxDepth, mutable, sorted));
}
return mutable ? copy : Collections.unmodifiableMap(copy);
}
@@ -155,7 +151,7 @@ public class Utils {
} else if (v instanceof IteratorWriter && maxDepth > 1) {
v = ((IteratorWriter) v).toList(new ArrayList<>());
if (sorted) {
- Collections.sort((List) v);
+ Collections.sort((List)v);
}
}
@@ -170,8 +166,8 @@ public class Utils {
public static InputStream toJavabin(Object o) throws IOException {
try (final JavaBinCodec jbc = new JavaBinCodec()) {
BinaryRequestWriter.BAOS baos = new BinaryRequestWriter.BAOS();
- jbc.marshal(o, baos);
- return new ByteBufferInputStream(ByteBuffer.wrap(baos.getbuf(), 0, baos.size()));
+ jbc.marshal(o,baos);
+ return new ByteBufferInputStream(ByteBuffer.wrap(baos.getbuf(),0,baos.size()));
}
}
@@ -182,10 +178,10 @@ public class Utils {
public static Collection getDeepCopy(Collection c, int maxDepth, boolean mutable, boolean sorted) {
if (c == null || maxDepth < 1) return c;
Collection result = c instanceof Set ?
- (sorted ? new TreeSet() : new HashSet()) : new ArrayList();
+ ( sorted? new TreeSet() : new HashSet()) : new ArrayList();
for (Object o : c) result.add(makeDeepCopy(o, maxDepth, mutable, sorted));
if (sorted && (result instanceof List)) {
- Collections.sort((List) result);
+ Collections.sort((List)result);
}
return mutable ? result : result instanceof Set ? unmodifiableSet((Set) result) : unmodifiableList((List) result);
}
@@ -212,7 +208,7 @@ public class Utils {
@Override
public void handleUnknownClass(Object o) {
if (o instanceof MapWriter) {
- Map m = ((MapWriter) o).toMap(new LinkedHashMap<>());
+ Map m = ((MapWriter)o).toMap(new LinkedHashMap<>());
write(m);
} else {
super.handleUnknownClass(o);
@@ -221,13 +217,13 @@ public class Utils {
}
public static byte[] toJSON(Object o) {
- if (o == null) return new byte[0];
+ if(o == null) return new byte[0];
CharArr out = new CharArr();
if (!(o instanceof List) && !(o instanceof Map)) {
- if (o instanceof MapWriter) {
- o = ((MapWriter) o).toMap(new LinkedHashMap<>());
- } else if (o instanceof IteratorWriter) {
- o = ((IteratorWriter) o).toList(new ArrayList<>());
+ if (o instanceof MapWriter) {
+ o = ((MapWriter)o).toMap(new LinkedHashMap<>());
+ } else if(o instanceof IteratorWriter){
+ o = ((IteratorWriter)o).toList(new ArrayList<>());
}
}
new MapWriterJSONWriter(out, 2).write(o); // indentation by default
@@ -278,11 +274,10 @@ public class Utils {
return propMap;
}
- public static Object fromJSON(InputStream is) {
+ public static Object fromJSON(InputStream is){
return fromJSON(new InputStreamReader(is, UTF_8));
}
-
- public static Object fromJSON(Reader is) {
+ public static Object fromJSON(Reader is){
try {
return STANDARDOBJBUILDER.apply(getJSONParser(is)).getVal();
} catch (IOException e) {
@@ -300,7 +295,7 @@ public class Utils {
};
public static final Function<JSONParser, ObjectBuilder> MAPWRITEROBJBUILDER = jsonParser -> {
try {
- return new ObjectBuilder(jsonParser) {
+ return new ObjectBuilder(jsonParser){
@Override
public Object newObject() {
return new LinkedHashMapWriter();
@@ -313,7 +308,7 @@ public class Utils {
public static final Function<JSONParser, ObjectBuilder> MAPOBJBUILDER = jsonParser -> {
try {
- return new ObjectBuilder(jsonParser) {
+ return new ObjectBuilder(jsonParser){
@Override
public Object newObject() {
return new HashMap();
@@ -341,11 +336,10 @@ public class Utils {
return fromJSON(stream);
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Resource error: " + e.getMessage(), e);
+ "Resource error: " + e.getMessage(), e);
}
}
-
- public static JSONParser getJSONParser(Reader reader) {
+ public static JSONParser getJSONParser(Reader reader){
JSONParser parser = new JSONParser(reader);
parser.setFlags(parser.getFlags() |
JSONParser.ALLOW_MISSING_COLON_COMMA_BEFORE_OBJECT |
@@ -353,11 +347,11 @@ public class Utils {
return parser;
}
- public static Object fromJSONString(String json) {
+ public static Object fromJSONString(String json) {
try {
return STANDARDOBJBUILDER.apply(getJSONParser(new StringReader(json))).getVal();
} catch (Exception e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Parse error : " + json, e);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Parse error : "+ json, e );
}
}
@@ -430,8 +424,8 @@ public class Utils {
public static Object getObjectByPath(Object root, boolean onlyPrimitive, List<String> hierarchy) {
- if (root == null) return null;
- if (!isMapLike(root)) return null;
+ if(root == null) return null;
+ if(!isMapLike(root)) return null;
Object obj = root;
for (int i = 0; i < hierarchy.size(); i++) {
int idx = -1;
@@ -524,7 +518,6 @@ public class Utils {
try {
((MapWriter) obj).writeMap(new MapWriter.EntryWriter() {
int count = -1;
-
@Override
public MapWriter.EntryWriter put(CharSequence k, Object v) {
if (result[0] != null) return this;
@@ -540,14 +533,15 @@ public class Utils {
throw new RuntimeException(e);
}
return result[0];
- } else if (obj instanceof Map) return ((Map) obj).get(key);
+ }
+ else if (obj instanceof Map) return ((Map) obj).get(key);
else throw new RuntimeException("must be a NamedList or Map");
}
/**
* If the passed entity has content, make sure it is fully
* read and closed.
- *
+ *
* @param entity to consume or null
*/
public static void consumeFully(HttpEntity entity) {
@@ -568,14 +562,13 @@ public class Utils {
/**
* Make sure the InputStream is fully read.
- *
+ *
* @param is to read
* @throws IOException on problem with IO
*/
private static void readFully(InputStream is) throws IOException {
is.skip(is.available());
- while (is.read() != -1) {
- }
+ while (is.read() != -1) {}
}
public static Map<String, Object> getJson(DistribStateManager distribStateManager, String path) throws InterruptedException, IOException, KeeperException {
@@ -592,8 +585,8 @@ public class Utils {
/**
* Assumes data in ZooKeeper is a JSON string, deserializes it and returns as a Map
*
- * @param zkClient the zookeeper client
- * @param path the path to the znode being read
+ * @param zkClient the zookeeper client
+ * @param path the path to the znode being read
* @param retryOnConnLoss whether to retry the operation automatically on connection loss, see {@link org.apache.solr.common.cloud.ZkCmdExecutor#retryOperation(ZkOperation)}
* @return a Map if the node exists and contains valid JSON or an empty map if znode does not exist or has a null data
*/
@@ -637,12 +630,11 @@ public class Utils {
}
}
- /**
- * Applies one json over other. The 'input' is applied over the sink
+ /**Applies one json over other. The 'input' is applied over the sink
* The values in input isapplied over the values in 'sink' . If a value is 'null'
* that value is removed from sink
*
- * @param sink the original json object to start with. Ensure that this Map is mutable
+ * @param sink the original json object to start with. Ensure that this Map is mutable
* @param input the json with new values
* @return whether there was any change made to sink or not.
*/
@@ -685,9 +677,9 @@ public class Utils {
if (_offset < 0) {
throw new IllegalArgumentException("nodeName does not contain expected '_' separator: " + nodeName);
}
- final String hostAndPort = nodeName.substring(0, _offset);
+ final String hostAndPort = nodeName.substring(0,_offset);
try {
- final String path = URLDecoder.decode(nodeName.substring(1 + _offset), "UTF-8");
+ final String path = URLDecoder.decode(nodeName.substring(1+_offset), "UTF-8");
return urlScheme + "://" + hostAndPort + (path.isEmpty() ? "" : ("/" + path));
} catch (UnsupportedEncodingException e) {
throw new IllegalStateException("JVM Does not seem to support UTF-8", e);
@@ -721,68 +713,4 @@ public class Utils {
return def;
}
- public interface InputStreamConsumer<T> {
-
- T accept(InputStream is) throws IOException;
-
- }
-
- public static final InputStreamConsumer<?> JAVABINCONSUMER = is -> new JavaBinCodec().unmarshal(is);
- public static final InputStreamConsumer<?> JSONCONSUMER = is -> Utils.fromJSON(is);
-
- public static InputStreamConsumer<ByteBuffer> newBytesConsumer(int maxSize) {
- return is -> {
- try (BinaryRequestWriter.BAOS bos = new BinaryRequestWriter.BAOS()) {
- long sz = 0;
- int next = is.read();
- while (next > -1) {
- if (++sz > maxSize) throw new BufferOverflowException();
- bos.write(next);
- next = is.read();
- }
- bos.flush();
- return ByteBuffer.wrap(bos.getbuf(), 0, bos.size());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- };
-
- }
-
-
- public static <T> T executeGET(HttpClient client, String url, InputStreamConsumer<T> consumer) throws SolrException {
- T result = null;
- HttpGet httpGet = new HttpGet(url);
- HttpResponse rsp = null;
- try {
- rsp = client.execute(httpGet);
- } catch (IOException e) {
- log.error("Error in request to url : " + url, e);
- throw new SolrException(SolrException.ErrorCode.UNKNOWN, "error sending request");
- }
- int statusCode = rsp.getStatusLine().getStatusCode();
- if (statusCode != 200) {
- try {
- log.error("Failed a request to: {}, status: {}, body: {}", url, rsp.getStatusLine(), EntityUtils.toString(rsp.getEntity(), StandardCharsets.UTF_8));
- } catch (IOException e) {
- log.error("could not print error", e);
- }
- throw new SolrException(SolrException.ErrorCode.getErrorCode(statusCode), "Unknown error");
- }
- HttpEntity entity = rsp.getEntity();
- try {
- InputStream is = entity.getContent();
- if (consumer != null) {
-
- result = consumer.accept(is);
- }
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.UNKNOWN, e);
- } finally {
- Utils.consumeFully(entity);
- }
- return result;
- }
-
-
}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
index 4ce7a5e..cb66ae9 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
@@ -206,8 +206,8 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
*
* @throws Exception if an error occurs on startup
*/
- public MiniSolrCloudCluster configure() throws Exception {
- return cluster = build();
+ public void configure() throws Exception {
+ cluster = build();
}
/**