You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/10/31 05:07:30 UTC
[44/50] [abbrv] lucene-solr:jira/http2_benchmark: SOLR-12861: Add
Solr factory for new ByteBuffersDirectory
SOLR-12861: Add Solr factory for new ByteBuffersDirectory
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d362439e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d362439e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d362439e
Branch: refs/heads/jira/http2_benchmark
Commit: d362439e277c63a0e3be539179d3b18cf96df617
Parents: 5fc4d51
Author: Steve Rowe <sa...@apache.org>
Authored: Mon Oct 29 08:23:14 2018 -0400
Committer: Steve Rowe <sa...@apache.org>
Committed: Mon Oct 29 08:23:14 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 5 ++
.../solr/core/ByteBuffersDirectoryFactory.java | 46 +++++++++++++
.../core/ByteBuffersDirectoryFactoryTest.java | 72 ++++++++++++++++++++
.../solr/core/CachingDirectoryFactoryTest.java | 7 +-
.../apache/solr/core/DirectoryFactoryTest.java | 45 +++++++-----
5 files changed, 155 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d362439e/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 2c79aaa..ef91ba3 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -129,6 +129,9 @@ Upgrade Notes
be set to -1 during collection creation to fall back to the old behavior of unlimited maxShardsPerNode when using
autoscaling policy.
+* SOLR-12861: Added a Solr factory for ByteBuffersDirectory, which will replace deprecated RAMDirectory in
+ a future version of Solr.
+
New Features
----------------------
@@ -190,6 +193,8 @@ Other Changes
* SOLR-12793: Move TestCloudJSONFacetJoinDomain amd TestCloudJSONFacetSKG to the facet test package (Varun Thacker)
+* SOLR-12861: Add Solr factory for ByteBuffersDirectory.
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d362439e/solr/core/src/java/org/apache/solr/core/ByteBuffersDirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/ByteBuffersDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/ByteBuffersDirectoryFactory.java
new file mode 100644
index 0000000..ba27650
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/ByteBuffersDirectoryFactory.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core;
+
+import java.io.IOException;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.LockFactory;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.SingleInstanceLockFactory;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+
+/**
+ * Factory to instantiate {@link org.apache.lucene.store.ByteBuffersDirectory}
+ */
+public class ByteBuffersDirectoryFactory extends EphemeralDirectoryFactory {
+
+ @Override
+ protected LockFactory createLockFactory(String rawLockType) throws IOException {
+ if (!(rawLockType == null || DirectoryFactory.LOCK_TYPE_SINGLE.equalsIgnoreCase(rawLockType.trim()))) {
+ throw new SolrException(ErrorCode.FORBIDDEN,
+ "ByteBuffersDirectory can only be used with the '"+DirectoryFactory.LOCK_TYPE_SINGLE+"' lock factory type.");
+ }
+ return new SingleInstanceLockFactory();
+ }
+
+ @Override
+ protected Directory create(String path, LockFactory lockFactory, DirContext dirContext) throws IOException {
+ return new ByteBuffersDirectory(lockFactory);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d362439e/solr/core/src/test/org/apache/solr/core/ByteBuffersDirectoryFactoryTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/ByteBuffersDirectoryFactoryTest.java b/solr/core/src/test/org/apache/solr/core/ByteBuffersDirectoryFactoryTest.java
new file mode 100644
index 0000000..45c7a15
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/core/ByteBuffersDirectoryFactoryTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core;
+
+import java.io.IOException;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.LockFactory;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.core.DirectoryFactory.DirContext;
+
+/**
+ * Test-case for ByteBuffersDirectoryFactory
+ */
+public class ByteBuffersDirectoryFactoryTest extends SolrTestCaseJ4 {
+
+ public void testOpenReturnsTheSameForSamePath() throws IOException {
+ final Directory directory = new ByteBuffersDirectory();
+ ByteBuffersDirectoryFactory factory = new ByteBuffersDirectoryFactory() {
+ @Override
+ protected Directory create(String path, LockFactory lockFactory, DirContext dirContext) {
+ return directory;
+ }
+ };
+ String path = "/fake/path";
+ Directory dir1 = factory.get(path, DirContext.DEFAULT, DirectoryFactory.LOCK_TYPE_SINGLE);
+ Directory dir2 = factory.get(path, DirContext.DEFAULT, DirectoryFactory.LOCK_TYPE_SINGLE);
+ assertEquals("ByteBuffersDirectoryFactory should not create new instance of ByteBuffersDirectory " +
+ "every time open() is called for the same path", dir1, dir2);
+
+ factory.release(dir1);
+ factory.release(dir2);
+ factory.close();
+ }
+
+ public void testOpenSucceedForEmptyDir() throws IOException {
+ ByteBuffersDirectoryFactory factory = new ByteBuffersDirectoryFactory();
+ Directory dir = factory.get("/fake/path", DirContext.DEFAULT, DirectoryFactory.LOCK_TYPE_SINGLE);
+ assertNotNull("ByteBuffersDirectoryFactory should create ByteBuffersDirectory even if the path doesn't lead " +
+ "to index directory on the file system", dir);
+ factory.release(dir);
+ factory.close();
+ }
+
+ public void testIndexRetrieve() throws Exception {
+ System.setProperty("solr.directoryFactory", "solr.ByteBuffersDirectoryFactory");
+ initCore("solrconfig-minimal.xml","schema-minimal.xml");
+ DirectoryFactory factory = h.getCore().getDirectoryFactory();
+ assertTrue("Found: " + factory.getClass().getName(), factory instanceof ByteBuffersDirectoryFactory);
+ for (int i = 0 ; i < 5 ; ++i) {
+ assertU(adoc("id", "" + i, "a_s", "_" + i + "_"));
+ }
+ assertU(commit());
+ assertQ(req("q", "a_s:_0_"), "//result[@numFound = '1']");
+ deleteCore();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d362439e/solr/core/src/test/org/apache/solr/core/CachingDirectoryFactoryTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/CachingDirectoryFactoryTest.java b/solr/core/src/test/org/apache/solr/core/CachingDirectoryFactoryTest.java
index 8714054..5c6bce7 100644
--- a/solr/core/src/test/org/apache/solr/core/CachingDirectoryFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/core/CachingDirectoryFactoryTest.java
@@ -49,8 +49,11 @@ public class CachingDirectoryFactoryTest extends SolrTestCaseJ4 {
@Test
public void stressTest() throws Exception {
- final CachingDirectoryFactory df = new RAMDirectoryFactory();
-
+ doStressTest(new RAMDirectoryFactory());
+ doStressTest(new ByteBuffersDirectoryFactory());
+ }
+
+ private void doStressTest(final CachingDirectoryFactory df) throws Exception {
List<Thread> threads = new ArrayList<>();
int threadCount = 11;
for (int i = 0; i < threadCount; i++) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d362439e/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java b/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java
index 869a4d2..dfe50bd 100755
--- a/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java
@@ -67,45 +67,54 @@ public class DirectoryFactoryTest extends LuceneTestCase {
}
@Test
- public void testGetDataHome() throws Exception {
+ public void testGetDataHomeRAMDirectory() throws Exception {
+ doTestGetDataHome(RAMDirectoryFactory.class);
+ }
+
+ @Test
+ public void testGetDataHomeByteBuffersDirectory() throws Exception {
+ doTestGetDataHome(ByteBuffersDirectoryFactory.class);
+ }
+
+ private void doTestGetDataHome(Class<? extends DirectoryFactory> directoryFactoryClass) throws Exception {
NodeConfig config = loadNodeConfig("/solr/solr-solrDataHome.xml");
CoreContainer cc = new CoreContainer(config);
Properties cp = cc.getContainerProperties();
- RAMDirectoryFactory rdf = new RAMDirectoryFactory();
- rdf.initCoreContainer(cc);
- rdf.init(new NamedList());
+ DirectoryFactory df = directoryFactoryClass.newInstance();
+ df.initCoreContainer(cc);
+ df.init(new NamedList());
// No solr.data.home property set. Absolute instanceDir
- assertDataHome("/tmp/inst1/data", "/tmp/inst1", rdf, cc);
+ assertDataHome("/tmp/inst1/data", "/tmp/inst1", df, cc);
// Simulate solr.data.home set in solrconfig.xml <directoryFactory> tag
NamedList args = new NamedList();
args.add("solr.data.home", "/solrdata/");
- rdf.init(args);
- assertDataHome("/solrdata/inst_dir/data", "inst_dir", rdf, cc);
+ df.init(args);
+ assertDataHome("/solrdata/inst_dir/data", "inst_dir", df, cc);
// solr.data.home set with System property, and relative path
System.setProperty("solr.data.home", "solrdata");
config = loadNodeConfig("/solr/solr-solrDataHome.xml");
cc = new CoreContainer(config);
- rdf = new RAMDirectoryFactory();
- rdf.initCoreContainer(cc);
- rdf.init(new NamedList());
- assertDataHome(solrHome.resolve("solrdata/inst_dir/data").toAbsolutePath().toString(), "inst_dir", rdf, cc);
+ df = directoryFactoryClass.newInstance();
+ df.initCoreContainer(cc);
+ df.init(new NamedList());
+ assertDataHome(solrHome.resolve("solrdata/inst_dir/data").toAbsolutePath().toString(), "inst_dir", df, cc);
// Test parsing last component of instanceDir, and using custom dataDir
- assertDataHome(solrHome.resolve("solrdata/myinst/mydata").toAbsolutePath().toString(), "/path/to/myinst", rdf, cc, "dataDir", "mydata");
+ assertDataHome(solrHome.resolve("solrdata/myinst/mydata").toAbsolutePath().toString(), "/path/to/myinst", df, cc, "dataDir", "mydata");
// solr.data.home set but also solrDataHome set in solr.xml, which should override the former
System.setProperty("test.solr.data.home", "/foo");
config = loadNodeConfig("/solr/solr-solrDataHome.xml");
cc = new CoreContainer(config);
- rdf = new RAMDirectoryFactory();
- rdf.initCoreContainer(cc);
- rdf.init(new NamedList());
- assertDataHome("/foo/inst_dir/data", "inst_dir", rdf, cc);
+ df = directoryFactoryClass.newInstance();
+ df.initCoreContainer(cc);
+ df.init(new NamedList());
+ assertDataHome("/foo/inst_dir/data", "inst_dir", df, cc);
}
- private void assertDataHome(String expected, String instanceDir, RAMDirectoryFactory rdf, CoreContainer cc, String... properties) throws IOException {
- String dataHome = rdf.getDataHome(new CoreDescriptor("core_name", Paths.get(instanceDir), cc.containerProperties, cc.isZooKeeperAware(), properties));
+ private void assertDataHome(String expected, String instanceDir, DirectoryFactory df, CoreContainer cc, String... properties) throws IOException {
+ String dataHome = df.getDataHome(new CoreDescriptor("core_name", Paths.get(instanceDir), cc.containerProperties, cc.isZooKeeperAware(), properties));
assertEquals(Paths.get(expected).toAbsolutePath(), Paths.get(dataHome).toAbsolutePath());
}