You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2017/10/03 13:39:14 UTC
[33/50] [abbrv] lucene-solr:feature/autoscaling: SOLR-11406: Solr 7.0
cannot read indexes from 6.x versions
SOLR-11406: Solr 7.0 cannot read indexes from 6.x versions
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/39ba1c3d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/39ba1c3d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/39ba1c3d
Branch: refs/heads/feature/autoscaling
Commit: 39ba1c3da56c5ac3e9476830efe1f4c396ccf7c0
Parents: c322e36
Author: Steve Rowe <sa...@gmail.com>
Authored: Fri Sep 29 16:26:59 2017 -0400
Committer: Steve Rowe <sa...@gmail.com>
Committed: Fri Sep 29 16:33:07 2017 -0400
----------------------------------------------------------------------
.../lucene/backward-codecs/pom.xml.template | 13 +++
.../maven/solr/core/src/test/pom.xml.template | 7 ++
.../index/TestBackwardsCompatibility.java | 12 ++-
solr/CHANGES.txt | 21 +++++
solr/common-build.xml | 6 ++
solr/core/build.xml | 3 +-
.../solr/index/SlowCompositeReaderWrapper.java | 16 ++--
.../solr/configsets/backcompat/conf/schema.xml | 22 +++++
.../configsets/backcompat/conf/solrconfig.xml | 43 +++++++++
.../backcompat/TestLuceneIndexBackCompat.java | 96 ++++++++++++++++++++
10 files changed, 230 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39ba1c3d/dev-tools/maven/lucene/backward-codecs/pom.xml.template
----------------------------------------------------------------------
diff --git a/dev-tools/maven/lucene/backward-codecs/pom.xml.template b/dev-tools/maven/lucene/backward-codecs/pom.xml.template
index e40148f..f6e6680 100644
--- a/dev-tools/maven/lucene/backward-codecs/pom.xml.template
+++ b/dev-tools/maven/lucene/backward-codecs/pom.xml.template
@@ -71,5 +71,18 @@
</excludes>
</testResource>
</testResources>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39ba1c3d/dev-tools/maven/solr/core/src/test/pom.xml.template
----------------------------------------------------------------------
diff --git a/dev-tools/maven/solr/core/src/test/pom.xml.template b/dev-tools/maven/solr/core/src/test/pom.xml.template
index a4e979f..fd6b8e7 100644
--- a/dev-tools/maven/solr/core/src/test/pom.xml.template
+++ b/dev-tools/maven/solr/core/src/test/pom.xml.template
@@ -61,6 +61,13 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-backward-codecs</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
@solr-core.internal.test.dependencies@
@solr-core.external.test.dependencies@
</dependencies>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39ba1c3d/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index abb189a..c202973 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -277,7 +277,11 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
final static String[] oldNames = {
};
-
+
+ public static String[] getOldNames() {
+ return oldNames;
+ }
+
final String[] unsupportedNames = {
"1.9.0-cfs",
"1.9.0-nocfs",
@@ -440,7 +444,11 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
// TODO: on 6.0.0 release, gen the single segment indices and add here:
final static String[] oldSingleSegmentNames = {
};
-
+
+ public static String[] getOldSingleSegmentNames() {
+ return oldSingleSegmentNames;
+ }
+
static Map<String,Directory> oldIndexDirs;
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39ba1c3d/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 13cee84..aed0c22 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -235,6 +235,27 @@ Other Changes
* SOLR-10451: Remove contrib/ltr/lib from lib includes in the techproducts example config (sungjunyoung via janhoy)
+================== 7.0.1 ==================
+
+Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
+
+Versions of Major Components
+---------------------
+Apache Tika 1.13
+Carrot2 3.15.0
+Velocity 1.7 and Velocity Tools 2.0
+Apache UIMA 2.3.1
+Apache ZooKeeper 3.4.10
+Jetty 9.3.14.v20161028
+
+Bug Fixes
+----------------------
+
+* SOLR-11297: Message "Lock held by this virtual machine" during startup. Solr is trying to start some cores twice.
+ (Luiz Armesto, Shawn Heisey, Erick Erickson)
+
+* SOLR-11406: Solr 7.0 cannot read indexes from 6.x versions. (Shawn Heisey, Steve Rowe)
+
================== 7.0.0 ==================
Versions of Major Components
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39ba1c3d/solr/common-build.xml
----------------------------------------------------------------------
diff --git a/solr/common-build.xml b/solr/common-build.xml
index 88f71ef..ec37848 100644
--- a/solr/common-build.xml
+++ b/solr/common-build.xml
@@ -493,6 +493,12 @@
</ant>
</target>
+ <target name="-compile-test-lucene-backward-codecs">
+ <ant dir="${common.dir}/backward-codecs" target="compile-test" inheritAll="false">
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ </ant>
+ </target>
+
<!-- Solr contrib targets -->
<target name="-compile-analysis-extras">
<ant dir="${common-solr.dir}/contrib/analysis-extras" target="compile" inheritAll="false">
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39ba1c3d/solr/core/build.xml
----------------------------------------------------------------------
diff --git a/solr/core/build.xml b/solr/core/build.xml
index f7a2a59..45bd1b4 100644
--- a/solr/core/build.xml
+++ b/solr/core/build.xml
@@ -31,7 +31,7 @@
<target name="compile-core" depends="compile-solrj,common-solr.compile-core"/>
- <target name="compile-test" depends="jar-analyzers-icu,-compile-test-lucene-queryparser,-compile-analysis-extras,common-solr.compile-test"/>
+ <target name="compile-test" depends="jar-analyzers-icu,-compile-test-lucene-queryparser,-compile-test-lucene-backward-codecs,-compile-analysis-extras,common-solr.compile-test"/>
<path id="test.classpath">
<path refid="solr.test.base.classpath"/>
@@ -39,6 +39,7 @@
<pathelement location="${analyzers-icu.jar}"/>
<pathelement location="${common-solr.dir}/build/contrib/solr-analysis-extras/classes/java"/>
<pathelement location="${common.dir}/build/queryparser/classes/test"/>
+ <pathelement location="${common.dir}/build/backward-codecs/classes/test"/>
<fileset dir="${common-solr.dir}/contrib/analysis-extras/lib" includes="icu4j*.jar"/>
</path>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39ba1c3d/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
index 2d612e9..283d245 100644
--- a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
+++ b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
@@ -69,12 +69,16 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
if (reader.leaves().isEmpty()) {
metaData = new LeafMetaData(Version.LATEST.major, Version.LATEST, null);
} else {
- Version minVersion = reader.leaves().stream()
- .map(LeafReaderContext::reader)
- .map(LeafReader::getMetaData)
- .map(LeafMetaData::getMinVersion)
- .reduce((v1, v2) -> v1 == null ? null : v2 == null ? null : v2.onOrAfter(v1) ? v1 : v2)
- .get();
+ Version minVersion = Version.LATEST;
+ for (LeafReaderContext leafReaderContext : reader.leaves()) {
+ Version leafVersion = leafReaderContext.reader().getMetaData().getMinVersion();
+ if (leafVersion == null) {
+ minVersion = null;
+ break;
+ } else if (minVersion.onOrAfter(leafVersion)) {
+ minVersion = leafVersion;
+ }
+ }
metaData = new LeafMetaData(reader.leaves().get(0).reader().getMetaData().getCreatedVersionMajor(), minVersion, null);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39ba1c3d/solr/core/src/test-files/solr/configsets/backcompat/conf/schema.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/configsets/backcompat/conf/schema.xml b/solr/core/src/test-files/solr/configsets/backcompat/conf/schema.xml
new file mode 100644
index 0000000..b49edd2
--- /dev/null
+++ b/solr/core/src/test-files/solr/configsets/backcompat/conf/schema.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<schema name="minimal" version="1.1">
+ <fieldType name="string" class="solr.StrField"/>
+ <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" />
+ <uniquekey>id</uniquekey>
+</schema>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39ba1c3d/solr/core/src/test-files/solr/configsets/backcompat/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/configsets/backcompat/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/backcompat/conf/solrconfig.xml
new file mode 100644
index 0000000..f82d5f4
--- /dev/null
+++ b/solr/core/src/test-files/solr/configsets/backcompat/conf/solrconfig.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Minimal solrconfig.xml with /select, /admin and /update only -->
+
+<config>
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <directoryFactory name="DirectoryFactory" class="solr.StandardDirectoryFactory"/>
+
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+ <commitWithin>
+ <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
+ </commitWithin>
+ </updateHandler>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="indent">true</str>
+ </lst>
+ </requestHandler>
+</config>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39ba1c3d/solr/core/src/test/org/apache/solr/backcompat/TestLuceneIndexBackCompat.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/backcompat/TestLuceneIndexBackCompat.java b/solr/core/src/test/org/apache/solr/backcompat/TestLuceneIndexBackCompat.java
new file mode 100644
index 0000000..2541d1f
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/backcompat/TestLuceneIndexBackCompat.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.backcompat;
+
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.lucene.index.TestBackwardsCompatibility;
+import org.apache.lucene.util.TestUtil;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.util.TestHarness;
+import org.junit.Test;
+
+/** Verify we can read/write previous versions' Lucene indexes. */
+public class TestLuceneIndexBackCompat extends SolrTestCaseJ4 {
+ private static final String[] oldNames = TestBackwardsCompatibility.getOldNames();
+ private static final String[] oldSingleSegmentNames = TestBackwardsCompatibility.getOldSingleSegmentNames();
+
+ @Test
+ public void testOldIndexes() throws Exception {
+ List<String> names = new ArrayList<>(oldNames.length + oldSingleSegmentNames.length);
+ names.addAll(Arrays.asList(oldNames));
+ names.addAll(Arrays.asList(oldSingleSegmentNames));
+ for (String name : names) {
+ setupCore(name);
+
+ assertQ(req("q", "*:*", "rows", "0"), "//result[@numFound='35']");
+
+ assertU(adoc("id", "id_123456789"));
+ assertU(commit());
+
+ deleteCore();
+ }
+ }
+
+ private void setupCore(String coreName) throws Exception {
+ if (h != null) {
+ h.close();
+ }
+ Path solrHome = createTempDir(coreName).toAbsolutePath();
+ Files.createDirectories(solrHome);
+ Path coreDir = solrHome.resolve(coreName);
+ Path confDir = coreDir.resolve("conf");
+ Files.createDirectories(confDir);
+ Path dataDir = coreDir.resolve("data");
+ Path indexDir = dataDir.resolve("index");
+ Files.createDirectories(indexDir);
+
+ Files.copy(getFile("solr/solr.xml").toPath(), solrHome.resolve("solr.xml"));
+ FileUtils.copyDirectory(configset("backcompat").toFile(), confDir.toFile());
+
+ try (Writer writer = new OutputStreamWriter(Files.newOutputStream(coreDir.resolve("core.properties")), StandardCharsets.UTF_8)) {
+ Properties coreProps = new Properties();
+ coreProps.put("name", coreName);
+ coreProps.store(writer, null);
+ }
+
+ InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream("index." + coreName + ".zip");
+ assertNotNull("Index name " + coreName + " not found", resource);
+ TestUtil.unzip(resource, indexDir);
+
+ configString = "solrconfig.xml";
+ schemaString = "schema.xml";
+ testSolrHome = solrHome;
+ System.setProperty("solr.solr.home", solrHome.toString());
+ ignoreException("ignore_exception");
+ solrConfig = TestHarness.createConfig(testSolrHome, coreName, getSolrConfigFile());
+ h = new TestHarness(coreName, dataDir.toString(), solrConfig, getSchemaFile());
+ lrf = h.getRequestFactory("",0,20, CommonParams.VERSION,"2.2");
+ }
+}