You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2024/02/20 21:13:42 UTC
(lucene) branch branch_9x updated: Fix bw index generation logic.
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new e65df33f34f Fix bw index generation logic.
e65df33f34f is described below
commit e65df33f34f66a7089d323dcc0e4601a1db25a32
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Tue Feb 20 22:10:01 2024 +0100
Fix bw index generation logic.
---
dev-tools/scripts/addBackcompatIndexes.py | 13 +++---------
.../BackwardsCompatibilityTestBase.java | 23 +++++++++++++++++++---
.../backward_index/TestGenerateBwcIndices.java | 2 ++
3 files changed, 25 insertions(+), 13 deletions(-)
diff --git a/dev-tools/scripts/addBackcompatIndexes.py b/dev-tools/scripts/addBackcompatIndexes.py
index bbaf0b40630..7faacb8b8e3 100755
--- a/dev-tools/scripts/addBackcompatIndexes.py
+++ b/dev-tools/scripts/addBackcompatIndexes.py
@@ -45,16 +45,13 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
'emptyIndex': 'empty'
}[indextype]
if indextype in ('cfs', 'nocfs'):
- dirname = 'index.%s' % indextype
filename = '%s.%s-%s.zip' % (prefix, index_version, indextype)
else:
- dirname = indextype
filename = '%s.%s.zip' % (prefix, index_version)
print(' creating %s...' % filename, end='', flush=True)
module = 'backward-codecs'
index_dir = os.path.join('lucene', module, 'src/test/org/apache/lucene/backward_index')
- test_file = os.path.join(index_dir, filename)
if os.path.exists(os.path.join(index_dir, filename)):
print('uptodate')
return
@@ -76,24 +73,20 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
'-Dtests.codec=default'
])
base_dir = os.getcwd()
- bc_index_dir = os.path.join(temp_dir, dirname)
- bc_index_file = os.path.join(bc_index_dir, filename)
+ bc_index_file = os.path.join(temp_dir, filename)
if os.path.exists(bc_index_file):
print('alreadyexists')
else:
- if os.path.exists(bc_index_dir):
- shutil.rmtree(bc_index_dir)
os.chdir(source)
scriptutil.run('./gradlew %s' % gradle_args)
- os.chdir(bc_index_dir)
- scriptutil.run('zip %s *' % filename)
+ if not os.path.exists(bc_index_file):
+ raise Exception("Expected file can't be found: %s" %bc_index_file)
print('done')
print(' adding %s...' % filename, end='', flush=True)
scriptutil.run('cp %s %s' % (bc_index_file, os.path.join(base_dir, index_dir)))
os.chdir(base_dir)
- scriptutil.run('rm -rf %s' % bc_index_dir)
print('done')
def update_backcompat_tests(index_version, current_version):
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java
index 8df28d40dbc..b131bb9497b 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java
@@ -17,6 +17,7 @@
package org.apache.lucene.backward_index;
import com.carrotsearch.randomizedtesting.annotations.Name;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.LineNumberReader;
@@ -38,11 +39,17 @@ import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.OutputStreamDataOutput;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.BytesRef;
@@ -253,10 +260,20 @@ public abstract class BackwardsCompatibilityTestBase extends LuceneTestCase {
protected abstract void createIndex(Directory directory) throws IOException;
public final void createBWCIndex() throws IOException {
- Path indexDir = getIndexDir().resolve(indexName(Version.LATEST));
- Files.deleteIfExists(indexDir);
- try (Directory dir = newFSDirectory(indexDir)) {
+ Path zipFile = getIndexDir().resolve(indexName(Version.LATEST));
+ Files.deleteIfExists(zipFile);
+ Path tmpDir = createTempDir();
+
+ try (Directory dir = FSDirectory.open(tmpDir);
+ ZipOutputStream zipOut = new ZipOutputStream(new FileOutputStream(zipFile.toFile()))) {
createIndex(dir);
+ for (String file : dir.listAll()) {
+ try (IndexInput in = dir.openInput(file, IOContext.READONCE)) {
+ zipOut.putNextEntry(new ZipEntry(file));
+ new OutputStreamDataOutput(zipOut).copyBytes(in, in.length());
+ zipOut.closeEntry();
+ }
+ }
}
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java
index 0cd9f37d5c3..c7b1ea3fb4a 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java
@@ -20,8 +20,10 @@ import static org.apache.lucene.backward_index.BackwardsCompatibilityTestBase.cr
import java.io.IOException;
import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.LuceneTestCase.SuppressFileSystems;
import org.apache.lucene.util.Version;
+@SuppressFileSystems("ExtrasFS")
public class TestGenerateBwcIndices extends LuceneTestCase {
// Backcompat index generation, described below, is mostly automated in: