You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2020/06/11 16:09:22 UTC
[lucene-solr] branch master updated: LUCENE-9356: Add a test that
verifies that Lucene catches bit flips. (#1569)
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new 36109ec LUCENE-9356: Add a test that verifies that Lucene catches bit flips. (#1569)
36109ec is described below
commit 36109ec36216141cb0fbf9fb09e9d74721a78bda
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Thu Jun 11 18:09:09 2020 +0200
LUCENE-9356: Add a test that verifies that Lucene catches bit flips. (#1569)
---
.../lucene/index/TestAllFilesDetectBitFlips.java | 139 +++++++++++++++++++++
1 file changed, 139 insertions(+)
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectBitFlips.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectBitFlips.java
new file mode 100644
index 0000000..854b292
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectBitFlips.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.BaseDirectoryWrapper;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.LineFileDocs;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
+import org.apache.lucene.util.TestUtil;
+
+/**
+ * Test that the default codec detects bit flips at open or checkIntegrity time.
+ */
+@SuppressFileSystems("ExtrasFS")
+public class TestAllFilesDetectBitFlips extends LuceneTestCase {
+
+ public void test() throws Exception {
+ doTest(false);
+ }
+
+ public void testCFS() throws Exception {
+ doTest(true);
+ }
+
+ public void doTest(boolean cfs) throws Exception {
+ Directory dir = newDirectory();
+
+ IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
+ conf.setCodec(TestUtil.getDefaultCodec());
+
+ if (cfs == false) {
+ conf.setUseCompoundFile(false);
+ conf.getMergePolicy().setNoCFSRatio(0.0);
+ }
+
+ RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
+ // Use LineFileDocs so we (hopefully) get most Lucene features
+ // tested, e.g. IntPoint was recently added to it:
+ LineFileDocs docs = new LineFileDocs(random());
+ for (int i = 0; i < 100; i++) {
+ riw.addDocument(docs.nextDoc());
+ if (random().nextInt(7) == 0) {
+ riw.commit();
+ }
+ if (random().nextInt(20) == 0) {
+ riw.deleteDocuments(new Term("docid", Integer.toString(i)));
+ }
+ if (random().nextInt(15) == 0) {
+ riw.updateNumericDocValue(new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i));
+ }
+ }
+ if (TEST_NIGHTLY == false) {
+ riw.forceMerge(1);
+ }
+ riw.close();
+ checkBitFlips(dir);
+ dir.close();
+ }
+
+ private void checkBitFlips(Directory dir) throws IOException {
+ for(String name : dir.listAll()) {
+ if (name.equals(IndexWriter.WRITE_LOCK_NAME) == false) {
+ corruptFile(dir, name);
+ }
+ }
+ }
+
+ private void corruptFile(Directory dir, String victim) throws IOException {
+ try (BaseDirectoryWrapper dirCopy = newDirectory()) {
+ dirCopy.setCheckIndexOnClose(false);
+
+ long victimLength = dir.fileLength(victim);
+ long flipOffset = TestUtil.nextLong(random(), 0, victimLength - 1);
+
+ if (VERBOSE) {
+ System.out.println("TEST: now corrupt file " + victim + " by changing byte at offset " + flipOffset + " (length= " + victimLength + ")");
+ }
+
+ for(String name : dir.listAll()) {
+ if (name.equals(victim) == false) {
+ dirCopy.copyFrom(dir, name, name, IOContext.DEFAULT);
+ } else {
+ try (IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT);
+ IndexInput in = dir.openInput(name, IOContext.DEFAULT)) {
+ out.copyBytes(in, flipOffset);
+ out.writeByte((byte) (in.readByte() + TestUtil.nextInt(random(), 0x01, 0xFF)));
+ out.copyBytes(in, victimLength - flipOffset - 1);
+ }
+ try (IndexInput in = dirCopy.openInput(name, IOContext.DEFAULT)) {
+ try {
+ CodecUtil.checksumEntireFile(in);
+ System.out.println("TEST: changing a byte in " + victim + " did not update the checksum)");
+ return;
+ } catch (CorruptIndexException e) {
+ // ok
+ }
+ }
+ }
+ dirCopy.sync(Collections.singleton(name));
+ }
+
+ // corruption must be detected
+ expectThrowsAnyOf(Arrays.asList(CorruptIndexException.class, IndexFormatTooOldException.class, IndexFormatTooNewException.class),
+ () -> {
+ try (IndexReader reader = DirectoryReader.open(dirCopy)) {
+ for (LeafReaderContext context : reader.leaves()) {
+ context.reader().checkIntegrity();
+ }
+ }
+ }
+ );
+ }
+ }
+}