You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2011/06/15 10:36:06 UTC
svn commit: r1135956 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/ lucene/backwards/src/test-framework/
lucene/backwards/src/test/ solr/ solr/contrib/dataimporthandler/
solr/contrib/dataimporthandler/src/main/java/org/apache/solr/ha...
Author: shalin
Date: Wed Jun 15 08:36:06 2011
New Revision: 1135956
URL: http://svn.apache.org/viewvc?rev=1135956&view=rev
Log:
SOLR-2551 -- Check dataimport.properties for write access (if delta-import is supported in DIH configuration) before starting an import
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test-framework/ (props changed)
lucene/dev/branches/branch_3x/solr/ (props changed)
lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.txt
lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java
lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
Modified: lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.txt?rev=1135956&r1=1135955&r2=1135956&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.txt Wed Jun 15 08:36:06 2011
@@ -11,7 +11,8 @@ $Id$
================== 3.3.0-dev ==============
-(No Changes)
+* SOLR-2551: Check dataimport.properties for write access (if delta-import is supported
+ in DIH configuration) before starting an import (C S, shalin)
================== 3.2.0 ==================
Modified: lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=1135956&r1=1135955&r2=1135956&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java (original)
+++ lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java Wed Jun 15 08:36:06 2011
@@ -39,6 +39,7 @@ import org.apache.commons.io.IOUtils;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.File;
import java.io.StringReader;
import java.text.SimpleDateFormat;
import java.util.*;
@@ -85,6 +86,8 @@ public class DataImporter {
private final Map<String , Object> coreScopeSession;
+ private boolean isDeltaImportSupported = false;
+
/**
* Only for testing purposes
*/
@@ -113,7 +116,9 @@ public class DataImporter {
initEntity(e, fields, false);
verifyWithSchema(fields);
identifyPk(e);
- }
+ if (e.allAttributes.containsKey(SqlEntityProcessor.DELTA_QUERY))
+ isDeltaImportSupported = true;
+ }
}
private void verifyWithSchema(Map<String, DataConfig.Field> fields) {
@@ -350,6 +355,7 @@ public class DataImporter {
try {
docBuilder = new DocBuilder(this, writer, requestParams);
+ checkWritablePersistFile(writer);
docBuilder.execute();
if (!requestParams.debug)
cumulativeStatistics.add(docBuilder.importStatistics);
@@ -364,6 +370,15 @@ public class DataImporter {
}
+ private void checkWritablePersistFile(SolrWriter writer) {
+ File persistFile = writer.getPersistFile();
+ boolean isWritable = persistFile.exists() ? persistFile.canWrite() : persistFile.getParentFile().canWrite();
+ if (isDeltaImportSupported && !isWritable) {
+ throw new DataImportHandlerException(SEVERE, persistFile.getAbsolutePath() +
+ " is not writable. Delta imports are supported by data config but will not work.");
+ }
+ }
+
public void doDeltaImport(SolrWriter writer, RequestParams requestParams) {
LOG.info("Starting Delta Import");
setStatus(Status.RUNNING_DELTA_DUMP);
@@ -371,6 +386,7 @@ public class DataImporter {
try {
setIndexStartTime(new Date());
docBuilder = new DocBuilder(this, writer, requestParams);
+ checkWritablePersistFile(writer);
docBuilder.execute();
if (!requestParams.debug)
cumulativeStatistics.add(docBuilder.importStatistics);
Modified: lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java?rev=1135956&r1=1135955&r2=1135956&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java (original)
+++ lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java Wed Jun 15 08:36:06 2011
@@ -100,13 +100,10 @@ public class SolrWriter {
try {
props.putAll(p);
- String filePath = configDir;
- if (configDir != null && !configDir.endsWith(File.separator))
- filePath += File.separator;
- filePath += persistFilename;
- propOutput = new FileOutputStream(filePath);
+ File persistFile = getPersistFile();
+ propOutput = new FileOutputStream(persistFile);
props.store(propOutput, null);
- log.info("Wrote last indexed time to " + persistFilename);
+ log.info("Wrote last indexed time to " + persistFile.getAbsolutePath());
} catch (FileNotFoundException e) {
throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
"Unable to persist Index Start Time", e);
@@ -123,6 +120,14 @@ public class SolrWriter {
}
}
+ File getPersistFile() {
+ String filePath = configDir;
+ if (configDir != null && !configDir.endsWith(File.separator))
+ filePath += File.separator;
+ filePath += persistFilename;
+ return new File(filePath);
+ }
+
void finish() {
try {
processor.finish();
Modified: lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java?rev=1135956&r1=1135955&r2=1135956&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java (original)
+++ lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java Wed Jun 15 08:36:06 2011
@@ -20,6 +20,8 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.io.File;
+import java.io.FileOutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -92,7 +94,37 @@ public class TestSqlEntityProcessorDelta
public void testCompositePk_FullImport() throws Exception {
add1document();
}
-
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testNonWritablePersistFile() throws Exception {
+ // See SOLR-2551
+ String configDir = h.getCore().getResourceLoader().getConfigDir();
+ String filePath = configDir;
+ if (configDir != null && !configDir.endsWith(File.separator))
+ filePath += File.separator;
+ filePath += "dataimport.properties";
+ File f = new File(filePath);
+ // execute the test only if we are able to set file to read only mode
+ if ((f.exists() || f.createNewFile()) && f.setReadOnly()) {
+ try {
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("id", "1"));
+ MockDataSource.setIterator(FULLIMPORT_QUERY, parentRow.iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("desc", "hello"));
+ MockDataSource.setIterator("select * from y where y.A='1'", childRow
+ .iterator());
+
+ runFullImport(dataConfig_delta);
+ assertQ(req("id:1"), "//*[@numFound='0']");
+ } finally {
+ f.setWritable(true);
+ }
+ }
+ }
+
// WORKS
@Test