You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Uwe Schindler <uw...@thetaphi.de> on 2011/06/16 21:35:17 UTC
RE: svn commit: r1135956 - in /lucene/dev/branches/branch_3x: ./ lucene/ lucene/backwards/ lucene/backwards/src/test-framework/ lucene/backwards/src/test/ solr/ solr/contrib/dataimporthandler/ solr/contrib/dataimporthandler/src/main/java/org/apache/solr/h
Shalin,
i had to comment out your test because the finally block does not compile with Java 5 (Solr 3.1), Jenkins is down at the moment, so did not catch earlier.
-----
Uwe Schindler
H.-H.-Meier-Allee 63, D-28213 Bremen
http://www.thetaphi.de
eMail: uwe@thetaphi.de
> -----Original Message-----
> From: shalin@apache.org [mailto:shalin@apache.org]
> Sent: Wednesday, June 15, 2011 10:36 AM
> To: commits@lucene.apache.org
> Subject: svn commit: r1135956 - in /lucene/dev/branches/branch_3x: ./
> lucene/ lucene/backwards/ lucene/backwards/src/test-framework/
> lucene/backwards/src/test/ solr/ solr/contrib/dataimporthandler/
> solr/contrib/dataimporthandler/src/main/java/org/apache/solr/ha...
>
> Author: shalin
> Date: Wed Jun 15 08:36:06 2011
> New Revision: 1135956
>
> URL: http://svn.apache.org/viewvc?rev=1135956&view=rev
> Log:
> SOLR-2551 -- Check dataimport.properties for write access (if delta-import is
> supported in DIH configuration) before starting an import
>
> Modified:
> lucene/dev/branches/branch_3x/ (props changed)
> lucene/dev/branches/branch_3x/lucene/ (props changed)
> lucene/dev/branches/branch_3x/lucene/backwards/ (props changed)
> lucene/dev/branches/branch_3x/lucene/backwards/src/test/ (props
> changed)
> lucene/dev/branches/branch_3x/lucene/backwards/src/test-framework/
> (props changed)
> lucene/dev/branches/branch_3x/solr/ (props changed)
>
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.
> txt
>
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/j
> ava/org/apache/solr/handler/dataimport/DataImporter.java
>
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/j
> ava/org/apache/solr/handler/dataimport/SolrWriter.java
>
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/ja
> va/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
>
> Modified:
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.
> txt
> URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib
> /dataimporthandler/CHANGES.txt?rev=1135956&r1=1135955&r2=1135956&vi
> ew=diff
> ==========================================================
> ====================
> ---
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.
> txt (original)
> +++
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES
> +++ .txt Wed Jun 15 08:36:06 2011
> @@ -11,7 +11,8 @@ $Id$
>
> ================== 3.3.0-dev ==============
>
> -(No Changes)
> +* SOLR-2551: Check dataimport.properties for write access (if
> +delta-import is supported
> + in DIH configuration) before starting an import (C S, shalin)
>
> ================== 3.2.0 ==================
>
>
> Modified:
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/j
> ava/org/apache/solr/handler/dataimport/DataImporter.java
> URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib
> /dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Dat
> aImporter.java?rev=1135956&r1=1135955&r2=1135956&view=diff
> ==========================================================
> ====================
> ---
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/j
> ava/org/apache/solr/handler/dataimport/DataImporter.java (original)
> +++
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/mai
> +++ n/java/org/apache/solr/handler/dataimport/DataImporter.java Wed
> Jun
> +++ 15 08:36:06 2011
> @@ -39,6 +39,7 @@ import org.apache.commons.io.IOUtils;
>
> import javax.xml.parsers.DocumentBuilder;
> import javax.xml.parsers.DocumentBuilderFactory;
> +import java.io.File;
> import java.io.StringReader;
> import java.text.SimpleDateFormat;
> import java.util.*;
> @@ -85,6 +86,8 @@ public class DataImporter {
>
> private final Map<String , Object> coreScopeSession;
>
> + private boolean isDeltaImportSupported = false;
> +
> /**
> * Only for testing purposes
> */
> @@ -113,7 +116,9 @@ public class DataImporter {
> initEntity(e, fields, false);
> verifyWithSchema(fields);
> identifyPk(e);
> - }
> + if (e.allAttributes.containsKey(SqlEntityProcessor.DELTA_QUERY))
> + isDeltaImportSupported = true;
> + }
> }
>
> private void verifyWithSchema(Map<String, DataConfig.Field> fields) { @@
> -350,6 +355,7 @@ public class DataImporter {
>
> try {
> docBuilder = new DocBuilder(this, writer, requestParams);
> + checkWritablePersistFile(writer);
> docBuilder.execute();
> if (!requestParams.debug)
> cumulativeStatistics.add(docBuilder.importStatistics);
> @@ -364,6 +370,15 @@ public class DataImporter {
>
> }
>
> + private void checkWritablePersistFile(SolrWriter writer) {
> + File persistFile = writer.getPersistFile();
> + boolean isWritable = persistFile.exists() ? persistFile.canWrite() :
> persistFile.getParentFile().canWrite();
> + if (isDeltaImportSupported && !isWritable) {
> + throw new DataImportHandlerException(SEVERE,
> persistFile.getAbsolutePath() +
> + " is not writable. Delta imports are supported by data config but will
> not work.");
> + }
> + }
> +
> public void doDeltaImport(SolrWriter writer, RequestParams
> requestParams) {
> LOG.info("Starting Delta Import");
> setStatus(Status.RUNNING_DELTA_DUMP);
> @@ -371,6 +386,7 @@ public class DataImporter {
> try {
> setIndexStartTime(new Date());
> docBuilder = new DocBuilder(this, writer, requestParams);
> + checkWritablePersistFile(writer);
> docBuilder.execute();
> if (!requestParams.debug)
> cumulativeStatistics.add(docBuilder.importStatistics);
>
> Modified:
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/j
> ava/org/apache/solr/handler/dataimport/SolrWriter.java
> URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib
> /dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Sol
> rWriter.java?rev=1135956&r1=1135955&r2=1135956&view=diff
> ==========================================================
> ====================
> ---
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/main/j
> ava/org/apache/solr/handler/dataimport/SolrWriter.java (original)
> +++
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/mai
> +++ n/java/org/apache/solr/handler/dataimport/SolrWriter.java Wed Jun 15
> +++ 08:36:06 2011
> @@ -100,13 +100,10 @@ public class SolrWriter {
>
> try {
> props.putAll(p);
> - String filePath = configDir;
> - if (configDir != null && !configDir.endsWith(File.separator))
> - filePath += File.separator;
> - filePath += persistFilename;
> - propOutput = new FileOutputStream(filePath);
> + File persistFile = getPersistFile();
> + propOutput = new FileOutputStream(persistFile);
> props.store(propOutput, null);
> - log.info("Wrote last indexed time to " + persistFilename);
> + log.info("Wrote last indexed time to " +
> + persistFile.getAbsolutePath());
> } catch (FileNotFoundException e) {
> throw new
> DataImportHandlerException(DataImportHandlerException.SEVERE,
> "Unable to persist Index Start Time", e); @@ -123,6 +120,14 @@
> public class SolrWriter {
> }
> }
>
> + File getPersistFile() {
> + String filePath = configDir;
> + if (configDir != null && !configDir.endsWith(File.separator))
> + filePath += File.separator;
> + filePath += persistFilename;
> + return new File(filePath);
> + }
> +
> void finish() {
> try {
> processor.finish();
>
> Modified:
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/ja
> va/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
> URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib
> /dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/Test
> SqlEntityProcessorDelta.java?rev=1135956&r1=1135955&r2=1135956&view=
> diff
> ==========================================================
> ====================
> ---
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/ja
> va/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
> (original)
> +++
> lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/tes
> +++ t/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelt
> +++ a.java Wed Jun 15 08:36:06 2011
> @@ -20,6 +20,8 @@ import org.junit.Before; import org.junit.BeforeClass;
> import org.junit.Test;
>
> +import java.io.File;
> +import java.io.FileOutputStream;
> import java.util.ArrayList;
> import java.util.Collections;
> import java.util.List;
> @@ -92,7 +94,37 @@ public class TestSqlEntityProcessorDelta
> public void testCompositePk_FullImport() throws Exception {
> add1document();
> }
> -
> +
> + @Test
> + @SuppressWarnings("unchecked")
> + public void testNonWritablePersistFile() throws Exception {
> + // See SOLR-2551
> + String configDir = h.getCore().getResourceLoader().getConfigDir();
> + String filePath = configDir;
> + if (configDir != null && !configDir.endsWith(File.separator))
> + filePath += File.separator;
> + filePath += "dataimport.properties";
> + File f = new File(filePath);
> + // execute the test only if we are able to set file to read only mode
> + if ((f.exists() || f.createNewFile()) && f.setReadOnly()) {
> + try {
> + List parentRow = new ArrayList();
> + parentRow.add(createMap("id", "1"));
> + MockDataSource.setIterator(FULLIMPORT_QUERY,
> + parentRow.iterator());
> +
> + List childRow = new ArrayList();
> + childRow.add(createMap("desc", "hello"));
> + MockDataSource.setIterator("select * from y where y.A='1'", childRow
> + .iterator());
> +
> + runFullImport(dataConfig_delta);
> + assertQ(req("id:1"), "//*[@numFound='0']");
> + } finally {
> + f.setWritable(true);
> + }
> + }
> + }
> +
> // WORKS
>
> @Test
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org