You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2018/06/29 11:58:22 UTC
lucene-solr:branch_6_6: SOLR-12450: Don't allow referal to external
resources in various config files
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6_6 3940e6a93 -> d1baf6ba5
SOLR-12450: Don't allow referal to external resources in various config files
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d1baf6ba
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d1baf6ba
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d1baf6ba
Branch: refs/heads/branch_6_6
Commit: d1baf6ba593561f39e2da0a71a8440797005b55c
Parents: 3940e6a
Author: Uwe Schindler <us...@apache.org>
Authored: Sun Jun 10 11:47:22 2018 +0200
Committer: Ishan Chattopadhyaya <is...@apache.org>
Committed: Fri Jun 29 17:27:59 2018 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 19 +++
.../handler/extraction/ParseContextConfig.java | 12 +-
.../org/apache/solr/util/SafeXMLParsing.java | 120 +++++++++++++++++++
.../apache/solr/util/TestSafeXMLParsing.java | 99 +++++++++++++++
4 files changed, 245 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d1baf6ba/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index f9a118a..83f1512 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -16,6 +16,25 @@ In this release, there is an example Solr server including a bundled
servlet container in the directory named "example".
See the Quick Start guide at http://lucene.apache.org/solr/quickstart.html
+================== 6.6.5 ==================
+
+Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
+
+Versions of Major Components
+---------------------
+Apache Tika 1.13
+Carrot2 3.15.0
+Velocity 1.7 and Velocity Tools 2.0
+Apache UIMA 2.3.1
+Apache ZooKeeper 3.4.10
+Jetty 9.3.14.v20161028
+
+Bug Fixes
+----------------------
+
+ * SOLR-12450: Don't allow referal to external resources in various config files.
+ (Yuyang Xiao, Uwe Schindler)
+
================== 6.6.4 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d1baf6ba/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ParseContextConfig.java
----------------------------------------------------------------------
diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ParseContextConfig.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ParseContextConfig.java
index 8b9f1ef..b9f58f0 100644
--- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ParseContextConfig.java
+++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ParseContextConfig.java
@@ -16,20 +16,22 @@
*/
package org.apache.solr.handler.extraction;
-import javax.xml.parsers.DocumentBuilderFactory;
import java.beans.BeanInfo;
import java.beans.Introspector;
import java.beans.PropertyDescriptor;
import java.beans.PropertyEditor;
import java.beans.PropertyEditorManager;
-import java.io.InputStream;
+import java.lang.invoke.MethodHandles;
import java.lang.reflect.Method;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.util.SafeXMLParsing;
import org.apache.tika.parser.ParseContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
@@ -37,6 +39,8 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
public class ParseContextConfig {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
private final Map<Class<?>, Object> entries = new HashMap<>();
/** Creates an empty Config without any settings (used as placeholder). */
@@ -54,9 +58,7 @@ public class ParseContextConfig {
}
private static Document loadConfigFile(SolrResourceLoader resourceLoader, String parseContextConfigLoc) throws Exception {
- try (InputStream in = resourceLoader.openResource(parseContextConfigLoc)) {
- return DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(in, parseContextConfigLoc);
- }
+ return SafeXMLParsing.parseConfigXML(log, resourceLoader, parseContextConfigLoc);
}
private void extract(Element element, SolrResourceLoader loader) throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d1baf6ba/solr/core/src/java/org/apache/solr/util/SafeXMLParsing.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/SafeXMLParsing.java b/solr/core/src/java/org/apache/solr/util/SafeXMLParsing.java
new file mode 100644
index 0000000..6d8dbf7
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/SafeXMLParsing.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.util;
+
+import java.io.FilterReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+import java.io.StringReader;
+
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+
+import org.apache.commons.io.input.CloseShieldInputStream;
+import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.solr.common.EmptyEntityResolver;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.SuppressForbidden;
+import org.apache.solr.common.util.XMLErrorLogger;
+import org.slf4j.Logger;
+import org.w3c.dom.Document;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+/**
+ * Some utility methods for parsing XML in a safe way. This class can be used to parse XML
+ * coming from network (completely untrusted) or it can load a config file from a
+ * {@link ResourceLoader}. In this case it allows external entities and xincludes, but only
+ * referring to files reachable by the loader.
+ */
+@SuppressForbidden(reason = "This class uses XML APIs directly that should not be used anywhere else in Solr code")
+public final class SafeXMLParsing {
+
+ public static final String SYSTEMID_UNTRUSTED = "untrusted://stream";
+
+ private SafeXMLParsing() {}
+
+ /** Parses a config file from ResourceLoader. Xinclude and external entities are enabled, but cannot escape the resource loader. */
+ public static Document parseConfigXML(Logger log, ResourceLoader loader, String file) throws SAXException, IOException {
+ try (InputStream in = loader.openResource(file)) {
+ final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ dbf.setValidating(false);
+ dbf.setNamespaceAware(true);
+ trySetDOMFeature(dbf, XMLConstants.FEATURE_SECURE_PROCESSING, true);
+ try {
+ dbf.setXIncludeAware(true);
+ } catch (UnsupportedOperationException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "XML parser doesn't support XInclude option", e);
+ }
+
+ final DocumentBuilder db = dbf.newDocumentBuilder();
+ db.setEntityResolver(new SystemIdResolver(loader));
+ db.setErrorHandler(new XMLErrorLogger(log));
+ return db.parse(in, SystemIdResolver.createSystemIdFromResourceName(file));
+ } catch (ParserConfigurationException pce) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "XML parser cannot be configured", pce);
+ }
+ }
+
+ /** Parses the given InputStream as XML, disabling any external entities with secure processing enabled.
+ * The given InputStream is not closed. */
+ public static Document parseUntrustedXML(Logger log, InputStream in) throws SAXException, IOException {
+ return getUntrustedDocumentBuilder(log).parse(new CloseShieldInputStream(in), SYSTEMID_UNTRUSTED);
+ }
+
+ /** Parses the given InputStream as XML, disabling any external entities with secure processing enabled.
+ * The given Reader is not closed. */
+ public static Document parseUntrustedXML(Logger log, Reader reader) throws SAXException, IOException {
+ final InputSource is = new InputSource(new FilterReader(reader) {
+ @Override public void close() {}
+ });
+ is.setSystemId(SYSTEMID_UNTRUSTED);
+ return getUntrustedDocumentBuilder(log).parse(is);
+ }
+
+ public static Document parseUntrustedXML(Logger log, String xml) throws SAXException, IOException {
+ return parseUntrustedXML(log, new StringReader(xml));
+ }
+
+ private static DocumentBuilder getUntrustedDocumentBuilder(Logger log) {
+ try {
+ final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ dbf.setValidating(false);
+ dbf.setNamespaceAware(true);
+ trySetDOMFeature(dbf, XMLConstants.FEATURE_SECURE_PROCESSING, true);
+
+ final DocumentBuilder db = dbf.newDocumentBuilder();
+ db.setEntityResolver(EmptyEntityResolver.SAX_INSTANCE);
+ db.setErrorHandler(new XMLErrorLogger(log));
+ return db;
+ } catch (ParserConfigurationException pce) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "XML parser cannot be configured", pce);
+ }
+ }
+
+ private static void trySetDOMFeature(DocumentBuilderFactory factory, String feature, boolean enabled) {
+ try {
+ factory.setFeature(feature, enabled);
+ } catch (Exception ex) {
+ // ignore
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d1baf6ba/solr/core/src/test/org/apache/solr/util/TestSafeXMLParsing.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/TestSafeXMLParsing.java b/solr/core/src/test/org/apache/solr/util/TestSafeXMLParsing.java
new file mode 100644
index 0000000..db71a8c
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/util/TestSafeXMLParsing.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.lucene.util.LuceneTestCase;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+
+public class TestSafeXMLParsing extends LuceneTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public void testUntrusted() throws Exception {
+ // TODO: Fix the underlying EmptyEntityResolver to not replace external entities by nothing and instead throw exception:
+ Document doc = SafeXMLParsing.parseUntrustedXML(log, "<!DOCTYPE test [\n" +
+ "<!ENTITY internalTerm \"foobar\">\n" +
+ "<!ENTITY externalTerm SYSTEM \"foo://bar.xyz/external\">\n" +
+ "]>\n" +
+ "<test>&internalTerm;&externalTerm;</test>");
+ assertEquals("foobar", doc.getDocumentElement().getTextContent());
+ }
+
+ InputStream getStringStream(String xml) {
+ return new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8));
+ }
+
+ public void testConfig() throws Exception {
+ final ResourceLoader loader = new ResourceLoader() {
+ @Override
+ public InputStream openResource(String resource) throws IOException {
+ switch (resource) {
+ case "source1.xml":
+ return getStringStream("<!DOCTYPE test [\n" +
+ "<!ENTITY externalTerm SYSTEM \"foo://bar.xyz/external\">\n" +
+ "]>\n" +
+ "<test>&externalTerm;</test>");
+ case "source2.xml":
+ return getStringStream("<!DOCTYPE test [\n" +
+ "<!ENTITY externalTerm SYSTEM \"./include1.xml\">\n" +
+ "]>\n" +
+ "<test>&externalTerm;</test>");
+ case "source3.xml":
+ return getStringStream("<foo xmlns:xi=\"http://www.w3.org/2001/XInclude\">\n" +
+ " <xi:include href=\"./include2.xml\"/>\n" +
+ "</foo>");
+ case "include1.xml":
+ return getStringStream("Make XML Great Again!™");
+ case "include2.xml":
+ return getStringStream("<bar>Make XML Great Again!™</bar>");
+ }
+ throw new IOException("Resource not found: " + resource);
+ }
+
+ @Override
+ public <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <T> T newInstance(String cname, Class<T> expectedType) {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+
+ IOException ioe = expectThrows(IOException.class, () -> {
+ SafeXMLParsing.parseConfigXML(log, loader, "source1.xml");
+ });
+ assertTrue(ioe.getMessage().contains("Cannot resolve absolute systemIDs"));
+
+ Document doc = SafeXMLParsing.parseConfigXML(log, loader, "source2.xml");
+ assertEquals("Make XML Great Again!™", doc.getDocumentElement().getTextContent());
+
+ doc = SafeXMLParsing.parseConfigXML(log, loader, "source3.xml");
+ assertEquals("Make XML Great Again!™", doc.getDocumentElement().getTextContent().trim());
+ }
+
+}