You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@shindig.apache.org by lr...@apache.org on 2009/02/05 02:41:59 UTC
svn commit: r740973 - in /incubator/shindig/trunk/java:
common/src/main/bundle/org/apache/shindig/common/cache/ehcache/
gadgets/src/main/java/org/apache/shindig/gadgets/
gadgets/src/main/java/org/apache/shindig/gadgets/http/
gadgets/src/main/java/org/a...
Author: lryan
Date: Thu Feb 5 01:41:58 2009
New Revision: 740973
URL: http://svn.apache.org/viewvc?rev=740973&view=rev
Log:
DOM based CSS rewriting for improved rendering performance
CSS Santization support in santizing rewriter
Image verification support in sanitizing rewriter
Removed dead lexer based rewriter code
Added:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/parse/caja/CajaCssParser.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/parse/caja/CajaCssSanitizer.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/parse/caja/CajaCssUtils.java
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/parse/caja/CajaCssParserTest.java
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/parse/caja/CajaCssSanitizerTest.java
Removed:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/CssRewriter.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/lexer/HtmlRewriter.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/lexer/HtmlTagTransformer.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/lexer/JavascriptTagMerger.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/lexer/LinkingTagRewriter.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/lexer/StyleTagRewriter.java
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/CssRewriterTest.java
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/lexer/HtmlRewriterTest.java
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/lexer/JavascriptTagMergerTest.java
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/lexer/LinkingTagRewriterTest.java
Modified:
incubator/shindig/trunk/java/common/src/main/bundle/org/apache/shindig/common/cache/ehcache/ehcacheConfig.xml
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/GadgetException.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpRequest.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/render/SanitizedRenderingContentRewriter.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/CSSContentRewriter.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/ContentRewriterFeatureFactory.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/HTMLContentRewriter.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ProxyBase.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ProxyHandler.java
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/render/SanitizedRenderingContentRewriterTest.java
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/CSSContentRewriterTest.java
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/HTMLContentRewriterTest.java
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/servlet/MakeRequestHandlerTest.java
incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-fragment-expected.html
incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-fragment.html
incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-headnobody-expected.html
incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-headnobody.html
incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/rewrite/rewritebasic-expected.css
incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/rewrite/rewritebasic.css
incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/rewrite/rewritestylebasic.html
Modified: incubator/shindig/trunk/java/common/src/main/bundle/org/apache/shindig/common/cache/ehcache/ehcacheConfig.xml
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/common/src/main/bundle/org/apache/shindig/common/cache/ehcache/ehcacheConfig.xml?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/common/src/main/bundle/org/apache/shindig/common/cache/ehcache/ehcacheConfig.xml (original)
+++ incubator/shindig/trunk/java/common/src/main/bundle/org/apache/shindig/common/cache/ehcache/ehcacheConfig.xml Thu Feb 5 01:41:58 2009
@@ -35,9 +35,17 @@
diskPersistent="false"
memoryStoreEvictionPolicy="LFU"/>
- <!-- By default do not cache any parsed documents. This is experimental -->
+ <!-- Used to cache parsed HTML DOMs based on their content -->
<cache name="parsedDocuments"
- maxElementsInMemory="0"
+ maxElementsInMemory="1000"
+ eternal="true"
+ overflowToDisk="false"
+ diskPersistent="false"
+ memoryStoreEvictionPolicy="LFU"/>
+
+ <!-- Used to cache parsed CSS DOMs based on their content -->
+ <cache name="parsedCss"
+ maxElementsInMemory="1000"
eternal="true"
overflowToDisk="false"
diskPersistent="false"
Modified: incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/GadgetException.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/GadgetException.java?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/GadgetException.java (original)
+++ incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/GadgetException.java Thu Feb 5 01:41:58 2009
@@ -62,6 +62,7 @@
// Parsing errors
CSS_PARSE_ERROR,
HTML_PARSE_ERROR,
+ IMAGE_PARSE_ERROR,
JS_PARSE_ERROR,
// View errors
Modified: incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpRequest.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpRequest.java?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpRequest.java (original)
+++ incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpRequest.java Thu Feb 5 01:41:58 2009
@@ -18,6 +18,8 @@
*/
package org.apache.shindig.gadgets.http;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.shindig.auth.SecurityToken;
import org.apache.shindig.common.uri.Uri;
import org.apache.shindig.config.ContainerConfig;
@@ -27,9 +29,6 @@
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.ArrayUtils;
-
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -60,6 +59,9 @@
private boolean ignoreCache;
private int cacheTtl = -1;
+ // Sanitization
+ private boolean sanitizationRequested;
+
// Whether to follow redirects
private boolean followRedirects = true;
@@ -200,6 +202,18 @@
}
/**
+ * Should content fetched in response to this request
+ * be sanitized based on the specified mime-type
+ */
+ public boolean isSanitizationRequested() {
+ return sanitizationRequested;
+ }
+
+ public void setSanitizationRequested(boolean sanitizationRequested) {
+ this.sanitizationRequested = sanitizationRequested;
+ }
+
+ /**
* @param cacheTtl The amount of time to cache the result object for, in seconds. If set to -1,
* HTTP cache control headers will be honored. Otherwise objects will be cached for the time
* specified.
Added: incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/parse/caja/CajaCssParser.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/parse/caja/CajaCssParser.java?rev=740973&view=auto
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/parse/caja/CajaCssParser.java (added)
+++ incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/parse/caja/CajaCssParser.java Thu Feb 5 01:41:58 2009
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.apache.shindig.gadgets.parse.caja;
+
+import org.apache.shindig.common.cache.Cache;
+import org.apache.shindig.common.cache.CacheProvider;
+import org.apache.shindig.common.util.HashUtil;
+import org.apache.shindig.gadgets.GadgetException;
+
+import com.google.caja.lexer.CharProducer;
+import com.google.caja.lexer.CssLexer;
+import com.google.caja.lexer.CssTokenType;
+import com.google.caja.lexer.InputSource;
+import com.google.caja.lexer.ParseException;
+import com.google.caja.lexer.Token;
+import com.google.caja.lexer.TokenQueue;
+import com.google.caja.lexer.TokenStream;
+import com.google.caja.parser.css.CssParser;
+import com.google.caja.parser.css.CssTree;
+import com.google.caja.render.CssPrettyPrinter;
+import com.google.caja.reporting.MessageContext;
+import com.google.caja.reporting.RenderContext;
+import com.google.caja.util.Criterion;
+import com.google.inject.Inject;
+
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.net.URI;
+import java.util.Collections;
+
+/** A CSS DOM parser using Caja. */
+public class CajaCssParser {
+
+ /**
+ * Dummy URI that is never read from. Needed to construct Caja parser
+ */
+ private static final URI FAKE_SOURCE = URI.create("http://a.dummy.url");
+
+ private static final String PARSED_CSS = "parsedCss";
+
+ private Cache<String, CssTree.StyleSheet> parsedCssCache;
+
+ @Inject
+ public void setCacheProvider(CacheProvider cacheProvider) {
+ parsedCssCache = cacheProvider.createCache(PARSED_CSS);
+ }
+
+ /**
+ * Parse CSS content into Caja's CSS DOM model
+ *
+ * @return A parsed stylesheet
+ */
+ public CssTree.StyleSheet parseDom(String content) throws GadgetException {
+ CssTree.StyleSheet parsedCss = null;
+ boolean shouldCache = shouldCache();
+ String key = null;
+ if (shouldCache) {
+ // TODO - Consider using the source if its under a certain size
+ key = HashUtil.rawChecksum(content.getBytes());
+ parsedCss = parsedCssCache.getElement(key);
+ }
+ if (parsedCss == null) {
+ try {
+ InputSource inputSource = new InputSource(FAKE_SOURCE);
+ CharProducer producer = CharProducer.Factory.create(new StringReader(content),
+ inputSource);
+ TokenStream<CssTokenType> lexer = new CssLexer(producer);
+ TokenQueue<CssTokenType> queue = new TokenQueue<CssTokenType>(lexer, inputSource,
+ new Criterion<Token<CssTokenType>>() {
+ public boolean accept(Token<CssTokenType> t) {
+ return CssTokenType.SPACE != t.type
+ && CssTokenType.COMMENT != t.type;
+ }
+ });
+ if (queue.isEmpty()) {
+ // Return empty stylesheet
+ return new CssTree.StyleSheet(null, Collections.<CssTree.CssStatement>emptyList());
+ }
+ CssParser parser = new CssParser(queue);
+ parsedCss = parser.parseStyleSheet();
+ if (shouldCache) {
+ parsedCssCache.addElement(key, parsedCss);
+ }
+ } catch (ParseException pe) {
+ throw new GadgetException(GadgetException.Code.CSS_PARSE_ERROR, pe);
+ }
+ }
+ if (shouldCache) {
+ return (CssTree.StyleSheet)parsedCss.clone();
+ }
+ return parsedCss;
+ }
+
+ /** Serialize a stylesheet to a String */
+ public String serialize(CssTree.StyleSheet styleSheet) {
+ StringWriter writer = new StringWriter();
+ serialize(styleSheet, writer);
+ return writer.toString();
+ }
+
+ /** Serialize a stylesheet to a Writer. */
+ public void serialize(CssTree.StyleSheet styleSheet, Writer writer) {
+ CssPrettyPrinter cssPrinter = new CssPrettyPrinter(writer, null);
+ styleSheet.render(new RenderContext(new MessageContext(), cssPrinter));
+ cssPrinter.noMoreTokens();
+ }
+
+ private boolean shouldCache() {
+ return parsedCssCache != null && parsedCssCache.getCapacity() != 0;
+ }
+}
Added: incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/parse/caja/CajaCssSanitizer.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/parse/caja/CajaCssSanitizer.java?rev=740973&view=auto
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/parse/caja/CajaCssSanitizer.java (added)
+++ incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/parse/caja/CajaCssSanitizer.java Thu Feb 5 01:41:58 2009
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.apache.shindig.gadgets.parse.caja;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.shindig.common.uri.Uri;
+import org.apache.shindig.gadgets.GadgetException;
+import org.apache.shindig.gadgets.rewrite.LinkRewriter;
+
+import com.google.caja.lang.css.CssSchema;
+import com.google.caja.parser.AbstractParseTreeNode;
+import com.google.caja.parser.AncestorChain;
+import com.google.caja.parser.Visitor;
+import com.google.caja.parser.css.CssTree;
+import com.google.caja.reporting.SimpleMessageQueue;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Inject;
+
+import org.w3c.dom.Element;
+
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Sanitize a CSS tree using Caja. Strip properties and functions that represent
+ * ways to execute script. Specifically
+ *
+ * - Use Caja's CSS property whitelist
+ * - Use Caja's CSS function whitelist
+ * - Force @import through the proxy and require sanitization. If they cant be parsed, remove them
+ * - Force @url references to have the HTTP/HTTPS protocol
+ */
+public class CajaCssSanitizer {
+
+ private static final Logger logger = Logger.getLogger(CajaCssSanitizer.class.getName());
+
+ private static final Set<String> ALLOWED_URI_SCHEMES = ImmutableSet.of("http", "https");
+
+ private final CajaCssParser parser;
+
+ private final CssSchema schema;
+
+ @Inject
+ public CajaCssSanitizer(CajaCssParser parser) {
+ this.parser = parser;
+ schema = CssSchema.getDefaultCss21Schema(new SimpleMessageQueue());
+ }
+
+ /**
+ * Sanitize the CSS content of a style tag.
+ * @param content to sanitize
+ * @param linkContext url of containing content
+ * @param importRewriter to rewrite links to sanitizing proxy
+ */
+ public String sanitize(String content, Uri linkContext, LinkRewriter importRewriter) {
+ try {
+ CssTree.StyleSheet stylesheet = parser.parseDom(content);
+ sanitize(stylesheet, linkContext, importRewriter);
+ // Write the rewritten CSS back into the element
+ return parser.serialize(stylesheet);
+ } catch (GadgetException ge) {
+ // Failed to parse stylesheet so log and continue
+ logger.log(Level.INFO, "Failed to parse stylesheet", ge);
+ return "";
+ }
+ }
+
+ /**
+ * Sanitize the CSS content of a style tag.
+ * @param styleElem to sanitize
+ * @param linkContext url of containing content
+ * @param importRewriter to rewrite links to sanitizing proxy
+ */
+ public void sanitize(Element styleElem, Uri linkContext, LinkRewriter importRewriter) {
+ String content = null;
+ try {
+ CssTree.StyleSheet stylesheet = parser.parseDom(styleElem.getTextContent());
+ sanitize(stylesheet, linkContext, importRewriter);
+ // Write the rewritten CSS back into the element
+ content = parser.serialize(stylesheet);
+ } catch (GadgetException ge) {
+ // Failed to parse stylesheet so log and continue
+ logger.log(Level.INFO, "Failed to parse stylesheet", ge);
+ }
+ if (StringUtils.isEmpty(content)) {
+ // Remove the owning node
+ styleElem.getParentNode().removeChild(styleElem);
+ } else {
+ styleElem.setTextContent(content);
+ }
+ }
+
+ /**
+ * Sanitize the given CSS tree in-place by removing all non-whitelisted function calls
+ * @param css DOM root
+ * @param linkContext url of containing content
+ * @param importRewriter to rewrite links to sanitizing proxy
+ */
+ public void sanitize(CssTree css, final Uri linkContext, final LinkRewriter importRewriter) {
+ css.acceptPreOrder(new Visitor() {
+ public boolean visit(AncestorChain<?> ancestorChain) {
+ if (ancestorChain.node instanceof CssTree.Property) {
+ if (!schema.isPropertyAllowed(((CssTree.Property) ancestorChain.node).
+ getPropertyName())) {
+ // Remove offending property
+ if (logger.isLoggable(Level.FINE)) {
+ logger.log(Level.FINE, "Removing property "
+ + ((CssTree.Property) ancestorChain.node).getPropertyName());
+ }
+ clean(ancestorChain);
+ }
+ } else if (ancestorChain.node instanceof CssTree.FunctionCall) {
+ if (!schema.isFunctionAllowed(((CssTree.FunctionCall)ancestorChain.node).getName())) {
+ // Remove offending node
+ if (logger.isLoggable(Level.FINE)) {
+ logger.log(Level.FINE, "Removing function "
+ + ((CssTree.FunctionCall) ancestorChain.node).getName());
+ }
+ clean(ancestorChain);
+ }
+ } else if (ancestorChain.node instanceof CssTree.UriLiteral) {
+ boolean validUri = false;
+ String uri = ((CssTree.UriLiteral)ancestorChain.node).getValue();
+ try {
+ String scheme = Uri.parse(uri).getScheme();
+ validUri = StringUtils.isEmpty(scheme) ||
+ ALLOWED_URI_SCHEMES.contains(scheme.toLowerCase());
+ } catch (RuntimeException re) {
+ if (logger.isLoggable(Level.FINE)) {
+ logger.log(Level.FINE, "Failed to parse URI in CSS " + uri, re);
+ }
+ }
+ if (!validUri) {
+ // Remove offending node
+ if (logger.isLoggable(Level.FINE)) {
+ logger.log(Level.FINE, "Removing invalid URI "
+ + ((CssTree.UriLiteral) ancestorChain.node).getValue());
+ }
+ clean(ancestorChain);
+ }
+ } else if (ancestorChain.node instanceof CssTree.Import) {
+ CssTree.Import importDecl = (CssTree.Import) ancestorChain.node;
+ importDecl.getUri()
+ .setValue(importRewriter.rewrite(importDecl.getUri().getValue(), linkContext));
+ }
+ return true;
+ }
+ }, null);
+ }
+
+ /**
+ * recurse up through chain to find a safe clean point
+ * @param chain chain of nodes
+ */
+ private static void clean(AncestorChain<?> chain) {
+ if (chain.node instanceof CssTree.Declaration ||
+ chain.node instanceof CssTree.Import) {
+ // Remove the entire subtree
+ ((AbstractParseTreeNode)chain.getParentNode()).removeChild(chain.node);
+ } else {
+ clean(chain.parent);
+ }
+ }
+}
Added: incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/parse/caja/CajaCssUtils.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/parse/caja/CajaCssUtils.java?rev=740973&view=auto
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/parse/caja/CajaCssUtils.java (added)
+++ incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/parse/caja/CajaCssUtils.java Thu Feb 5 01:41:58 2009
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.apache.shindig.gadgets.parse.caja;
+
+import com.google.caja.parser.AncestorChain;
+import com.google.caja.parser.Visitor;
+import com.google.caja.parser.css.CssTree;
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+/**
+ * Utility functions for traversing Caja's CSS DOM
+ */
+class CajaCssUtils {
+
+ /**
+ * Get the immediate children of the passed node with the specified node type
+ */
+ public static <T extends CssTree> List<T> children(CssTree node, Class<T> nodeType) {
+ List<T> result = Lists.newArrayList();
+ for (CssTree child : node.children()) {
+ if (nodeType.isAssignableFrom(child.getClass())) {
+ result.add(nodeType.cast(child));
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Get all descendants of the passed node with the specified node type
+ */
+ public static <T extends CssTree> List<T> descendants(CssTree node, final Class<T> nodeType) {
+ final List<T> descendants = Lists.newArrayList();
+ node.acceptPreOrder(new Visitor() {
+ public boolean visit(AncestorChain<?> ancestorChain) {
+ if (nodeType.isAssignableFrom(ancestorChain.node.getClass())) {
+ descendants.add(nodeType.cast(ancestorChain.node));
+ }
+ return true;
+ }
+ }, null);
+ return descendants;
+ }
+}
+
Modified: incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/render/SanitizedRenderingContentRewriter.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/render/SanitizedRenderingContentRewriter.java?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/render/SanitizedRenderingContentRewriter.java (original)
+++ incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/render/SanitizedRenderingContentRewriter.java Thu Feb 5 01:41:58 2009
@@ -18,17 +18,29 @@
*/
package org.apache.shindig.gadgets.render;
+import org.apache.sanselan.ImageFormat;
+import org.apache.sanselan.ImageReadException;
+import org.apache.sanselan.Sanselan;
+import org.apache.sanselan.common.byteSources.ByteSourceInputStream;
import org.apache.shindig.common.uri.Uri;
import org.apache.shindig.gadgets.Gadget;
import org.apache.shindig.gadgets.http.HttpRequest;
import org.apache.shindig.gadgets.http.HttpResponse;
+import org.apache.shindig.gadgets.parse.caja.CajaCssSanitizer;
import org.apache.shindig.gadgets.rewrite.ContentRewriter;
+import org.apache.shindig.gadgets.rewrite.ContentRewriterFeature;
+import org.apache.shindig.gadgets.rewrite.ContentRewriterFeatureFactory;
+import org.apache.shindig.gadgets.rewrite.LinkRewriter;
import org.apache.shindig.gadgets.rewrite.MutableContent;
+import org.apache.shindig.gadgets.rewrite.ProxyingLinkRewriter;
import org.apache.shindig.gadgets.rewrite.RewriterResults;
+import org.apache.shindig.gadgets.servlet.ProxyBase;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.inject.BindingAnnotation;
import com.google.inject.Inject;
+import com.google.inject.name.Named;
import org.w3c.dom.Attr;
import org.w3c.dom.Element;
@@ -36,13 +48,17 @@
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
+import java.io.IOException;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
/**
* A content rewriter that will sanitize output for simple 'badge' like display.
@@ -55,81 +71,216 @@
* rendering, OSML, etc.)
*/
public class SanitizedRenderingContentRewriter implements ContentRewriter {
+ private static final Logger logger =
+ Logger.getLogger(SanitizedRenderingContentRewriter.class.getName());
+
private static final Set<String> URI_ATTRIBUTES = ImmutableSet.of("href", "src");
+ // Attributes to forcibly rewrite and require an image mime type
+ private static final Map<String, ImmutableSet<String>> PROXY_IMAGE_ATTRIBUTES =
+ ImmutableMap.of("img", ImmutableSet.of("src"));
+
private final Set<String> allowedTags;
private final Set<String> allowedAttributes;
+ private final CajaCssSanitizer cssSanitizer;
+ private final ContentRewriterFeatureFactory rewriterFeatureFactory;
+ private final String proxyBaseNoGadget;
@Inject
public SanitizedRenderingContentRewriter(@AllowedTags Set<String> allowedTags,
- @AllowedAttributes Set<String> allowedAttributes) {
+ @AllowedAttributes Set<String> allowedAttributes,
+ ContentRewriterFeatureFactory rewriterFeatureFactory,
+ @Named("shindig.content-rewrite.proxy-url")String proxyBaseNoGadget,
+ CajaCssSanitizer cssSanitizer) {
this.allowedTags = allowedTags;
this.allowedAttributes = allowedAttributes;
+ this.cssSanitizer = cssSanitizer;
+ this.rewriterFeatureFactory = rewriterFeatureFactory;
+ this.proxyBaseNoGadget = proxyBaseNoGadget;
}
public RewriterResults rewrite(HttpRequest request, HttpResponse resp, MutableContent content) {
- return null;
+ // Content fetched through the proxy can stipulate that it must be sanitized.
+ if (request.isSanitizationRequested()) {
+ ContentRewriterFeature rewriterFeature =
+ rewriterFeatureFactory.createRewriteAllFeature(request.getCacheTtl());
+ if (request.getRewriteMimeType().equalsIgnoreCase("text/css")) {
+ return rewriteProxiedCss(request, resp, content, rewriterFeature);
+ } else if (request.getRewriteMimeType().toLowerCase().startsWith("image/")) {
+ return rewriteProxiedImage(request, resp, content);
+ } else {
+ logger.log(Level.WARNING, "Request to sanitize unknown content type "
+ + request.getRewriteMimeType()
+ + " for " + request.getUri().toString());
+ content.setContent("");
+ return RewriterResults.notCacheable();
+ }
+ } else {
+ // No Op
+ return null;
+ }
}
public RewriterResults rewrite(Gadget gadget, MutableContent content) {
- if ("1".equals(gadget.getContext().getParameter("sanitize"))) {
- sanitize(content.getDocument().getDocumentElement());
- content.documentChanged();
+ if ("1".equals(gadget.getContext().getParameter(ProxyBase.SANITIZE_CONTENT_PARAM))) {
+ boolean sanitized = false;
+ try {
+ new NodeSanitizer(gadget).sanitize(content.getDocument().getDocumentElement());
+ content.documentChanged();
+ sanitized = true;
+ } finally {
+ // Defensively cleat the content in case of failure
+ if (!sanitized) {
+ content.setContent("");
+ }
+ }
}
-
return RewriterResults.notCacheable();
}
- private void sanitize(Node node) {
- switch (node.getNodeType()) {
- case Node.CDATA_SECTION_NODE:
- case Node.TEXT_NODE:
- case Node.ENTITY_REFERENCE_NODE:
- break;
- case Node.ELEMENT_NODE:
- case Node.DOCUMENT_NODE:
- Element element = (Element) node;
- if (allowedTags.contains(element.getTagName().toLowerCase())) {
- filterAttributes(element);
- for (Node child : toList(node.getChildNodes())) {
- sanitize(child);
+ /**
+ * We don't actually rewrite the image we just ensure that it is in fact a valid
+ * and known image type.
+ */
+ private RewriterResults rewriteProxiedImage(HttpRequest request, HttpResponse resp,
+ MutableContent content) {
+ boolean imageIsSafe = false;
+ try {
+ String contentType = resp.getHeader("Content-Type");
+ if (contentType == null || contentType.toLowerCase().startsWith("image/")) {
+ // Unspecified or unknown image mime type.
+ try {
+ ImageFormat imageFormat = Sanselan
+ .guessFormat(new ByteSourceInputStream(resp.getResponse(),
+ request.getUri().getPath()));
+ if (imageFormat == ImageFormat.IMAGE_FORMAT_UNKNOWN) {
+ logger.log(Level.INFO, "Unable to sanitize unknown image type "
+ + request.getUri().toString());
+ return RewriterResults.notCacheable();
}
- } else {
- node.getParentNode().removeChild(node);
+ imageIsSafe = true;
+ // Return null to indicate that no rewriting occurred
+ return null;
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ } catch (ImageReadException ire) {
+ throw new RuntimeException(ire);
}
- break;
- case Node.COMMENT_NODE:
- default:
- // Must remove all comments to avoid conditional comment evaluation.
- // There might be other, unknown types as well. Don't trust them.
- node.getParentNode().removeChild(node);
- break;
+ } else {
+ return RewriterResults.notCacheable();
+ }
+ } finally {
+ if (!imageIsSafe) {
+ content.setContent("");
+ }
+ }
+ }
+
+ /**
+ * Sanitize a CSS file.
+ */
+ private RewriterResults rewriteProxiedCss(HttpRequest request, HttpResponse response,
+ MutableContent content, ContentRewriterFeature rewriterFeature) {
+ String sanitized = "";
+ try {
+ String contentType = response.getHeader("Content-Type");
+ if (contentType == null || contentType.toLowerCase().startsWith("text/")) {
+ SanitizingProxyingLinkRewriter cssLinkRewriter = new SanitizingProxyingLinkRewriter(
+ request.getGadget(), rewriterFeature, proxyBaseNoGadget, "text/css");
+ sanitized = cssSanitizer.sanitize(content.getContent(), request.getUri(), cssLinkRewriter);
+ return RewriterResults.cacheable(response.getCacheTtl());
+ } else {
+ return RewriterResults.notCacheable();
+ }
+ } finally {
+ // Set sanitized content in finally to ensure it is always cleared in
+ // the case of errors
+ content.setContent(sanitized);
}
}
- private void filterAttributes(Element element) {
-
- for (Attr attribute : toList(element.getAttributes())) {
- String name = attribute.getNodeName();
- if (allowedAttributes.contains(name)) {
- if (URI_ATTRIBUTES.contains(name)) {
- try {
- Uri uri = Uri.parse(attribute.getNodeValue());
- String scheme = uri.getScheme();
- if (!isAllowedScheme(scheme)) {
+ /**
+ * Utiliity class to sanitize HTML nodes recursively.
+ */
+ class NodeSanitizer {
+
+ private final LinkRewriter cssRewriter;
+ private final LinkRewriter imageRewriter;
+ private final Uri context;
+
+ NodeSanitizer(Gadget gadget) {
+ this.context = gadget.getSpec().getUrl();
+ Integer expires = rewriterFeatureFactory.getDefault().getExpires();
+ ContentRewriterFeature rewriterFeature =
+ rewriterFeatureFactory.createRewriteAllFeature(expires == null ? -1 : expires);
+
+ cssRewriter = new SanitizingProxyingLinkRewriter(gadget.getSpec().getUrl(),
+ rewriterFeature, proxyBaseNoGadget, "text/css");
+ imageRewriter = new SanitizingProxyingLinkRewriter(gadget.getSpec().getUrl(),
+ rewriterFeature, proxyBaseNoGadget, "image/*");
+ }
+
+ private void sanitize(Node node) {
+ switch (node.getNodeType()) {
+ case Node.CDATA_SECTION_NODE:
+ case Node.TEXT_NODE:
+ case Node.ENTITY_REFERENCE_NODE:
+ break;
+ case Node.ELEMENT_NODE:
+ case Node.DOCUMENT_NODE:
+ Element element = (Element) node;
+ if (allowedTags.contains(element.getTagName().toLowerCase())) {
+ // TODO - Add special case for stylesheet LINK nodes.
+ // Special case handling for style nodes
+ if (element.getTagName().equalsIgnoreCase("style")) {
+ cssSanitizer.sanitize(element, context, cssRewriter);
+ }
+ filterAttributes(element);
+ for (Node child : toList(node.getChildNodes())) {
+ sanitize(child);
+ }
+ } else {
+ node.getParentNode().removeChild(node);
+ }
+ break;
+ case Node.COMMENT_NODE:
+ default:
+ // Must remove all comments to avoid conditional comment evaluation.
+ // There might be other, unknown types as well. Don't trust them.
+ node.getParentNode().removeChild(node);
+ break;
+ }
+ }
+
+ private void filterAttributes(Element element) {
+ Set<String> rewriteImageAttrs = PROXY_IMAGE_ATTRIBUTES.get(element.getNodeName().toLowerCase());
+ for (Attr attribute : toList(element.getAttributes())) {
+ String name = attribute.getNodeName().toLowerCase();
+ if (allowedAttributes.contains(name)) {
+ if (URI_ATTRIBUTES.contains(name)) {
+ try {
+ Uri uri = Uri.parse(attribute.getNodeValue());
+ String scheme = uri.getScheme();
+ if (!isAllowedScheme(scheme)) {
+ element.removeAttributeNode(attribute);
+ } else if (rewriteImageAttrs != null && rewriteImageAttrs.contains(name)) {
+ // Force rewrite the src of the image through the proxy. This is necessary
+ // because IE will run arbitrary script in files referenced from src
+ attribute.setValue(imageRewriter.rewrite(attribute.getNodeValue(), context));
+ }
+ } catch (IllegalArgumentException e) {
+ // Not a valid URI.
element.removeAttributeNode(attribute);
}
- } catch (IllegalArgumentException e) {
- // Not a valid URI.
- element.removeAttributeNode(attribute);
}
+ } else {
+ element.removeAttributeNode(attribute);
}
- } else {
- element.removeAttributeNode(attribute);
}
}
}
+
/** Convert a NamedNodeMap to a list for easy and safe operations */
private static List<Attr> toList(NamedNodeMap nodes) {
List<Attr> list = new ArrayList<Attr>(nodes.getLength());
@@ -156,6 +307,35 @@
return scheme == null || scheme.equals("http") || scheme.equals("https");
}
+ /**
+ * Forcible rewrite the link through the proxy and force sanitization with
+ * an expected mime type
+ */
+ static class SanitizingProxyingLinkRewriter extends ProxyingLinkRewriter {
+
+ private final String expectedMime;
+
+ SanitizingProxyingLinkRewriter(Uri gadgetUri, ContentRewriterFeature rewriterFeature,
+ String prefix, String expectedMime) {
+ super(gadgetUri, rewriterFeature, prefix);
+ this.expectedMime = expectedMime;
+ }
+
+ @Override
+ public String rewrite(String link, Uri context) {
+ try {
+ Uri.parse(link);
+ } catch (RuntimeException re) {
+ // Any failure in parse
+ return "about:blank";
+ }
+ String rewritten = super.rewrite(link, context);
+ rewritten += "&" + ProxyBase.SANITIZE_CONTENT_PARAM + "=1";
+ rewritten += "&" + ProxyBase.REWRITE_MIME_TYPE_PARAM + "=" + expectedMime;
+ return rewritten;
+ }
+ }
+
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.PARAMETER)
@BindingAnnotation
Modified: incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/CSSContentRewriter.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/CSSContentRewriter.java?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/CSSContentRewriter.java (original)
+++ incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/CSSContentRewriter.java Thu Feb 5 01:41:58 2009
@@ -17,27 +17,48 @@
*/
package org.apache.shindig.gadgets.rewrite;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.shindig.common.uri.Uri;
import org.apache.shindig.gadgets.Gadget;
+import org.apache.shindig.gadgets.GadgetException;
import org.apache.shindig.gadgets.http.HttpRequest;
import org.apache.shindig.gadgets.http.HttpResponse;
+import org.apache.shindig.gadgets.parse.caja.CajaCssParser;
+import com.google.caja.parser.AbstractParseTreeNode;
+import com.google.caja.parser.AncestorChain;
+import com.google.caja.parser.Visitor;
+import com.google.caja.parser.css.CssTree;
+import com.google.common.collect.Lists;
import com.google.inject.Inject;
import com.google.inject.name.Named;
+import org.w3c.dom.Element;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.List;
+
/**
- * Rewrite links to referenced content in stylsheets
+ * Rewrite links to referenced content in a stylesheet
*/
public class CSSContentRewriter implements ContentRewriter {
private final ContentRewriterFeatureFactory rewriterFeatureFactory;
private final String proxyBaseNoGadget;
+ private final CajaCssParser cssParser;
@Inject
public CSSContentRewriter(ContentRewriterFeatureFactory rewriterFeatureFactory,
- @Named("shindig.content-rewrite.proxy-url")String proxyBaseNoGadget) {
+ @Named("shindig.content-rewrite.proxy-url")String proxyBaseNoGadget,
+ CajaCssParser cssParser) {
this.rewriterFeatureFactory = rewriterFeatureFactory;
this.proxyBaseNoGadget = proxyBaseNoGadget;
+ this.cssParser = cssParser;
}
public RewriterResults rewrite(Gadget gadget, MutableContent content) {
@@ -45,17 +66,109 @@
return null;
}
- public RewriterResults rewrite(HttpRequest request, HttpResponse original, MutableContent content) {
+ public RewriterResults rewrite(HttpRequest request, HttpResponse original,
+ MutableContent content) {
if (!RewriterUtils.isCss(request, original)) {
- return null;
+ return null;
}
ContentRewriterFeature feature = rewriterFeatureFactory.get(request);
- content.setContent(CssRewriter.rewrite(content.getContent(), request.getUri(),
- createLinkRewriter(request.getGadget(), feature)));
+ String css = content.getContent();
+ StringWriter sw = new StringWriter((css.length() * 110) / 100);
+ rewrite(new StringReader(css), request.getUri(),
+ createLinkRewriter(request.getGadget(), feature), sw, false);
+ content.setContent(sw.toString());
return RewriterResults.cacheableIndefinitely();
}
+ /**
+ * Rewrite the given CSS content and optionally extract the import references.
+ * @param content CSS content
+ * @param source Uri of content
+ * @param rewriter Rewrite urls
+ * @param writer Output
+ * @param extractImports If true remove the import statements from the output and return their
+ * referenced URIs.
+ * @return Empty list of extracted import URIs.
+ */
+ public List<String> rewrite(Reader content, Uri source,
+ LinkRewriter rewriter, Writer writer, boolean extractImports) {
+ try {
+ CssTree.StyleSheet stylesheet = cssParser.parseDom(IOUtils.toString(content));
+ List<String> stringList = rewrite(stylesheet, source, rewriter, extractImports);
+ // Serialize the stylesheet
+ cssParser.serialize(stylesheet, writer);
+ return stringList;
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ } catch (GadgetException ge) {
+ throw new RuntimeException(ge);
+ }
+ }
+
+ /**
+ * Rewrite the CSS content in a style DOM node.
+ * @param styleNode Rewrite the CSS content of this node
+ * @param source Uri of content
+ * @param rewriter Rewrite urls
+ * @param extractImports If true remove the import statements from the output and return their
+ * referenced URIs.
+ * @return Empty list of extracted import URIs.
+ */
+ public List<String> rewrite(Element styleNode, Uri source,
+ LinkRewriter rewriter, boolean extractImports) {
+ try {
+ CssTree.StyleSheet stylesheet = cssParser.parseDom(styleNode.getTextContent());
+ List<String> imports = rewrite(stylesheet, source, rewriter, extractImports);
+ // Write the rewritten CSS back into the element
+ String content = cssParser.serialize(stylesheet);
+ if (StringUtils.isEmpty(content)) {
+ // Remove the owning node
+ styleNode.getParentNode().removeChild(styleNode);
+ } else {
+ styleNode.setTextContent(content);
+ }
+ return imports;
+ } catch (GadgetException ge) {
+ throw new RuntimeException(ge);
+ }
+ }
+
+ /**
+ * Rewrite the CSS DOM in place.
+ * @param styleSheet To rewrite
+ * @param source Uri of content
+ * @param rewriter Rewrite urls
+ * @param extractImports If true remove the import statements from the output and return their
+ * referenced URIs.
+ * @return Empty list of extracted import URIs.
+ */
+ public static List<String> rewrite(CssTree.StyleSheet styleSheet, final Uri source,
+ final LinkRewriter rewriter, final boolean extractImports) {
+ final List<String> imports = Lists.newArrayList();
+
+ styleSheet.acceptPostOrder(new Visitor() {
+ public boolean visit(AncestorChain<?> chain) {
+ if (extractImports && chain.node instanceof CssTree.Import) {
+ // Extract the import statment, add its URI to the returned list
+ // and remove it from the DOM.
+ CssTree.Import importNode = (CssTree.Import) chain.node;
+ imports.add(importNode.getUri().getValue());
+ ((AbstractParseTreeNode) chain.getParentNode()).removeChild(chain.node);
+ } else if (chain.node instanceof CssTree.UriLiteral) {
+ // If were not extracting imports then rewrite its link
+ if (!(chain.getParentNode() instanceof CssTree.Import && extractImports)) {
+ String rewritten = rewriter
+ .rewrite(((CssTree.UriLiteral) chain.node).getValue(), source);
+ ((CssTree.UriLiteral) chain.node).setValue(rewritten);
+ }
+ }
+ return true;
+ }}, null);
+
+ return imports;
+ }
+
protected LinkRewriter createLinkRewriter(Uri gadgetUri, ContentRewriterFeature feature) {
return new ProxyingLinkRewriter(gadgetUri, feature, proxyBaseNoGadget);
}
Modified: incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/ContentRewriterFeatureFactory.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/ContentRewriterFeatureFactory.java?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/ContentRewriterFeatureFactory.java (original)
+++ incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/ContentRewriterFeatureFactory.java Thu Feb 5 01:41:58 2009
@@ -24,12 +24,12 @@
import org.apache.shindig.gadgets.spec.GadgetSpec;
import com.google.common.collect.Sets;
-
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.google.inject.name.Named;
import java.net.URI;
+import java.util.Collections;
import java.util.Set;
/**
@@ -97,4 +97,13 @@
spec.setAttribute("content-rewriter", rewriterFeature);
return rewriterFeature;
}
+
+ /**
+ * Create a rewriter feature that allows all URIs to be rewritten.
+ */
+ public ContentRewriterFeature createRewriteAllFeature(int ttl) {
+ return new ContentRewriterFeature(null,
+ ".*", "", (ttl == -1) ? "HTTP" : Integer.toString(ttl),
+ Collections.<String>emptySet());
+ }
}
Modified: incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/HTMLContentRewriter.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/HTMLContentRewriter.java?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/HTMLContentRewriter.java (original)
+++ incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/HTMLContentRewriter.java Thu Feb 5 01:41:58 2009
@@ -17,6 +17,15 @@
*/
package org.apache.shindig.gadgets.rewrite;
+import org.apache.shindig.common.uri.Uri;
+import org.apache.shindig.common.util.Utf8UrlCoder;
+import org.apache.shindig.common.xml.DomUtil;
+import org.apache.shindig.gadgets.Gadget;
+import org.apache.shindig.gadgets.http.HttpRequest;
+import org.apache.shindig.gadgets.http.HttpResponse;
+import org.apache.shindig.gadgets.servlet.ProxyBase;
+import org.apache.shindig.gadgets.spec.View;
+
import com.google.common.base.Nullable;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableMap;
@@ -27,20 +36,10 @@
import com.google.inject.Inject;
import com.google.inject.name.Named;
-import org.apache.shindig.common.uri.Uri;
-import org.apache.shindig.common.util.Utf8UrlCoder;
-import org.apache.shindig.common.xml.DomUtil;
-import org.apache.shindig.gadgets.Gadget;
-import org.apache.shindig.gadgets.http.HttpRequest;
-import org.apache.shindig.gadgets.http.HttpResponse;
-import org.apache.shindig.gadgets.servlet.ProxyBase;
-import org.apache.shindig.gadgets.spec.View;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
-import java.io.StringReader;
-import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.LinkedHashSet;
@@ -69,14 +68,17 @@
private final ContentRewriterFeatureFactory rewriterFeatureFactory;
private final String proxyBaseNoGadget;
private final String concatBaseNoGadget;
+ private final CSSContentRewriter cssRewriter;
@Inject
public HTMLContentRewriter(ContentRewriterFeatureFactory rewriterFeatureFactory,
@Named("shindig.content-rewrite.proxy-url")String proxyBaseNoGadget,
- @Named("shindig.content-rewrite.concat-url")String concatBaseNoGadget) {
+ @Named("shindig.content-rewrite.concat-url")String concatBaseNoGadget,
+ CSSContentRewriter cssRewriter) {
this.rewriterFeatureFactory = rewriterFeatureFactory;
this.concatBaseNoGadget = concatBaseNoGadget;
this.proxyBaseNoGadget = proxyBaseNoGadget;
+ this.cssRewriter = cssRewriter;
}
public RewriterResults rewrite(HttpRequest request, HttpResponse original,
@@ -98,7 +100,7 @@
return rewriteImpl(feature, gadget.getSpec().getUrl(), contentBase, content);
}
- protected RewriterResults rewriteImpl(ContentRewriterFeature feature, Uri gadgetUri,
+ RewriterResults rewriteImpl(ContentRewriterFeature feature, Uri gadgetUri,
Uri contentBase, MutableContent content) {
if (!feature.isRewriteEnabled() || content.getDocument() == null) {
return null;
@@ -130,7 +132,7 @@
return RewriterResults.cacheableIndefinitely();
}
- protected boolean rewriteStyleTags(Element head, List<Element> elementList,
+ boolean rewriteStyleTags(Element head, List<Element> elementList,
ContentRewriterFeature feature, Uri gadgetUri, Uri contentBase) {
if (!feature.getIncludedTags().contains("style")) {
return false;
@@ -153,19 +155,9 @@
styleTag.getParentNode().removeChild(styleTag);
head.appendChild(styleTag);
}
- String styleText = styleTag.getTextContent();
- StringWriter sw = new StringWriter(styleText.length());
- List<String> extractedUrls = CssRewriter.rewrite(new StringReader(styleText),
- contentBase, linkRewriter, sw, true);
- styleText = sw.toString().trim();
- if (styleText.length() == 0 || (styleText.length() < 25 &&
- styleText.replace("<!--", "").replace("//-->", "").
- replace("-->", "").trim().length() == 0)) {
- styleTag.getParentNode().removeChild(styleTag);
- elementList.remove(styleTag);
- } else {
- styleTag.setTextContent(styleText);
- }
+
+ List<String> extractedUrls = cssRewriter.rewrite(
+ styleTag, contentBase, linkRewriter, true);
for (String extractedUrl : extractedUrls) {
// Add extracted urls as link elements to head
Element newLink = head.getOwnerDocument().createElement("link");
@@ -194,11 +186,11 @@
return mutated;
}
- protected LinkRewriter createLinkRewriter(Uri gadgetUri, ContentRewriterFeature feature) {
+ LinkRewriter createLinkRewriter(Uri gadgetUri, ContentRewriterFeature feature) {
return new ProxyingLinkRewriter(gadgetUri, feature, proxyBaseNoGadget);
}
- protected String getConcatBase(Uri gadgetUri, ContentRewriterFeature feature, String mimeType) {
+ String getConcatBase(Uri gadgetUri, ContentRewriterFeature feature, String mimeType) {
return concatBaseNoGadget +
ProxyBase.REWRITE_MIME_TYPE_PARAM +
'=' + mimeType +
@@ -206,7 +198,7 @@
"&fp=" + feature.getFingerprint() +'&';
}
- protected boolean rewriteJsTags(List<Element> elementList, ContentRewriterFeature feature,
+ boolean rewriteJsTags(List<Element> elementList, ContentRewriterFeature feature,
Uri gadgetUri, Uri contentBase) {
if (!feature.getIncludedTags().contains("script")) {
return false;
@@ -252,7 +244,7 @@
return mutated;
}
- protected boolean rewriteContentReferences(List<Element> elementList,
+ boolean rewriteContentReferences(List<Element> elementList,
ContentRewriterFeature feature, Uri gadgetUri, Uri contentBase) {
boolean mutated = false;
LinkRewriter rewriter = createLinkRewriter(gadgetUri, feature);
Modified: incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ProxyBase.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ProxyBase.java?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ProxyBase.java (original)
+++ incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ProxyBase.java Thu Feb 5 01:41:58 2009
@@ -44,6 +44,7 @@
// Public because of rewriter. Rewriter should be cleaned up.
public static final String REWRITE_MIME_TYPE_PARAM = "rewriteMime";
+ public static final String SANITIZE_CONTENT_PARAM = "sanitize";
/**
* Validates the given url.
Modified: incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ProxyHandler.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ProxyHandler.java?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ProxyHandler.java (original)
+++ incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ProxyHandler.java Thu Feb 5 01:41:58 2009
@@ -18,6 +18,7 @@
*/
package org.apache.shindig.gadgets.servlet;
+import org.apache.commons.io.IOUtils;
import org.apache.shindig.common.uri.Uri;
import org.apache.shindig.gadgets.GadgetException;
import org.apache.shindig.gadgets.LockedDomainService;
@@ -30,8 +31,6 @@
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import org.apache.commons.io.IOUtils;
-
import java.io.IOException;
import java.util.List;
import java.util.Map;
@@ -94,6 +93,8 @@
req.setIgnoreCache(getIgnoreCache(request));
+ req.setSanitizationRequested("1".equals(request.getParameter(SANITIZE_CONTENT_PARAM)));
+
// If the proxy request specifies a refresh param then we want to force the min TTL for
// the retrieved entry in the cache regardless of the headers on the content when it
// is fetched from the original source.
Added: incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/parse/caja/CajaCssParserTest.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/parse/caja/CajaCssParserTest.java?rev=740973&view=auto
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/parse/caja/CajaCssParserTest.java (added)
+++ incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/parse/caja/CajaCssParserTest.java Thu Feb 5 01:41:58 2009
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.apache.shindig.gadgets.parse.caja;
+
+import com.google.caja.parser.css.CssTree;
+
+import junit.framework.TestCase;
+
+import java.util.List;
+
+/**
+ * Basic CSS parse tests
+ */
+public class CajaCssParserTest extends TestCase {
+
+ private CajaCssParser cajaCssParser;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ cajaCssParser = new CajaCssParser();
+ }
+
+ public void testBasicCssParse() throws Exception {
+ String css = ".xyz { font : bold; } A { color : #7f7f7f }";
+ CssTree.StyleSheet styleSheet = cajaCssParser.parseDom(css);
+ List<CssTree.SimpleSelector> selectorList = CajaCssUtils.descendants(styleSheet,
+ CssTree.SimpleSelector.class);
+ assertEquals(2, selectorList.size());
+ assertEquals(CssTree.SimpleSelector.class, selectorList.get(0).getClass());
+ }
+}
Added: incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/parse/caja/CajaCssSanitizerTest.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/parse/caja/CajaCssSanitizerTest.java?rev=740973&view=auto
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/parse/caja/CajaCssSanitizerTest.java (added)
+++ incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/parse/caja/CajaCssSanitizerTest.java Thu Feb 5 01:41:58 2009
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.apache.shindig.gadgets.parse.caja;
+
+import org.apache.shindig.common.EasyMockTestCase;
+import org.apache.shindig.common.uri.Uri;
+import org.apache.shindig.gadgets.rewrite.LinkRewriter;
+import org.apache.shindig.gadgets.servlet.ProxyBase;
+
+import com.google.caja.parser.css.CssTree;
+
+/**
+ *
+ */
+public class CajaCssSanitizerTest extends EasyMockTestCase {
+
+ private CajaCssParser parser;
+ private CajaCssSanitizer sanitizer;
+ private final Uri DUMMY = Uri.parse("http://www.example.org/base");
+ private LinkRewriter fakeRewriter;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ parser = new CajaCssParser();
+ sanitizer = new CajaCssSanitizer(parser);
+ fakeRewriter = new LinkRewriter() {
+ public String rewrite(String link, Uri context) {
+ return link + "&" + ProxyBase.SANITIZE_CONTENT_PARAM + "=1";
+ }
+ };
+ }
+
+ public void testPreserveSafe() throws Exception {
+ String css = ".xyz { font: bold;} A { color: #7f7f7f}";
+ CssTree.StyleSheet styleSheet = parser.parseDom(css);
+ sanitizer.sanitize(styleSheet, DUMMY, fakeRewriter);
+ assertStyleEquals(css, styleSheet);
+ }
+
+ public void testSanitizeFunctionCall() throws Exception {
+ String css = ".xyz { font : iamevil(bold); }";
+ CssTree.StyleSheet styleSheet = parser.parseDom(css);
+ sanitizer.sanitize(styleSheet, DUMMY, fakeRewriter);
+ assertStyleEquals(".xyz {}", styleSheet);
+ }
+
+ public void testSanitizeUnsafeProperties() throws Exception {
+ String css = ".xyz { behavior: url('xyz.htc'); -moz-binding:url(\"http://ha.ckers.org/xssmoz.xml#xss\") }";
+ CssTree.StyleSheet styleSheet = parser.parseDom(css);
+ sanitizer.sanitize(styleSheet, DUMMY, fakeRewriter);
+ assertStyleEquals(".xyz {}", styleSheet);
+ }
+
+ public void testSanitizeScriptUrls() throws Exception {
+ String css = ".xyz { background: url('javascript:doevill'); background : url(vbscript:moreevil); }";
+ CssTree.StyleSheet styleSheet = parser.parseDom(css);
+ sanitizer.sanitize(styleSheet, DUMMY, fakeRewriter);
+ assertStyleEquals(".xyz {}", styleSheet);
+ }
+
+ public void assertStyleEquals(String expected, CssTree.StyleSheet styleSheet) throws Exception {
+ assertEquals(parser.serialize(parser.parseDom(expected)), parser.serialize(styleSheet));
+ }
+}
Modified: incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/render/SanitizedRenderingContentRewriterTest.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/render/SanitizedRenderingContentRewriterTest.java?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/render/SanitizedRenderingContentRewriterTest.java (original)
+++ incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/render/SanitizedRenderingContentRewriterTest.java Thu Feb 5 01:41:58 2009
@@ -18,15 +18,23 @@
*/
package org.apache.shindig.gadgets.render;
-import static org.junit.Assert.assertEquals;
-
+import org.apache.commons.io.IOUtils;
import org.apache.shindig.common.PropertiesModule;
+import org.apache.shindig.common.uri.Uri;
import org.apache.shindig.gadgets.Gadget;
import org.apache.shindig.gadgets.GadgetContext;
+import org.apache.shindig.gadgets.http.HttpRequest;
+import org.apache.shindig.gadgets.http.HttpResponse;
+import org.apache.shindig.gadgets.http.HttpResponseBuilder;
import org.apache.shindig.gadgets.parse.GadgetHtmlParser;
+import org.apache.shindig.gadgets.parse.caja.CajaCssParser;
+import org.apache.shindig.gadgets.parse.caja.CajaCssSanitizer;
import org.apache.shindig.gadgets.parse.nekohtml.NekoHtmlParser;
import org.apache.shindig.gadgets.rewrite.ContentRewriter;
+import org.apache.shindig.gadgets.rewrite.ContentRewriterFeatureFactory;
import org.apache.shindig.gadgets.rewrite.MutableContent;
+import org.apache.shindig.gadgets.servlet.ProxyBase;
+import org.apache.shindig.gadgets.spec.GadgetSpec;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
@@ -35,11 +43,14 @@
import com.google.inject.Injector;
import com.google.inject.Provider;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import org.junit.Before;
import org.junit.Test;
import org.w3c.dom.DOMImplementation;
import org.w3c.dom.bootstrap.DOMImplementationRegistry;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
@@ -49,10 +60,12 @@
private static final Set<String> DEFAULT_TAGS = ImmutableSet.of("html", "head", "body");
private static final Pattern BODY_REGEX = Pattern.compile(".*<body>(.*)</body>.*");
+ private static final Uri CONTENT_URI = Uri.parse("www.example.org/content");
+
private final GadgetContext sanitaryGadgetContext = new GadgetContext() {
@Override
public String getParameter(String name) {
- return "sanitize".equals(name) ? "1" : null;
+ return ProxyBase.SANITIZE_CONTENT_PARAM.equals(name) ? "1" : null;
}
};
@@ -60,10 +73,15 @@
private GadgetHtmlParser parser;
+ private Gadget gadget;
+
@Before
public void setUp() throws Exception {
Injector injector = Guice.createInjector(new TestParseModule(), new PropertiesModule());
parser = injector.getInstance(GadgetHtmlParser.class);
+ gadget = new Gadget().setContext(sanitaryGadgetContext);
+ gadget.setSpec(new GadgetSpec(Uri.parse("www.example.org/gadget.xml"),
+ "<Module><ModulePrefs title=''/><Content type='html'/></Module>"));
}
private String rewrite(Gadget gadget, String content, Set<String> tags, Set<String> attributes) {
@@ -79,6 +97,18 @@
return mc.getContent();
}
+ private String rewrite(HttpRequest request, HttpResponse response) {
+ request.setSanitizationRequested(true);
+ ContentRewriter rewriter = createRewriter(Collections.<String>emptySet(),
+ Collections.<String>emptySet());
+
+ MutableContent mc = new MutableContent(parser, response);
+ if (rewriter.rewrite(request, response, mc) == null) {
+ return null;
+ }
+ return mc.getContent();
+ }
+
private static Set<String> set(String... items) {
return Sets.newHashSet(items);
}
@@ -86,32 +116,112 @@
private ContentRewriter createRewriter(Set<String> tags, Set<String> attributes) {
Set<String> newTags = new HashSet<String>(tags);
newTags.addAll(DEFAULT_TAGS);
- return new SanitizedRenderingContentRewriter(newTags, attributes);
+ ContentRewriterFeatureFactory rewriterFeatureFactory =
+ new ContentRewriterFeatureFactory(null, ".*", "", "HTTP", "embed,img,script,link,style");
+ return new SanitizedRenderingContentRewriter(newTags, attributes, rewriterFeatureFactory,
+ "http://www.test.com/base", new CajaCssSanitizer(new CajaCssParser()));
}
@Test
- public void enforceTagWhiteList() {
+ public void enforceTagWhiteList() throws Exception {
String markup =
- "<p><style type=\"text/css\">styles</style>text <b>bold text</b></p>" +
+ "<p><style type=\"text/css\">A { font : bold }</style>text <b>bold text</b></p>" +
"<b>Bold text</b><i>Italic text<b>Bold text</b></i>";
String sanitized = "<p>text <b>bold text</b></p><b>Bold text</b>";
+ assertEquals(sanitized, rewrite(gadget, markup, set("p", "b"), set()));
+ }
+
+ @Test
+ public void enforceStyleSanitized() {
+ String markup =
+ "<p><style type=\"text/css\">A { font : bold; behavior : bad }</style>text <b>bold text</b></p>" +
+ "<b>Bold text</b><i>Italic text<b>Bold text</b></i>";
+ String sanitized = "<html><head></head><body><p><style>A {\n font: bold\n}</style>text " +
+ "<b>bold text</b></p><b>Bold text</b></body></html>";
+ assertEquals(sanitized, rewrite(gadget, markup, set("p", "b", "style"), set()));
+ }
- Gadget gadget = new Gadget().setContext(sanitaryGadgetContext);
+ @Test
+ public void enforceCssImportLinkRewritten() {
+ String markup =
+ "<style type=\"text/css\">@import url('www.evil.com/x.js');</style>";
+ String sanitized = "<style>@import url('http\\3A//www.test.com/basewww.example.org%2Fwww.evil.com%2Fx.js\\26gadget\\3Dwww.example.org%2Fgadget.xml\\26 fp\\3D 45508\\26sanitize\\3D 1\\26rewriteMime\\3Dtext/css');</style>";
+ assertEquals(sanitized, rewrite(gadget, markup, set("style"), set()));
+ }
- assertEquals(sanitized, rewrite(gadget, markup, set("p", "b"), set()));
+ @Test
+ public void enforceCssImportBadLinkStripped() {
+ String markup =
+ "<style type=\"text/css\">@import url('javascript:doevil()'); A { font : bold }</style>";
+ String sanitized = "<html><head></head><body><style>A {\n"
+ + " font: bold\n"
+ + "}</style></body></html>";
+ assertEquals(sanitized, rewrite(gadget, markup, set("style"), set()));
}
@Test
public void enforceAttributeWhiteList() {
String markup = "<p foo=\"bar\" bar=\"baz\">Paragraph</p>";
String sanitized = "<p bar=\"baz\">Paragraph</p>";
+ assertEquals(sanitized, rewrite(gadget, markup, set("p"), set("bar")));
+ }
- Gadget gadget = new Gadget().setContext(sanitaryGadgetContext);
+ @Test
+ public void enforceImageSrcProxied() {
+ String markup = "<img src='http://www.evil.com/x.js'>Evil happens</img>";
+ String sanitized = "<img src=\"http://www.test.com/basehttp%3A%2F%2Fwww.evil.com%2Fx.js&gadget=www.example.org%2Fgadget.xml&fp=45508&sanitize=1&rewriteMime=image/*\">Evil happens";
+ assertEquals(sanitized, rewrite(gadget, markup, set("img"), set("src")));
+ }
- assertEquals(sanitized, rewrite(gadget, markup, set("p"), set("bar")));
+ @Test
+ public void enforceBadImageUrlStripped() {
+ String markup = "<img src='java\\ script:evil()'>Evil happens</img>";
+ String sanitized = "<img>Evil happens";
+ assertEquals(sanitized, rewrite(gadget, markup, set("img"), set("src")));
+ }
+
+ @Test
+ public void enforceInvalidProxedCssRejected() {
+ HttpRequest req = new HttpRequest(CONTENT_URI);
+ req.setRewriteMimeType("text/css");
+ HttpResponse response = new HttpResponseBuilder().setResponseString("doEvil()").create();
+ String sanitized = "";
+ assertEquals(sanitized, rewrite(req, response));
+ }
+
+ @Test
+ public void enforceValidProxedCssAccepted() {
+ HttpRequest req = new HttpRequest(CONTENT_URI);
+ req.setRewriteMimeType("text/css");
+ HttpResponse response = new HttpResponseBuilder().setResponseString(
+ "@import url('http://www.evil.com/more.css'); A { font : BOLD }").create();
+ String sanitized = "@import url('http\\3A//www.test.com/basehttp%3A%2F%2Fwww.evil.com%2Fmore.css\\26 fp\\3D 45508\\26sanitize\\3D 1\\26rewriteMime\\3Dtext/css');\n"
+ + "A {\n"
+ + " font: BOLD\n"
+ + "}";
+ assertEquals(sanitized, rewrite(req, response));
+ }
+
+ @Test
+ public void enforceInvalidProxedImageRejected() {
+ HttpRequest req = new HttpRequest(CONTENT_URI);
+ req.setRewriteMimeType("image/*");
+ HttpResponse response = new HttpResponseBuilder().setResponse("NOTIMAGE".getBytes()).create();
+ String sanitized = "";
+ assertEquals(sanitized, rewrite(req, response));
+ }
+
+ @Test
+ public void validProxiedImageAccepted() throws Exception {
+ HttpRequest req = new HttpRequest(CONTENT_URI);
+ req.setRewriteMimeType("image/*");
+ HttpResponse response = new HttpResponseBuilder().setResponse(
+ IOUtils.toByteArray(getClass().getClassLoader().getResourceAsStream(
+ "org/apache/shindig/gadgets/rewrite/image/inefficient.png"))).create();
+ assertNull(rewrite(req, response));
}
@Test
@@ -149,26 +259,19 @@
"href=\"//example.org/valid-href\" " +
"src=\"//example.org/valid-src\"></element>";
- Gadget gadget = new Gadget().setContext(sanitaryGadgetContext);
-
assertEquals(sanitized, rewrite(gadget, markup, set("element"), set("href", "src")));
}
@Test
public void allCommentsStripped() {
String markup = "<b>Hello, world</b><!--<b>evil</b>-->";
-
- Gadget gadget = new Gadget().setContext(sanitaryGadgetContext);
-
assertEquals("<b>Hello, world</b>", rewrite(gadget, markup, set("b"), set()));
}
@Test
public void doesNothingWhenNotSanitized() {
String markup = "<script src=\"http://evil.org/evil\"></script> <b>hello</b>";
-
- Gadget gadget = new Gadget().setContext(unsanitaryGadgetContext);
-
+ gadget.setContext(unsanitaryGadgetContext);
assertEquals(markup, rewrite(gadget, markup, set("b"), set()));
}
Modified: incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/CSSContentRewriterTest.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/CSSContentRewriterTest.java?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/CSSContentRewriterTest.java (original)
+++ incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/CSSContentRewriterTest.java Thu Feb 5 01:41:58 2009
@@ -17,19 +17,27 @@
*/
package org.apache.shindig.gadgets.rewrite;
+import org.apache.commons.io.IOUtils;
import org.apache.shindig.common.uri.Uri;
import org.apache.shindig.gadgets.http.HttpRequest;
import org.apache.shindig.gadgets.http.HttpResponse;
import org.apache.shindig.gadgets.http.HttpResponseBuilder;
+import org.apache.shindig.gadgets.parse.caja.CajaCssParser;
+
+import com.google.common.collect.Lists;
-import org.apache.commons.io.IOUtils;
import org.easymock.EasyMock;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.util.List;
+
/**
*
*/
public class CSSContentRewriterTest extends BaseRewriterTestCase {
private CSSContentRewriter rewriter;
+ private Uri dummyUri;
@Override
protected void setUp() throws Exception {
@@ -38,7 +46,8 @@
rewriterFeatureFactory.get(createSpecWithRewrite(".*", ".*exclude.*", "HTTP",
HTMLContentRewriter.TAGS));
ContentRewriterFeatureFactory factory = mockContentRewriterFeatureFactory(overrideFeature);
- rewriter = new CSSContentRewriter(factory, DEFAULT_PROXY_BASE);
+ rewriter = new CSSContentRewriter(factory, DEFAULT_PROXY_BASE, new CajaCssParser());
+ dummyUri = Uri.parse("http://www.w3c.org");
}
public void testCssBasic() throws Exception {
@@ -68,4 +77,63 @@
assertNull(rewriter.rewrite(req, fakeResponse, mc));
verify();
}
+
+ private void validateRewritten(String content, Uri base,
+ LinkRewriter linkRewriter, String expected) {
+ MutableContent mc = new MutableContent(null, content);
+ HttpRequest request = new HttpRequest(base);
+ rewriter.rewrite(request,
+ new HttpResponseBuilder().setHeader("Content-Type", "text/css").create(), mc);
+ assertEquals(expected, mc.getContent());
+ }
+
+ private void validateRewritten(String content, String expected) {
+ validateRewritten(content, dummyUri, defaultLinkRewriter, expected);
+ }
+
+ public void testUrlDeclarationRewrite() {
+ String original =
+ "div {list-style-image:url('http://a.b.com/bullet.gif');list-style-position:outside;margin:5px;padding:0}\n" +
+ ".someid {background-image:url(http://a.b.com/bigimg.png);float:right;width:165px;height:23px;margin-top:4px;margin-left:5px}";
+ String rewritten =
+ "div {\n" +
+ " list-style-image: url('http\\3A//www.test.com/dir/proxy?url\\3Dhttp%3A%2F%2Fa.b.com%2Fbullet.gif\\26 fp\\3D 1150739864');\n" +
+ " list-style-position: outside;\n" +
+ " margin: 5px;\n" +
+ " padding: 0\n" +
+ "}\n" +
+ ".someid {\n" +
+ " background-image: url('http\\3A//www.test.com/dir/proxy?url\\3Dhttp%3A%2F%2Fa.b.com%2Fbigimg.png\\26 fp\\3D 1150739864');\n" +
+ " float: right;\n" +
+ " width: 165px;\n" +
+ " height: 23px;\n" +
+ " margin-top: 4px;\n" +
+ " margin-left: 5px\n" +
+ "}";
+ validateRewritten(original, rewritten);
+ }
+
+ public void testExtractImports() {
+ String original = " @import url(www.example.org/some.css);\n" +
+ "@import url('www.example.org/someother.css');\n" +
+ "@import url(\"www.example.org/another.css\");\n" +
+ " div { color: blue; }\n" +
+ " p { color: black; }\n" +
+ " span { color: red; }";
+ String expected = "div {\n"
+ + " color: blue;\n"
+ + "}\n"
+ + "p {\n"
+ + " color: black;\n"
+ + "}\n"
+ + "span {\n"
+ + " color: red;\n"
+ + "}";
+ StringWriter sw = new StringWriter();
+ List<String> stringList = rewriter
+ .rewrite(new StringReader(original), dummyUri, defaultLinkRewriter, sw, true);
+ assertEquals(expected, sw.toString());
+ assertEquals(stringList, Lists.newArrayList("www.example.org/some.css",
+ "www.example.org/someother.css", "www.example.org/another.css"));
+ }
}
Modified: incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/HTMLContentRewriterTest.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/HTMLContentRewriterTest.java?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/HTMLContentRewriterTest.java (original)
+++ incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/HTMLContentRewriterTest.java Thu Feb 5 01:41:58 2009
@@ -17,9 +17,10 @@
*/
package org.apache.shindig.gadgets.rewrite;
+import org.apache.commons.io.IOUtils;
import org.apache.shindig.gadgets.http.HttpRequest;
+import org.apache.shindig.gadgets.parse.caja.CajaCssParser;
-import org.apache.commons.io.IOUtils;
import org.easymock.EasyMock;
import org.w3c.dom.Document;
@@ -36,8 +37,8 @@
rewriterFeatureFactory.get(createSpecWithRewrite(".*", ".*exclude.*", "HTTP",
HTMLContentRewriter.TAGS));
ContentRewriterFeatureFactory factory = mockContentRewriterFeatureFactory(overrideFeature);
- rewriter = new HTMLContentRewriter(factory, DEFAULT_PROXY_BASE, DEFAULT_CONCAT_BASE);
-
+ rewriter = new HTMLContentRewriter(factory, DEFAULT_PROXY_BASE, DEFAULT_CONCAT_BASE,
+ new CSSContentRewriter(factory, DEFAULT_PROXY_BASE, new CajaCssParser()));
}
public void testScriptsBasic() throws Exception {
@@ -153,8 +154,7 @@
assertEquals(1, wrapper.getNodeList("//style").getLength());
// All @imports are stripped
- assertEquals("div { color : black; }",
- wrapper.getValue("//style[1]"));
+ assertEquals("div {\n color: black;\n}", wrapper.getValue("//style[1]"));
}
public void testNoRewriteUnknownMimeType() {
Modified: incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/servlet/MakeRequestHandlerTest.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/servlet/MakeRequestHandlerTest.java?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/servlet/MakeRequestHandlerTest.java (original)
+++ incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/servlet/MakeRequestHandlerTest.java Thu Feb 5 01:41:58 2009
@@ -18,9 +18,6 @@
*/
package org.apache.shindig.gadgets.servlet;
-import static junitx.framework.StringAssert.assertStartsWith;
-import static org.easymock.EasyMock.expect;
-
import org.apache.shindig.auth.AuthInfo;
import org.apache.shindig.auth.SecurityToken;
import org.apache.shindig.common.testing.FakeGadgetToken;
@@ -30,18 +27,23 @@
import org.apache.shindig.gadgets.http.HttpRequest;
import org.apache.shindig.gadgets.http.HttpResponse;
import org.apache.shindig.gadgets.http.HttpResponseBuilder;
-import org.easymock.IAnswer;
+import com.google.common.collect.Lists;
+
+import static org.easymock.EasyMock.expect;
+import org.easymock.IAnswer;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.junit.Test;
-import java.util.*;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.List;
import javax.servlet.http.HttpServletRequest;
-import com.google.common.collect.Lists;
+import static junitx.framework.StringAssert.assertStartsWith;
/**
* Tests for MakeRequestHandler.
@@ -232,8 +234,8 @@
private void expectParameters(HttpServletRequest request, String... params) {
final List<String> v = Lists.newArrayList(params);
- expect(request.getParameterNames()).andStubAnswer(new IAnswer<Enumeration<String>>() {
- public Enumeration<String> answer() throws Throwable{
+ expect(request.getParameterNames()).andStubAnswer(new IAnswer<Enumeration>() {
+ public Enumeration<String> answer() throws Throwable {
return Collections.enumeration(v);
}
});
Modified: incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-fragment-expected.html
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-fragment-expected.html?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-fragment-expected.html (original)
+++ incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-fragment-expected.html Thu Feb 5 01:41:58 2009
@@ -1,2 +1,2 @@
<html><head></head><body><script>document.write("dont add to head or else")</script>
-<style type="text/css"> can go in head</style></body></html>
\ No newline at end of file
+<style type="text/css"> A { font : bold; }</style></body></html>
\ No newline at end of file
Modified: incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-fragment.html
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-fragment.html?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-fragment.html (original)
+++ incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-fragment.html Thu Feb 5 01:41:58 2009
@@ -1,2 +1,2 @@
<script>document.write("dont add to head or else")</script>
-<style type="text/css"> can go in head</style>
\ No newline at end of file
+<style type="text/css"> A { font : bold; }</style>
\ No newline at end of file
Modified: incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-headnobody-expected.html
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-headnobody-expected.html?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-headnobody-expected.html (original)
+++ incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-headnobody-expected.html Thu Feb 5 01:41:58 2009
@@ -2,4 +2,4 @@
<!-- A head tag but no body tag is not good -->
</head><body>
<script>document.write("dont add to head or else")</script>
-<style type="text/css"> can go in head</style></body></html>
\ No newline at end of file
+<style type="text/css"> A { font : bold; } </style></body></html>
\ No newline at end of file
Modified: incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-headnobody.html
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-headnobody.html?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-headnobody.html (original)
+++ incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/parse/nekohtml/test-headnobody.html Thu Feb 5 01:41:58 2009
@@ -2,4 +2,4 @@
<!-- A head tag but no body tag is not good -->
</head>
<script>document.write("dont add to head or else")</script>
-<style type="text/css"> can go in head</style>
\ No newline at end of file
+<style type="text/css"> A { font : bold; } </style>
\ No newline at end of file
Modified: incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/rewrite/rewritebasic-expected.css
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/rewrite/rewritebasic-expected.css?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/rewrite/rewritebasic-expected.css (original)
+++ incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/rewrite/rewritebasic-expected.css Thu Feb 5 01:41:58 2009
@@ -1,12 +1,6 @@
-
-// Rewrite various url forms in input statements, absolute and host/path relative.
-// Test exclusions
-@import url("http://www.test.com/dir/proxy?url=http%3A%2F%2Fwww.example.org%2Fother1.css&gadget=http%3A%2F%2Fwww.example.org%2Fdir%2Fg.xml&fp=1150739864");
+@import url('http\3A//www.test.com/dir/proxy?url\3Dhttp%3A%2F%2Fwww.example.org%2Fother1.css\26gadget\3Dhttp%3A%2F%2Fwww.example.org%2Fdir%2Fg.xml\26 fp\3D 1150739864');
+@import url('http\3A//www.test.com/dir/proxy?url\3Dhttp%3A%2F%2Fwww.example.org%2Fpath%2Frelative%2Fother2.css\26gadget\3Dhttp%3A%2F%2Fwww.example.org%2Fdir%2Fg.xml\26 fp\3D 1150739864');
+@import url('http\3A//www.example.org/hostrelative/excluded/other1.css');
DiV {
font: arial;
-}
-
-@import url("http://www.test.com/dir/proxy?url=http%3A%2F%2Fwww.example.org%2Fpath%2Frelative%2Fother2.css&gadget=http%3A%2F%2Fwww.example.org%2Fdir%2Fg.xml&fp=1150739864");
-@import url("http://www.example.org/hostrelative/excluded/other1.css");
-
-// A comment?
\ No newline at end of file
+}
\ No newline at end of file
Modified: incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/rewrite/rewritebasic.css
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/rewrite/rewritebasic.css?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/rewrite/rewritebasic.css (original)
+++ incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/rewrite/rewritebasic.css Thu Feb 5 01:41:58 2009
@@ -1,12 +1,12 @@
-// Rewrite various url forms in input statements, absolute and host/path relative.
-// Test exclusions
+/* Rewrite various url forms in input statements, absolute and host/path relative.
+ Test exclusions */
@import url(http://www.example.org/other1.css);
+@import url("relative/other2.css");
+@import url('/hostrelative/excluded/other1.css');
+
DiV {
font: arial;
}
-@import url("relative/other2.css");
-@import url('/hostrelative/excluded/other1.css');
-
-// A comment?
\ No newline at end of file
+/* A comment? */
\ No newline at end of file
Modified: incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/rewrite/rewritestylebasic.html
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/rewrite/rewritestylebasic.html?rev=740973&r1=740972&r2=740973&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/rewrite/rewritestylebasic.html (original)
+++ incubator/shindig/trunk/java/gadgets/src/test/resources/org/apache/shindig/gadgets/rewrite/rewritestylebasic.html Thu Feb 5 01:41:58 2009
@@ -12,8 +12,8 @@
</style>
<div>somecontent</div>
<style>
- div { color : black; }
@import url("/importedstyle4.css");
+ div { color : black; }
</style>
</body>
-</html>
\ No newline at end of file
+</html>