You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jspwiki.apache.org by gm...@apache.org on 2013/06/23 17:15:52 UTC
svn commit: r1495828 [2/3] - in /incubator/jspwiki/trunk: ./ jspwiki-war/
jspwiki-war/src/main/java/org/apache/wiki/
jspwiki-war/src/main/webapp/scripts/
jspwiki-war/src/main/webapp/templates/default/
Modified: incubator/jspwiki/trunk/jspwiki-war/src/main/webapp/scripts/prettify.js
URL: http://svn.apache.org/viewvc/incubator/jspwiki/trunk/jspwiki-war/src/main/webapp/scripts/prettify.js?rev=1495828&r1=1495827&r2=1495828&view=diff
==============================================================================
--- incubator/jspwiki/trunk/jspwiki-war/src/main/webapp/scripts/prettify.js (original)
+++ incubator/jspwiki/trunk/jspwiki-war/src/main/webapp/scripts/prettify.js Sun Jun 23 15:15:52 2013
@@ -17,1465 +17,1640 @@
* @fileoverview
* some functions for browser-side pretty printing of code contained in html.
*
- * The lexer should work on a number of languages including C and friends,
- * Java, Python, Bash, SQL, HTML, XML, CSS, Javascript, and Makefiles.
- * It works passably on Ruby, PHP and Awk and a decent subset of Perl, but,
- * because of commenting conventions, doesn't work on Smalltalk, Lisp-like, or
- * CAML-like languages.
- *
- * If there's a language not mentioned here, then I don't know it, and don't
- * know whether it works. If it has a C-like, Bash-like, or XML-like syntax
- * then it should work passably.
- *
- * Usage:
- * 1) include this source file in an html page via
- * <script type="text/javascript" src="/path/to/prettify.js"></script>
- * 2) define style rules. See the example page for examples.
- * 3) mark the <pre> and <code> tags in your source with class=prettyprint.
- * You can also use the (html deprecated) <xmp> tag, but the pretty printer
- * needs to do more substantial DOM manipulations to support that, so some
- * css styles may not be preserved.
+ * <p>
+ * For a fairly comprehensive set of languages see the
+ * <a href="http://google-code-prettify.googlecode.com/svn/trunk/README.html#langs">README</a>
+ * file that came with this source. At a minimum, the lexer should work on a
+ * number of languages including C and friends, Java, Python, Bash, SQL, HTML,
+ * XML, CSS, Javascript, and Makefiles. It works passably on Ruby, PHP and Awk
+ * and a subset of Perl, but, because of commenting conventions, doesn't work on
+ * Smalltalk, Lisp-like, or CAML-like languages without an explicit lang class.
+ * <p>
+ * Usage: <ol>
+ * <li> include this source file in an html page via
+ * {@code <script type="text/javascript" src="/path/to/prettify.js"></script>}
+ * <li> define style rules. See the example page for examples.
+ * <li> mark the {@code <pre>} and {@code <code>} tags in your source with
+ * {@code class=prettyprint.}
+ * You can also use the (html deprecated) {@code <xmp>} tag, but the pretty
+ * printer needs to do more substantial DOM manipulations to support that, so
+ * some css styles may not be preserved.
+ * </ol>
* That's it. I wanted to keep the API as simple as possible, so there's no
- * need to specify which language the code is in.
- *
- * Change log:
+ * need to specify which language the code is in, but if you wish, you can add
+ * another class to the {@code <pre>} or {@code <code>} element to specify the
+ * language, as in {@code <pre class="prettyprint lang-java">}. Any class that
+ * starts with "lang-" followed by a file extension, specifies the file type.
+ * See the "lang-*.js" files in this directory for code that implements
+ * per-language file handlers.
+ * <p>
+ * Change log:<br>
* cbeust, 2006/08/22
+ * <blockquote>
* Java annotations (start with "@") are now captured as literals ("lit")
+ * </blockquote>
+ * @requires console
*/
-var PR_keywords = {};
-/** initialize the keyword list for our target languages. */
-(function () {
- var CPP_KEYWORDS = "abstract bool break case catch char class const " +
- "const_cast continue default delete deprecated dllexport dllimport do " +
- "double dynamic_cast else enum explicit extern false float for friend " +
- "goto if inline int long mutable naked namespace new noinline noreturn " +
- "nothrow novtable operator private property protected public register " +
- "reinterpret_cast return selectany short signed sizeof static " +
- "static_cast struct switch template this thread throw true try typedef " +
- "typeid typename union unsigned using declaration, directive uuid " +
- "virtual void volatile while typeof";
- var CSHARP_KEYWORDS = "as base by byte checked decimal delegate descending " +
- "event finally fixed foreach from group implicit in interface internal " +
- "into is lock null object out override orderby params readonly ref sbyte " +
- "sealed stackalloc string select uint ulong unchecked unsafe ushort var";
- var JAVA_KEYWORDS = "package synchronized boolean implements import throws " +
- "instanceof transient extends final strictfp native super";
- var JSCRIPT_KEYWORDS = "debugger export function with NaN Infinity";
- var PERL_KEYWORDS = "require sub unless until use elsif BEGIN END";
- var PYTHON_KEYWORDS = "and assert def del elif except exec global lambda " +
- "not or pass print raise yield False True None";
- var RUBY_KEYWORDS = "then end begin rescue ensure module when undef next " +
- "redo retry alias defined";
- var SH_KEYWORDS = "done fi";
-
- var KEYWORDS = [CPP_KEYWORDS, CSHARP_KEYWORDS, JAVA_KEYWORDS,
- JSCRIPT_KEYWORDS, PERL_KEYWORDS, PYTHON_KEYWORDS,
- RUBY_KEYWORDS, SH_KEYWORDS];
- for (var k = 0; k < KEYWORDS.length; k++) {
- var kw = KEYWORDS[k].split(' ');
- for (var i = 0; i < kw.length; i++) {
- if (kw[i]) { PR_keywords[kw[i]] = true; }
- }
- }
-}).call(this);
+// JSLint declarations
-// token style names. correspond to css classes
-/** token style for a string literal */
-var PR_STRING = 'str';
-/** token style for a keyword */
-var PR_KEYWORD = 'kwd';
-/** token style for a comment */
-var PR_COMMENT = 'com';
-/** token style for a type */
-var PR_TYPE = 'typ';
-/** token style for a literal value. e.g. 1, null, true. */
-var PR_LITERAL = 'lit';
-/** token style for a punctuation string. */
-var PR_PUNCTUATION = 'pun';
-/** token style for a punctuation string. */
-var PR_PLAIN = 'pln';
-
-/** token style for an sgml tag. */
-var PR_TAG = 'tag';
-/** token style for a markup declaration such as a DOCTYPE. */
-var PR_DECLARATION = 'dec';
-/** token style for embedded source. */
-var PR_SOURCE = 'src';
-/** token style for an sgml attribute name. */
-var PR_ATTRIB_NAME = 'atn';
-/** token style for an sgml attribute value. */
-var PR_ATTRIB_VALUE = 'atv';
-
-/** the number of characters between tab columns */
-var PR_TAB_WIDTH = 8;
-
-/** the position of the end of a token during. A division of a string into
- * n tokens can be represented as a series n - 1 token ends, as long as
- * runs of whitespace warrant their own token.
- * @private
- */
-function PR_TokenEnd(end, style) {
- if (undefined === style) { throw new Error('BAD'); }
- if ('number' != typeof(end)) { throw new Error('BAD'); }
- this.end = end;
- this.style = style;
-}
-PR_TokenEnd.prototype.toString = function () {
- return '[PR_TokenEnd ' + this.end +
- (this.style ? ':' + this.style : '') + ']';
-};
-
-
-/** a chunk of text with a style. These are used to represent both the output
- * from the lexing functions as well as intermediate results.
- * @constructor
- * @param token the token text
- * @param style one of the token styles defined in designdoc-template, or null
- * for a styleless token, such as an embedded html tag.
- * @private
- */
-function PR_Token(token, style) {
- if (undefined === style) { throw new Error('BAD'); }
- this.token = token;
- this.style = style;
-}
-
-PR_Token.prototype.toString = function () {
- return '[PR_Token ' + this.token + (this.style ? ':' + this.style : '') + ']';
-};
-
-
-/** a helper class that decodes common html entities used to escape special
- * characters in source code.
- * @constructor
- * @private
- */
-function PR_DecodeHelper() {
- this.next = 0;
- this.ch = '\0';
-}
-
-var PR_NAMED_ENTITIES = {
- 'lt': '<',
- 'gt': '>',
- 'quot': '"',
- 'apos': "'",
- 'amp': '&' // reencoding requires that & always be decoded properly
-};
-
-PR_DecodeHelper.prototype.decode = function (s, i) {
- var next = i + 1;
- var ch = s.charAt(i);
- if ('&' === ch) {
- var semi = s.indexOf(';', next);
- if (semi >= 0 && semi < next + 4) {
- var entityName = s.substring(next, semi);
- var decoded = null;
- if (entityName.charAt(0) === '#') { // check for numeric entity
- var ch1 = entityName.charAt(1);
- var charCode;
- if (ch1 === 'x' || ch1 === 'X') { // like  
- charCode = parseInt(entityName.substring(2), 16);
- } else { // like  
- charCode = parseInt(entityName.substring(1), 10);
- }
- if (!isNaN(charCode)) {
- decoded = String.fromCharCode(charCode);
- }
- }
- if (!decoded) {
- decoded = PR_NAMED_ENTITIES[entityName.toLowerCase()];
- }
- if (decoded) {
- ch = decoded;
- next = semi + 1;
- } else { // skip over unrecognized entity
- next = i + 1;
- ch = '\0';
- }
- }
- }
- this.next = next;
- this.ch = ch;
- return this.ch;
-};
-
-
-// some string utilities
-function PR_isWordChar(ch) {
- return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z');
-}
-
-function PR_isIdentifierStart(ch) {
- return PR_isWordChar(ch) || ch == '_' || ch == '$' || ch == '@';
-}
-
-function PR_isIdentifierPart(ch) {
- return PR_isIdentifierStart(ch) || PR_isDigitChar(ch);
-}
-
-function PR_isSpaceChar(ch) {
- return "\t \r\n".indexOf(ch) >= 0;
-}
-
-function PR_isDigitChar(ch) {
- return ch >= '0' && ch <= '9';
-}
-
-function PR_trim(s) {
- var i = 0, j = s.length - 1;
- while (i <= j && PR_isSpaceChar(s.charAt(i))) { ++i; }
- while (j > i && PR_isSpaceChar(s.charAt(j))) { --j; }
- return s.substring(i, j + 1);
-}
-
-function PR_startsWith(s, prefix) {
- return s.length >= prefix.length && prefix == s.substring(0, prefix.length);
-}
-
-function PR_endsWith(s, suffix) {
- return s.length >= suffix.length &&
- suffix == s.substring(s.length - suffix.length, s.length);
-}
-
-/** true iff prefix matches the first prefix characters in chars[0:len].
- * @private
- */
-function PR_prefixMatch(chars, len, prefix) {
- if (len < prefix.length) { return false; }
- for (var i = 0, n = prefix.length; i < n; ++i) {
- if (prefix.charAt(i) != chars[i]) { return false; }
- }
- return true;
-}
+/*global console, document, navigator, setTimeout, window, define */
-/** like textToHtml but escapes double quotes to be attribute safe. */
-function PR_attribToHtml(str) {
- return str.replace(/&/g, '&')
- .replace(/</g, '<')
- .replace(/>/g, '>')
- .replace(/\"/g, '"')
- .replace(/\xa0/, ' ');
-}
-
-/** escapest html special characters to html. */
-function PR_textToHtml(str) {
- return str.replace(/&/g, '&')
- .replace(/</g, '<')
- .replace(/>/g, '>')
- .replace(/\xa0/g, ' ');
-}
-
-/** is the given node's innerHTML normally unescaped? */
-function PR_isRawContent(node) {
- return 'XMP' == node.tagName;
-}
-
-var PR_innerHtmlWorks = null;
-function PR_getInnerHtml(node) {
- // inner html is hopelessly broken in Safari 2.0.4 when the content is
- // an html description of well formed XML and the containing tag is a PRE
- // tag, so we detect that case and emulate innerHTML.
- if (null == PR_innerHtmlWorks) {
- var testNode = document.createElement('PRE');
- testNode.appendChild(
- document.createTextNode('<!DOCTYPE foo PUBLIC "foo bar">\n<foo />'));
- PR_innerHtmlWorks = !/</.test(testNode.innerHTML);
- }
+/** @define {boolean} */
+var IN_GLOBAL_SCOPE = true;
- if (PR_innerHtmlWorks) {
- var content = node.innerHTML;
- // XMP tags contain unescaped entities so require special handling.
- if (PR_isRawContent(node)) {
- content = PR_textToHtml(content);
- }
- return content;
- }
-
- var out = [];
- for (var child = node.firstChild; child; child = child.nextSibling) {
- PR_normalizedHtml(child, out);
- }
- return out.join('');
-}
+/**
+ * Split {@code prettyPrint} into multiple timeouts so as not to interfere with
+ * UI events.
+ * If set to {@code false}, {@code prettyPrint()} is synchronous.
+ */
+window['PR_SHOULD_USE_CONTINUATION'] = true;
/**
- * walks the DOM returning a properly escaped version of innerHTML.
+ * Pretty print a chunk of code.
+ * @param {string} sourceCodeHtml The HTML to pretty print.
+ * @param {string} opt_langExtension The language name to use.
+ * Typically, a filename extension like 'cpp' or 'java'.
+ * @param {number|boolean} opt_numberLines True to number lines,
+ * or the 1-indexed number of the first line in sourceCodeHtml.
+ * @return {string} code as html, but prettier
*/
-function PR_normalizedHtml(node, out) {
- switch (node.nodeType) {
- case 1: // an element
- var name = node.tagName.toLowerCase();
- out.push('\074', name);
- for (var i = 0; i < node.attributes.length; ++i) {
- var attr = node.attributes[i];
- if (!attr.specified) { continue; }
- out.push(' ');
- PR_normalizedHtml(attr, out);
- }
- out.push('>');
- for (var child = node.firstChild; child; child = child.nextSibling) {
- PR_normalizedHtml(child, out);
- }
- if (node.firstChild || !/^(?:br|link|img)$/.test(name)) {
- out.push('<\/', name, '>');
- }
- break;
- case 2: // an attribute
- out.push(node.name.toLowerCase(), '="', PR_attribToHtml(node.value), '"');
- break;
- case 3: case 4: // text
- out.push(PR_textToHtml(node.nodeValue));
- break;
- }
-}
+var prettyPrintOne;
+/**
+ * Find all the {@code <pre>} and {@code <code>} tags in the DOM with
+ * {@code class=prettyprint} and prettify them.
+ *
+ * @param {Function} opt_whenDone called when prettifying is done.
+ * @param {HTMLElement|HTMLDocument} opt_root an element or document
+ * containing all the elements to pretty print.
+ * Defaults to {@code document.body}.
+ */
+var prettyPrint;
-/** expand tabs to spaces
- * @param {Array} chunks PR_Tokens possibly containing tabs
- * @param {Number} tabWidth number of spaces between tab columns
- * @return {Array} chunks with tabs replaced with spaces
- */
-function PR_expandTabs(chunks, tabWidth) {
- var SPACES = ' ';
-
- var charInLine = 0;
- var decodeHelper = new PR_DecodeHelper();
-
- var chunksOut = []
- for (var chunkIndex = 0; chunkIndex < chunks.length; ++chunkIndex) {
- var chunk = chunks[chunkIndex];
- if (chunk.style == null) {
- chunksOut.push(chunk);
- continue;
- }
-
- var s = chunk.token;
- var pos = 0; // index of last character output
- var out = [];
-
- // walk over each character looking for tabs and newlines.
- // On tabs, expand them. On newlines, reset charInLine.
- // Otherwise increment charInLine
- for (var charIndex = 0, n = s.length; charIndex < n;
- charIndex = decodeHelper.next) {
- decodeHelper.decode(s, charIndex);
- var ch = decodeHelper.ch;
-
- switch (ch) {
- case '\t':
- out.push(s.substring(pos, charIndex));
- // calculate how much space we need in front of this part
- // nSpaces is the amount of padding -- the number of spaces needed to
- // move us to the next column, where columns occur at factors of
- // tabWidth.
- var nSpaces = tabWidth - (charInLine % tabWidth);
- charInLine += nSpaces;
- for (; nSpaces >= 0; nSpaces -= SPACES.length) {
- out.push(SPACES.substring(0, nSpaces));
- }
- pos = decodeHelper.next;
- break;
- case '\n': case '\r':
- charInLine = 0;
- break;
- default:
- ++charInLine;
- }
- }
- out.push(s.substring(pos));
- chunksOut.push(new PR_Token(out.join(''), chunk.style));
- }
- return chunksOut
-}
-/** split markup into chunks of html tags (style null) and
- * plain text (style {@link #PR_PLAIN}).
- *
- * @param {String} s html.
- * @return {Array} of PR_Tokens of style PR_PLAIN, and null.
- * @private
- */
-function PR_chunkify(s) {
- // The below pattern matches one of the following
- // (1) /[^<]+/ : A run of characters other than '<'
- // (2) /<\/?[a-zA-Z][^>]*>/ : A probably tag that should not be highlighted
- // (3) /</ : A '<' that does not begin a larger chunk. Treated as 1
- var chunkPattern = /(?:[^<]+|<\/?[a-zA-Z][^>]*>|<)/g;
- // since the pattern has the 'g' modifier and defines no capturing groups,
- // this will return a list of all chunks which we then classify and wrap as
- // PR_Tokens
- var matches = s.match(chunkPattern);
- var chunks = [];
- if (matches) {
- var lastChunk = null;
- for (var i = 0, n = matches.length; i < n; ++i) {
- var chunkText = matches[i];
- var style;
- if (chunkText.length < 2 || chunkText.charAt(0) !== '<') {
- if (lastChunk && lastChunk.style === PR_PLAIN) {
- lastChunk.token += chunkText;
- continue;
- }
- style = PR_PLAIN;
- } else { // a tag
- style = null;
+(function () {
+ var win = window;
+ // Keyword lists for various languages.
+ // We use things that coerce to strings to make them compact when minified
+ // and to defeat aggressive optimizers that fold large string constants.
+ var FLOW_CONTROL_KEYWORDS = ["break,continue,do,else,for,if,return,while"];
+ var C_KEYWORDS = [FLOW_CONTROL_KEYWORDS,"auto,case,char,const,default," +
+ "double,enum,extern,float,goto,inline,int,long,register,short,signed," +
+ "sizeof,static,struct,switch,typedef,union,unsigned,void,volatile"];
+ var COMMON_KEYWORDS = [C_KEYWORDS,"catch,class,delete,false,import," +
+ "new,operator,private,protected,public,this,throw,true,try,typeof"];
+ var CPP_KEYWORDS = [COMMON_KEYWORDS,"alignof,align_union,asm,axiom,bool," +
+ "concept,concept_map,const_cast,constexpr,decltype,delegate," +
+ "dynamic_cast,explicit,export,friend,generic,late_check," +
+ "mutable,namespace,nullptr,property,reinterpret_cast,static_assert," +
+ "static_cast,template,typeid,typename,using,virtual,where"];
+ var JAVA_KEYWORDS = [COMMON_KEYWORDS,
+ "abstract,assert,boolean,byte,extends,final,finally,implements,import," +
+ "instanceof,interface,null,native,package,strictfp,super,synchronized," +
+ "throws,transient"];
+ var CSHARP_KEYWORDS = [JAVA_KEYWORDS,
+ "as,base,by,checked,decimal,delegate,descending,dynamic,event," +
+ "fixed,foreach,from,group,implicit,in,internal,into,is,let," +
+ "lock,object,out,override,orderby,params,partial,readonly,ref,sbyte," +
+ "sealed,stackalloc,string,select,uint,ulong,unchecked,unsafe,ushort," +
+ "var,virtual,where"];
+ var COFFEE_KEYWORDS = "all,and,by,catch,class,else,extends,false,finally," +
+ "for,if,in,is,isnt,loop,new,no,not,null,of,off,on,or,return,super,then," +
+ "throw,true,try,unless,until,when,while,yes";
+ var JSCRIPT_KEYWORDS = [COMMON_KEYWORDS,
+ "debugger,eval,export,function,get,null,set,undefined,var,with," +
+ "Infinity,NaN"];
+ var PERL_KEYWORDS = "caller,delete,die,do,dump,elsif,eval,exit,foreach,for," +
+ "goto,if,import,last,local,my,next,no,our,print,package,redo,require," +
+ "sub,undef,unless,until,use,wantarray,while,BEGIN,END";
+ var PYTHON_KEYWORDS = [FLOW_CONTROL_KEYWORDS, "and,as,assert,class,def,del," +
+ "elif,except,exec,finally,from,global,import,in,is,lambda," +
+ "nonlocal,not,or,pass,print,raise,try,with,yield," +
+ "False,True,None"];
+ var RUBY_KEYWORDS = [FLOW_CONTROL_KEYWORDS, "alias,and,begin,case,class," +
+ "def,defined,elsif,end,ensure,false,in,module,next,nil,not,or,redo," +
+ "rescue,retry,self,super,then,true,undef,unless,until,when,yield," +
+ "BEGIN,END"];
+ var RUST_KEYWORDS = [FLOW_CONTROL_KEYWORDS, "as,assert,const,copy,drop," +
+ "enum,extern,fail,false,fn,impl,let,log,loop,match,mod,move,mut,priv," +
+ "pub,pure,ref,self,static,struct,true,trait,type,unsafe,use"];
+ var SH_KEYWORDS = [FLOW_CONTROL_KEYWORDS, "case,done,elif,esac,eval,fi," +
+ "function,in,local,set,then,until"];
+ var ALL_KEYWORDS = [
+ CPP_KEYWORDS, CSHARP_KEYWORDS, JSCRIPT_KEYWORDS, PERL_KEYWORDS,
+ PYTHON_KEYWORDS, RUBY_KEYWORDS, SH_KEYWORDS];
+ var C_TYPES = /^(DIR|FILE|vector|(de|priority_)?queue|list|stack|(const_)?iterator|(multi)?(set|map)|bitset|u?(int|float)\d*)\b/;
+
+ // token style names. correspond to css classes
+ /**
+ * token style for a string literal
+ * @const
+ */
+ var PR_STRING = 'str';
+ /**
+ * token style for a keyword
+ * @const
+ */
+ var PR_KEYWORD = 'kwd';
+ /**
+ * token style for a comment
+ * @const
+ */
+ var PR_COMMENT = 'com';
+ /**
+ * token style for a type
+ * @const
+ */
+ var PR_TYPE = 'typ';
+ /**
+ * token style for a literal value. e.g. 1, null, true.
+ * @const
+ */
+ var PR_LITERAL = 'lit';
+ /**
+ * token style for a punctuation string.
+ * @const
+ */
+ var PR_PUNCTUATION = 'pun';
+ /**
+ * token style for plain text.
+ * @const
+ */
+ var PR_PLAIN = 'pln';
+
+ /**
+ * token style for an sgml tag.
+ * @const
+ */
+ var PR_TAG = 'tag';
+ /**
+ * token style for a markup declaration such as a DOCTYPE.
+ * @const
+ */
+ var PR_DECLARATION = 'dec';
+ /**
+ * token style for embedded source.
+ * @const
+ */
+ var PR_SOURCE = 'src';
+ /**
+ * token style for an sgml attribute name.
+ * @const
+ */
+ var PR_ATTRIB_NAME = 'atn';
+ /**
+ * token style for an sgml attribute value.
+ * @const
+ */
+ var PR_ATTRIB_VALUE = 'atv';
+
+ /**
+ * A class that indicates a section of markup that is not code, e.g. to allow
+ * embedding of line numbers within code listings.
+ * @const
+ */
+ var PR_NOCODE = 'nocode';
+
+
+
+ /**
+ * A set of tokens that can precede a regular expression literal in
+ * javascript
+ * http://web.archive.org/web/20070717142515/http://www.mozilla.org/js/language/js20/rationale/syntax.html
+ * has the full list, but I've removed ones that might be problematic when
+ * seen in languages that don't support regular expression literals.
+ *
+ * <p>Specifically, I've removed any keywords that can't precede a regexp
+ * literal in a syntactically legal javascript program, and I've removed the
+ * "in" keyword since it's not a keyword in many languages, and might be used
+ * as a count of inches.
+ *
+ * <p>The link above does not accurately describe EcmaScript rules since
+ * it fails to distinguish between (a=++/b/i) and (a++/b/i) but it works
+ * very well in practice.
+ *
+ * @private
+ * @const
+ */
+ var REGEXP_PRECEDER_PATTERN = '(?:^^\\.?|[+-]|[!=]=?=?|\\#|%=?|&&?=?|\\(|\\*=?|[+\\-]=|->|\\/=?|::?|<<?=?|>>?>?=?|,|;|\\?|@|\\[|~|{|\\^\\^?=?|\\|\\|?=?|break|case|continue|delete|do|else|finally|instanceof|return|throw|try|typeof)\\s*';
+
+ // CAVEAT: this does not properly handle the case where a regular
+ // expression immediately follows another since a regular expression may
+ // have flags for case-sensitivity and the like. Having regexp tokens
+ // adjacent is not valid in any language I'm aware of, so I'm punting.
+ // TODO: maybe style special characters inside a regexp as punctuation.
+
+ /**
+ * Given a group of {@link RegExp}s, returns a {@code RegExp} that globally
+ * matches the union of the sets of strings matched by the input RegExp.
+ * Since it matches globally, if the input strings have a start-of-input
+ * anchor (/^.../), it is ignored for the purposes of unioning.
+ * @param {Array.<RegExp>} regexs non multiline, non-global regexs.
+ * @return {RegExp} a global regex.
+ */
+ function combinePrefixPatterns(regexs) {
+ var capturedGroupIndex = 0;
+
+ var needToFoldCase = false;
+ var ignoreCase = false;
+ for (var i = 0, n = regexs.length; i < n; ++i) {
+ var regex = regexs[i];
+ if (regex.ignoreCase) {
+ ignoreCase = true;
+ } else if (/[a-z]/i.test(regex.source.replace(
+ /\\u[0-9a-f]{4}|\\x[0-9a-f]{2}|\\[^ux]/gi, ''))) {
+ needToFoldCase = true;
+ ignoreCase = false;
+ break;
}
- lastChunk = new PR_Token(chunkText, style);
- chunks.push(lastChunk);
}
- }
- return chunks;
-}
-
-/** walk the tokenEnds list and the chunk list in parallel to generate a list
- * of split tokens.
- * @private
- */
-function PR_splitChunks(chunks, tokenEnds) {
- var tokens = []; // the output
-
- var ci = 0; // index into chunks
- // position of beginning of amount written so far in absolute space.
- var posAbs = 0;
- // position of amount written so far in chunk space
- var posChunk = 0;
-
- // current chunk
- var chunk = new PR_Token('', null);
-
- for (var ei = 0, ne = tokenEnds.length, lastEnd = 0; ei < ne; ++ei) {
- var tokenEnd = tokenEnds[ei];
- var end = tokenEnd.end;
- if (end === lastEnd) { continue; } // skip empty regions
-
- var tokLen = end - posAbs;
- var remainingInChunk = chunk.token.length - posChunk;
- while (remainingInChunk <= tokLen) {
- if (remainingInChunk > 0) {
- tokens.push(
- new PR_Token(chunk.token.substring(posChunk, chunk.token.length),
- null == chunk.style ? null : tokenEnd.style));
- }
- posAbs += remainingInChunk;
- posChunk = 0;
- if (ci < chunks.length) {
- chunk = chunks[ci++];
- }
-
- tokLen = end - posAbs;
- remainingInChunk = chunk.token.length - posChunk;
- }
-
- if (tokLen) {
- tokens.push(
- new PR_Token(chunk.token.substring(posChunk, posChunk + tokLen),
- tokenEnd.style));
- posAbs += tokLen;
- posChunk += tokLen;
+
+ var escapeCharToCodeUnit = {
+ 'b': 8,
+ 't': 9,
+ 'n': 0xa,
+ 'v': 0xb,
+ 'f': 0xc,
+ 'r': 0xd
+ };
+
+ function decodeEscape(charsetPart) {
+ var cc0 = charsetPart.charCodeAt(0);
+ if (cc0 !== 92 /* \\ */) {
+ return cc0;
+ }
+ var c1 = charsetPart.charAt(1);
+ cc0 = escapeCharToCodeUnit[c1];
+ if (cc0) {
+ return cc0;
+ } else if ('0' <= c1 && c1 <= '7') {
+ return parseInt(charsetPart.substring(1), 8);
+ } else if (c1 === 'u' || c1 === 'x') {
+ return parseInt(charsetPart.substring(2), 16);
+ } else {
+ return charsetPart.charCodeAt(1);
+ }
}
- }
-
- return tokens;
-}
-
-/** splits markup tokens into declarations, tags, and source chunks.
- * @private
- */
-function PR_splitMarkup(chunks) {
- // A state machine to split out declarations, tags, etc.
- // This state machine deals with absolute space in the text, indexed by k,
- // and position in the current chunk, indexed by pos and tokenStart to
- // generate a list of the ends of tokens.
- // Absolute space is calculated by considering the chunks as appended into
- // one big string, as they were before being split.
-
- // Known failure cases
- // Server side scripting sections such as <?...?> in attributes.
- // i.e. <span class="<? foo ?>">
- // Handling this would require a stack, and we don't use PHP.
-
- // The output: a list of pairs of PR_TokenEnd instances
- var tokenEnds = [];
-
- var state = 0; // FSM state variable
- var k = 0; // position in absolute space of the start of the current chunk
- var tokenStart = -1; // the start of the current token
-
- // Try to find a closing tag for any open <style> or <script> tags
- // We can't do this at a later stage because then the following case
- // would fail:
- // <script>document.writeln('<!--');</script>
-
- // We use tokenChars[:tokenCharsI] to accumulate the tag name so that we
- // can check whether to enter into a no scripting section when the tag ends.
- var tokenChars = new Array(12);
- var tokenCharsI = 0;
- // if non null, the tag prefix that we need to see to break out.
- var endScriptTag = null;
- var decodeHelper = new PR_DecodeHelper();
-
- for (var ci = 0, nc = chunks.length; ci < nc; ++ci) {
- var chunk = chunks[ci];
- if (PR_PLAIN != chunk.style) {
- k += chunk.token.length;
- continue;
- }
-
- var s = chunk.token;
- var pos = 0; // the position past the last character processed so far in s
-
- for (var i = 0, n = s.length; i < n; /* i = next at bottom */) {
- decodeHelper.decode(s, i);
- var ch = decodeHelper.ch;
- var next = decodeHelper.next;
-
- var tokenStyle = null;
- switch (state) {
- case 0:
- if ('<' == ch) { state = 1; }
- break;
- case 1:
- tokenCharsI = 0;
- if ('/' == ch) { // only consider close tags if we're in script/style
- state = 7;
- } else if (null == endScriptTag) {
- if ('!' == ch) {
- state = 2;
- } else if (PR_isWordChar(ch)) {
- state = 8;
- } else if ('?' == ch) {
- state = 9;
- } else if ('%' == ch) {
- state = 11;
- } else if ('<' != ch) {
- state = 0;
- }
- } else if ('<' != ch) {
- state = 0;
- }
- break;
- case 2:
- if ('-' == ch) {
- state = 4;
- } else if (PR_isWordChar(ch)) {
- state = 3;
- } else if ('<' == ch) {
- state = 1;
- } else {
- state = 0;
- }
- break;
- case 3:
- if ('>' == ch) {
- state = 0;
- tokenStyle = PR_DECLARATION;
- }
- break;
- case 4:
- if ('-' == ch) { state = 5; }
- break;
- case 5:
- if ('-' == ch) { state = 6; }
- break;
- case 6:
- if ('>' == ch) {
- state = 0;
- tokenStyle = PR_COMMENT;
- } else if ('-' == ch) {
- state = 6;
- } else {
- state = 4;
- }
- break;
- case 7:
- if (PR_isWordChar(ch)) {
- state = 8;
- } else if ('<' == ch) {
- state = 1;
+
+ function encodeEscape(charCode) {
+ if (charCode < 0x20) {
+ return (charCode < 0x10 ? '\\x0' : '\\x') + charCode.toString(16);
+ }
+ var ch = String.fromCharCode(charCode);
+ return (ch === '\\' || ch === '-' || ch === ']' || ch === '^')
+ ? "\\" + ch : ch;
+ }
+
+ function caseFoldCharset(charSet) {
+ var charsetParts = charSet.substring(1, charSet.length - 1).match(
+ new RegExp(
+ '\\\\u[0-9A-Fa-f]{4}'
+ + '|\\\\x[0-9A-Fa-f]{2}'
+ + '|\\\\[0-3][0-7]{0,2}'
+ + '|\\\\[0-7]{1,2}'
+ + '|\\\\[\\s\\S]'
+ + '|-'
+ + '|[^-\\\\]',
+ 'g'));
+ var ranges = [];
+ var inverse = charsetParts[0] === '^';
+
+ var out = ['['];
+ if (inverse) { out.push('^'); }
+
+ for (var i = inverse ? 1 : 0, n = charsetParts.length; i < n; ++i) {
+ var p = charsetParts[i];
+ if (/\\[bdsw]/i.test(p)) { // Don't muck with named groups.
+ out.push(p);
+ } else {
+ var start = decodeEscape(p);
+ var end;
+ if (i + 2 < n && '-' === charsetParts[i + 1]) {
+ end = decodeEscape(charsetParts[i + 2]);
+ i += 2;
} else {
- state = 0;
+ end = start;
}
- break;
- case 8:
- if ('>' == ch) {
- state = 0;
- tokenStyle = PR_TAG;
- }
- break;
- case 9:
- if ('?' == ch) { state = 10; }
- break;
- case 10:
- if ('>' == ch) {
- state = 0;
- tokenStyle = PR_SOURCE;
- } else if ('?' != ch) {
- state = 9;
- }
- break;
- case 11:
- if ('%' == ch) { state = 12; }
- break;
- case 12:
- if ('>' == ch) {
- state = 0;
- tokenStyle = PR_SOURCE;
- } else if ('%' != ch) {
- state = 11;
- }
- break;
- }
-
- if (tokenCharsI < tokenChars.length) {
- tokenChars[tokenCharsI++] = ch.toLowerCase();
- }
- if (1 == state) { tokenStart = k + i; }
- i = next;
- if (tokenStyle != null) {
- if (null != tokenStyle) {
- if (endScriptTag) {
- if (PR_prefixMatch(tokenChars, tokenCharsI, endScriptTag)) {
- endScriptTag = null;
+ ranges.push([start, end]);
+ // If the range might intersect letters, then expand it.
+ // This case handling is too simplistic.
+ // It does not deal with non-latin case folding.
+ // It works for latin source code identifiers though.
+ if (!(end < 65 || start > 122)) {
+ if (!(end < 65 || start > 90)) {
+ ranges.push([Math.max(65, start) | 32, Math.min(end, 90) | 32]);
}
- } else {
- if (PR_prefixMatch(tokenChars, tokenCharsI, 'script')) {
- endScriptTag = '/script';
- } else if (PR_prefixMatch(tokenChars, tokenCharsI, 'style')) {
- endScriptTag = '/style';
- } else if (PR_prefixMatch(tokenChars, tokenCharsI, 'xmp')) {
- endScriptTag = '/xmp';
+ if (!(end < 97 || start > 122)) {
+ ranges.push([Math.max(97, start) & ~32, Math.min(end, 122) & ~32]);
}
}
- // disallow the tag if endScriptTag is set and this was not an open
- // tag.
- if (endScriptTag && tokenCharsI && '/' == tokenChars[0]) {
- tokenStyle = null;
- }
- }
- if (null != tokenStyle) {
- tokenEnds.push(new PR_TokenEnd(tokenStart, PR_PLAIN));
- tokenEnds.push(new PR_TokenEnd(k + next, tokenStyle));
}
}
- }
- k += chunk.token.length;
- }
- tokenEnds.push(new PR_TokenEnd(k, PR_PLAIN));
-
- return tokenEnds;
-}
-
-/** splits the given string into comment, string, and "other" tokens.
- * @return {Array} of PR_Tokens with style in
- * (PR_STRING, PR_COMMENT, PR_PLAIN, null)
- * The result array may contain spurious zero length tokens. Ignore them.
- *
- * @private
- */
-function PR_splitStringAndCommentTokens(chunks) {
- // a state machine to split out comments, strings, and other stuff
- var tokenEnds = []; // positions of ends of tokens in absolute space
- var state = 0; // FSM state variable
- var delim = -1; // string delimiter
- var k = 0; // absolute position of beginning of current chunk
-
- for (var ci = 0, nc = chunks.length; ci < nc; ++ci) {
- var chunk = chunks[ci];
- var s = chunk.token;
- if (PR_PLAIN == chunk.style) {
- var decodeHelper = new PR_DecodeHelper();
- var last = -1;
- var next;
- for (var i = 0, n = s.length; i < n; last = i, i = next) {
- decodeHelper.decode(s, i);
- var ch = decodeHelper.ch;
- next = decodeHelper.next;
- if (0 == state) {
- if (ch == '"' || ch == '\'' || ch == '`') {
- tokenEnds.push(new PR_TokenEnd(k + i, PR_PLAIN));
- state = 1;
- delim = ch;
- } else if (ch == '/') {
- state = 3;
- } else if (ch == '#') {
- tokenEnds.push(new PR_TokenEnd(k + i, PR_PLAIN));
- state = 4;
- }
- } else if (1 == state) {
- if (ch == delim) {
- state = 0;
- tokenEnds.push(new PR_TokenEnd(k + next, PR_STRING));
- } else if (ch == '\\') {
- state = 2;
- }
- } else if (2 == state) {
- state = 1;
- } else if (3 == state) {
- if (ch == '/') {
- state = 4;
- tokenEnds.push(new PR_TokenEnd(k + last, PR_PLAIN));
- } else if (ch == '*') {
- state = 5;
- tokenEnds.push(new PR_TokenEnd(k + last, PR_PLAIN));
- } else {
- state = 0;
- // next loop will reenter state 0 without same value of i, so
- // ch will be reconsidered as start of new token.
- next = i;
- }
- } else if (4 == state) {
- if (ch == '\r' || ch == '\n') {
- state = 0;
- tokenEnds.push(new PR_TokenEnd(k + i, PR_COMMENT));
- }
- } else if (5 == state) {
- if (ch == '*') {
- state = 6;
- }
- } else if (6 == state) {
- if (ch == '/') {
- state = 0;
- tokenEnds.push(new PR_TokenEnd(k + next, PR_COMMENT));
- } else if (ch != '*') {
- state = 5;
- }
- }
- }
- }
- k += s.length;
- }
- var endTokenType;
- switch (state) {
- case 1: case 2:
- endTokenType = PR_STRING;
- break;
- case 4: case 5: case 6:
- endTokenType = PR_COMMENT;
- break;
- default:
- endTokenType = PR_PLAIN;
- break;
- }
- // handle unclosed token which can legally happen for line comments (state 4)
- tokenEnds.push(new PR_TokenEnd(k, endTokenType)); // a token ends at the end
-
- return PR_splitChunks(chunks, tokenEnds);
-}
-
-/** used by lexSource to split a non string, non comment token.
- * @private
- */
-function PR_splitNonStringNonCommentToken(s, outlist) {
- var pos = 0;
- var state = 0;
-
- var decodeHelper = new PR_DecodeHelper();
- var next;
- for (var i = 0; i <= s.length; i = next) {
- if (i == s.length) {
- // nstate will not be equal to state, so it will append the token
- nstate = -2;
- next = i + 1;
- } else {
- decodeHelper.decode(s, i);
- next = decodeHelper.next;
- var ch = decodeHelper.ch;
-
- // the next state.
- // if set to -1 then it will cause a reentry to state 0 without consuming
- // another character.
- var nstate = state;
-
- switch (state) {
- case 0: // whitespace state
- if (PR_isIdentifierStart(ch)) {
- nstate = 1;
- } else if (PR_isDigitChar(ch)) {
- nstate = 2;
- } else if (!PR_isSpaceChar(ch)) {
- nstate = 3;
- }
- if (nstate && pos < i) {
- var t = s.substring(pos, i);
- outlist.push(new PR_Token(t, PR_PLAIN));
- pos = i;
- }
- break;
- case 1: // identifier state
- if (!PR_isIdentifierPart(ch)) {
- nstate = -1;
- }
- break;
- case 2: // number literal state
- // handle numeric literals like
- // 0x7f 300UL 100_000
-
- // this does not treat floating point values as a single literal
- // 0.1 and 3e-6
- // are each split into multiple tokens
- if (!(PR_isDigitChar(ch) || PR_isWordChar(ch) || ch == '_')) {
- nstate = -1;
- }
- break;
- case 3: // punctuation state
- if (PR_isIdentifierStart(ch) || PR_isDigitChar(ch) ||
- PR_isSpaceChar(ch)) {
- nstate = -1;
+
+ // [[1, 10], [3, 4], [8, 12], [14, 14], [16, 16], [17, 17]]
+ // -> [[1, 12], [14, 14], [16, 17]]
+ ranges.sort(function (a, b) { return (a[0] - b[0]) || (b[1] - a[1]); });
+ var consolidatedRanges = [];
+ var lastRange = [];
+ for (var i = 0; i < ranges.length; ++i) {
+ var range = ranges[i];
+ if (range[0] <= lastRange[1] + 1) {
+ lastRange[1] = Math.max(lastRange[1], range[1]);
+ } else {
+ consolidatedRanges.push(lastRange = range);
}
- break;
}
- }
-
- if (nstate != state) {
- if (nstate < 0) {
- if (i > pos) {
- var t = s.substring(pos, i);
- var wordDecodeHelper = new PR_DecodeHelper();
- wordDecodeHelper.decode(t, 0);
- var ch0 = wordDecodeHelper.ch;
- var isSingleCharacter = wordDecodeHelper.next == t.length;
- var style;
- if (PR_isIdentifierStart(ch0)) {
- if (PR_keywords[t]) {
- style = PR_KEYWORD;
- } else if (ch0 === '@') {
- style = PR_LITERAL;
+
+ for (var i = 0; i < consolidatedRanges.length; ++i) {
+ var range = consolidatedRanges[i];
+ out.push(encodeEscape(range[0]));
+ if (range[1] > range[0]) {
+ if (range[1] + 1 > range[0]) { out.push('-'); }
+ out.push(encodeEscape(range[1]));
+ }
+ }
+ out.push(']');
+ return out.join('');
+ }
+
+ function allowAnywhereFoldCaseAndRenumberGroups(regex) {
+ // Split into character sets, escape sequences, punctuation strings
+ // like ('(', '(?:', ')', '^'), and runs of characters that do not
+ // include any of the above.
+ var parts = regex.source.match(
+ new RegExp(
+ '(?:'
+ + '\\[(?:[^\\x5C\\x5D]|\\\\[\\s\\S])*\\]' // a character set
+ + '|\\\\u[A-Fa-f0-9]{4}' // a unicode escape
+ + '|\\\\x[A-Fa-f0-9]{2}' // a hex escape
+ + '|\\\\[0-9]+' // a back-reference or octal escape
+ + '|\\\\[^ux0-9]' // other escape sequence
+ + '|\\(\\?[:!=]' // start of a non-capturing group
+ + '|[\\(\\)\\^]' // start/end of a group, or line start
+ + '|[^\\x5B\\x5C\\(\\)\\^]+' // run of other characters
+ + ')',
+ 'g'));
+ var n = parts.length;
+
+ // Maps captured group numbers to the number they will occupy in
+ // the output or to -1 if that has not been determined, or to
+ // undefined if they need not be capturing in the output.
+ var capturedGroups = [];
+
+ // Walk over and identify back references to build the capturedGroups
+ // mapping.
+ for (var i = 0, groupIndex = 0; i < n; ++i) {
+ var p = parts[i];
+ if (p === '(') {
+ // groups are 1-indexed, so max group index is count of '('
+ ++groupIndex;
+ } else if ('\\' === p.charAt(0)) {
+ var decimalValue = +p.substring(1);
+ if (decimalValue) {
+ if (decimalValue <= groupIndex) {
+ capturedGroups[decimalValue] = -1;
} else {
- // Treat any word that starts with an uppercase character and
- // contains at least one lowercase character as a type, or
- // ends with _t.
- // This works perfectly for Java, pretty well for C++, and
- // passably for Python. The _t catches C structs.
- var isType = false;
- if (ch0 >= 'A' && ch0 <= 'Z') {
- for (var j = wordDecodeHelper.next;
- j < t.length; j = wordDecodeHelper.next) {
- wordDecodeHelper.decode(t, j);
- var ch1 = wordDecodeHelper.ch;
- if (ch1 >= 'a' && ch1 <= 'z') {
- isType = true;
- break;
- }
- }
- if (!isType && !isSingleCharacter &&
- t.substring(t.length - 2) == '_t') {
- isType = true;
- }
- }
- style = isType ? PR_TYPE : PR_PLAIN;
+ // Replace with an unambiguous escape sequence so that
+ // an octal escape sequence does not turn into a backreference
+ // to a capturing group from an earlier regex.
+ parts[i] = encodeEscape(decimalValue);
}
- } else if (PR_isDigitChar(ch0)) {
- style = PR_LITERAL;
- } else if (!PR_isSpaceChar(ch0)) {
- style = PR_PUNCTUATION;
- } else {
- style = PR_PLAIN;
}
- pos = i;
- outlist.push(new PR_Token(t, style));
- }
-
- state = 0;
- if (nstate == -1) {
- // don't increment. This allows us to use state 0 to redispatch based
- // on the current character.
- next = i;
- continue;
}
}
- state = nstate;
- }
- }
-
-}
-
-/** split a group of chunks of markup.
- * @private
- */
-function PR_tokenizeMarkup(chunks) {
- if (!(chunks && chunks.length)) { return chunks; }
-
- var tokenEnds = PR_splitMarkup(chunks);
- return PR_splitChunks(chunks, tokenEnds);
-}
-
-/** split tags attributes and their values out from the tag name, and
- * recursively lex source chunks.
- * @private
- */
-function PR_splitTagAttributes(tokens) {
- var tokensOut = [];
- var state = 0;
- var stateStyle = PR_TAG;
- var delim = null; // attribute delimiter for quoted value state.
- var decodeHelper = new PR_DecodeHelper();
- for (var ci = 0; ci < tokens.length; ++ci) {
- var tok = tokens[ci];
- if (PR_TAG == tok.style) {
- var s = tok.token;
- var start = 0;
- for (var i = 0; i < s.length; /* i = next at bottom */) {
- decodeHelper.decode(s, i);
- var ch = decodeHelper.ch;
- var next = decodeHelper.next;
-
- var emitEnd = null; // null or position of end of chunk to emit.
- var nextStyle = null; // null or next value of stateStyle
- if (ch == '>') {
- if (PR_TAG != stateStyle) {
- emitEnd = i;
- nextStyle = PR_TAG;
+
+ // Renumber groups and reduce capturing groups to non-capturing groups
+ // where possible.
+ for (var i = 1; i < capturedGroups.length; ++i) {
+ if (-1 === capturedGroups[i]) {
+ capturedGroups[i] = ++capturedGroupIndex;
+ }
+ }
+ for (var i = 0, groupIndex = 0; i < n; ++i) {
+ var p = parts[i];
+ if (p === '(') {
+ ++groupIndex;
+ if (!capturedGroups[groupIndex]) {
+ parts[i] = '(?:';
+ }
+ } else if ('\\' === p.charAt(0)) {
+ var decimalValue = +p.substring(1);
+ if (decimalValue && decimalValue <= groupIndex) {
+ parts[i] = '\\' + capturedGroups[decimalValue];
+ }
+ }
+ }
+
+ // Remove any prefix anchors so that the output will match anywhere.
+ // ^^ really does mean an anchored match though.
+ for (var i = 0; i < n; ++i) {
+ if ('^' === parts[i] && '^' !== parts[i + 1]) { parts[i] = ''; }
+ }
+
+ // Expand letters to groups to handle mixing of case-sensitive and
+ // case-insensitive patterns if necessary.
+ if (regex.ignoreCase && needToFoldCase) {
+ for (var i = 0; i < n; ++i) {
+ var p = parts[i];
+ var ch0 = p.charAt(0);
+ if (p.length >= 2 && ch0 === '[') {
+ parts[i] = caseFoldCharset(p);
+ } else if (ch0 !== '\\') {
+ // TODO: handle letters in numeric escapes.
+ parts[i] = p.replace(
+ /[a-zA-Z]/g,
+ function (ch) {
+ var cc = ch.charCodeAt(0);
+ return '[' + String.fromCharCode(cc & ~32, cc | 32) + ']';
+ });
+ }
+ }
+ }
+
+ return parts.join('');
+ }
+
+ var rewritten = [];
+ for (var i = 0, n = regexs.length; i < n; ++i) {
+ var regex = regexs[i];
+ if (regex.global || regex.multiline) { throw new Error('' + regex); }
+ rewritten.push(
+ '(?:' + allowAnywhereFoldCaseAndRenumberGroups(regex) + ')');
+ }
+
+ return new RegExp(rewritten.join('|'), ignoreCase ? 'gi' : 'g');
+ }
+
+ /**
+ * Split markup into a string of source code and an array mapping ranges in
+ * that string to the text nodes in which they appear.
+ *
+ * <p>
+ * The HTML DOM structure:</p>
+ * <pre>
+ * (Element "p"
+ * (Element "b"
+ * (Text "print ")) ; #1
+ * (Text "'Hello '") ; #2
+ * (Element "br") ; #3
+ * (Text " + 'World';")) ; #4
+ * </pre>
+ * <p>
+ * corresponds to the HTML
+ * {@code <p><b>print </b>'Hello '<br> + 'World';</p>}.</p>
+ *
+ * <p>
+ * It will produce the output:</p>
+ * <pre>
+ * {
+ * sourceCode: "print 'Hello '\n + 'World';",
+ * // 1 2
+ * // 012345678901234 5678901234567
+ * spans: [0, #1, 6, #2, 14, #3, 15, #4]
+ * }
+ * </pre>
+ * <p>
+ * where #1 is a reference to the {@code "print "} text node above, and so
+ * on for the other text nodes.
+ * </p>
+ *
+ * <p>
+ * The {@code} spans array is an array of pairs. Even elements are the start
+ * indices of substrings, and odd elements are the text nodes (or BR elements)
+ * that contain the text for those substrings.
+ * Substrings continue until the next index or the end of the source.
+ * </p>
+ *
+ * @param {Node} node an HTML DOM subtree containing source-code.
+ * @param {boolean} isPreformatted true if white-space in text nodes should
+ * be considered significant.
+ * @return {Object} source code and the text nodes in which they occur.
+ */
+ function extractSourceSpans(node, isPreformatted) {
+ var nocode = /(?:^|\s)nocode(?:\s|$)/;
+
+ var chunks = [];
+ var length = 0;
+ var spans = [];
+ var k = 0;
+
+ function walk(node) {
+ var type = node.nodeType;
+ if (type == 1) { // Element
+ if (nocode.test(node.className)) { return; }
+ for (var child = node.firstChild; child; child = child.nextSibling) {
+ walk(child);
+ }
+ var nodeName = node.nodeName.toLowerCase();
+ if ('br' === nodeName || 'li' === nodeName) {
+ chunks[k] = '\n';
+ spans[k << 1] = length++;
+ spans[(k++ << 1) | 1] = node;
+ }
+ } else if (type == 3 || type == 4) { // Text
+ var text = node.nodeValue;
+ if (text.length) {
+ if (!isPreformatted) {
+ text = text.replace(/[ \t\r\n]+/g, ' ');
+ } else {
+ text = text.replace(/\r\n?/g, '\n'); // Normalize newlines.
}
+ // TODO: handle tabs here?
+ chunks[k] = text;
+ spans[k << 1] = length;
+ length += text.length;
+ spans[(k++ << 1) | 1] = node;
+ }
+ }
+ }
+
+ walk(node);
+
+ return {
+ sourceCode: chunks.join('').replace(/\n$/, ''),
+ spans: spans
+ };
+ }
+
+ /**
+ * Apply the given language handler to sourceCode and add the resulting
+ * decorations to out.
+ * @param {number} basePos the index of sourceCode within the chunk of source
+ * whose decorations are already present on out.
+ */
+ function appendDecorations(basePos, sourceCode, langHandler, out) {
+ if (!sourceCode) { return; }
+ var job = {
+ sourceCode: sourceCode,
+ basePos: basePos
+ };
+ langHandler(job);
+ out.push.apply(out, job.decorations);
+ }
+
+ var notWs = /\S/;
+
+ /**
+ * Given an element, if it contains only one child element and any text nodes
+ * it contains contain only space characters, return the sole child element.
+ * Otherwise returns undefined.
+ * <p>
+ * This is meant to return the CODE element in {@code <pre><code ...>} when
+ * there is a single child element that contains all the non-space textual
+ * content, but not to return anything where there are multiple child elements
+ * as in {@code <pre><code>...</code><code>...</code></pre>} or when there
+ * is textual content.
+ */
+ function childContentWrapper(element) {
+ var wrapper = undefined;
+ for (var c = element.firstChild; c; c = c.nextSibling) {
+ var type = c.nodeType;
+ wrapper = (type === 1) // Element Node
+ ? (wrapper ? element : c)
+ : (type === 3) // Text Node
+ ? (notWs.test(c.nodeValue) ? element : wrapper)
+ : wrapper;
+ }
+ return wrapper === element ? undefined : wrapper;
+ }
+
+ /** Given triples of [style, pattern, context] returns a lexing function,
+ * The lexing function interprets the patterns to find token boundaries and
+ * returns a decoration list of the form
+ * [index_0, style_0, index_1, style_1, ..., index_n, style_n]
+ * where index_n is an index into the sourceCode, and style_n is a style
+ * constant like PR_PLAIN. index_n-1 <= index_n, and style_n-1 applies to
+ * all characters in sourceCode[index_n-1:index_n].
+ *
+ * The stylePatterns is a list whose elements have the form
+ * [style : string, pattern : RegExp, DEPRECATED, shortcut : string].
+ *
+ * Style is a style constant like PR_PLAIN, or can be a string of the
+ * form 'lang-FOO', where FOO is a language extension describing the
+ * language of the portion of the token in $1 after pattern executes.
+ * E.g., if style is 'lang-lisp', and group 1 contains the text
+ * '(hello (world))', then that portion of the token will be passed to the
+ * registered lisp handler for formatting.
+ * The text before and after group 1 will be restyled using this decorator
+ * so decorators should take care that this doesn't result in infinite
+ * recursion. For example, the HTML lexer rule for SCRIPT elements looks
+ * something like ['lang-js', /<[s]cript>(.+?)<\/script>/]. This may match
+ * '<script>foo()<\/script>', which would cause the current decorator to
+ * be called with '<script>' which would not match the same rule since
+ * group 1 must not be empty, so it would be instead styled as PR_TAG by
+ * the generic tag rule. The handler registered for the 'js' extension would
+ * then be called with 'foo()', and finally, the current decorator would
+ * be called with '<\/script>' which would not match the original rule and
+ * so the generic tag rule would identify it as a tag.
+ *
+ * Pattern must only match prefixes, and if it matches a prefix, then that
+ * match is considered a token with the same style.
+ *
+ * Context is applied to the last non-whitespace, non-comment token
+ * recognized.
+ *
+ * Shortcut is an optional string of characters, any of which, if the first
+ * character, gurantee that this pattern and only this pattern matches.
+ *
+ * @param {Array} shortcutStylePatterns patterns that always start with
+ * a known character. Must have a shortcut string.
+ * @param {Array} fallthroughStylePatterns patterns that will be tried in
+ * order if the shortcut ones fail. May have shortcuts.
+ *
+ * @return {function (Object)} a
+ * function that takes source code and returns a list of decorations.
+ */
+ function createSimpleLexer(shortcutStylePatterns, fallthroughStylePatterns) {
+ var shortcuts = {};
+ var tokenizer;
+ (function () {
+ var allPatterns = shortcutStylePatterns.concat(fallthroughStylePatterns);
+ var allRegexs = [];
+ var regexKeys = {};
+ for (var i = 0, n = allPatterns.length; i < n; ++i) {
+ var patternParts = allPatterns[i];
+ var shortcutChars = patternParts[3];
+ if (shortcutChars) {
+ for (var c = shortcutChars.length; --c >= 0;) {
+ shortcuts[shortcutChars.charAt(c)] = patternParts;
+ }
+ }
+ var regex = patternParts[1];
+ var k = '' + regex;
+ if (!regexKeys.hasOwnProperty(k)) {
+ allRegexs.push(regex);
+ regexKeys[k] = null;
+ }
+ }
+ allRegexs.push(/[\0-\uffff]/);
+ tokenizer = combinePrefixPatterns(allRegexs);
+ })();
+
+ var nPatterns = fallthroughStylePatterns.length;
+
+ /**
+ * Lexes job.sourceCode and produces an output array job.decorations of
+ * style classes preceded by the position at which they start in
+ * job.sourceCode in order.
+ *
+ * @param {Object} job an object like <pre>{
+ * sourceCode: {string} sourceText plain text,
+ * basePos: {int} position of job.sourceCode in the larger chunk of
+ * sourceCode.
+ * }</pre>
+ */
+ var decorate = function (job) {
+ var sourceCode = job.sourceCode, basePos = job.basePos;
+ /** Even entries are positions in source in ascending order. Odd enties
+ * are style markers (e.g., PR_COMMENT) that run from that position until
+ * the end.
+ * @type {Array.<number|string>}
+ */
+ var decorations = [basePos, PR_PLAIN];
+ var pos = 0; // index into sourceCode
+ var tokens = sourceCode.match(tokenizer) || [];
+ var styleCache = {};
+
+ for (var ti = 0, nTokens = tokens.length; ti < nTokens; ++ti) {
+ var token = tokens[ti];
+ var style = styleCache[token];
+ var match = void 0;
+
+ var isEmbedded;
+ if (typeof style === 'string') {
+ isEmbedded = false;
} else {
- switch (state) {
- case 0:
- if ('<' == ch) { state = 1; }
- break;
- case 1:
- if (PR_isSpaceChar(ch)) { state = 2; }
- break;
- case 2:
- if (!PR_isSpaceChar(ch)) {
- nextStyle = PR_ATTRIB_NAME;
- emitEnd = i;
- state = 3;
- }
- break;
- case 3:
- if ('=' == ch) {
- emitEnd = i;
- nextStyle = PR_TAG;
- state = 5;
- } else if (PR_isSpaceChar(ch)) {
- emitEnd = i;
- nextStyle = PR_TAG;
- state = 4;
- }
- break;
- case 4:
- if ('=' == ch) {
- state = 5;
- } else if (!PR_isSpaceChar(ch)) {
- emitEnd = i;
- nextStyle = PR_ATTRIB_NAME;
- state = 3;
- }
- break;
- case 5:
- if ('"' == ch || '\'' == ch) {
- emitEnd = i;
- nextStyle = PR_ATTRIB_VALUE;
- state = 6;
- delim = ch;
- } else if (!PR_isSpaceChar(ch)) {
- emitEnd = i;
- nextStyle = PR_ATTRIB_VALUE;
- state = 7;
- }
- break;
- case 6:
- if (ch == delim) {
- emitEnd = next;
- nextStyle = PR_TAG;
- state = 2;
- }
- break;
- case 7:
- if (PR_isSpaceChar(ch)) {
- emitEnd = i;
- nextStyle = PR_TAG;
- state = 2;
+ var patternParts = shortcuts[token.charAt(0)];
+ if (patternParts) {
+ match = token.match(patternParts[1]);
+ style = patternParts[0];
+ } else {
+ for (var i = 0; i < nPatterns; ++i) {
+ patternParts = fallthroughStylePatterns[i];
+ match = token.match(patternParts[1]);
+ if (match) {
+ style = patternParts[0];
+ break;
}
- break;
- }
- }
- if (emitEnd) {
- if (emitEnd > start) {
- tokensOut.push(
- new PR_Token(s.substring(start, emitEnd), stateStyle));
- start = emitEnd;
- }
- stateStyle = nextStyle;
- }
- i = next;
- }
- if (s.length > start) {
- tokensOut.push(new PR_Token(s.substring(start, s.length), stateStyle));
- }
- } else {
- if (tok.style) {
- state = 0;
- stateStyle = PR_TAG;
- }
- tokensOut.push(tok);
- }
- }
- return tokensOut;
-}
+ }
-/** identify regions of markup that are really source code, and recursivley
- * lex them.
- * @private
- */
-function PR_splitSourceNodes(tokens) {
- var tokensOut = [];
- // when we see a <script> tag, store '/' here so that we know to end the
- // source processing
- var endScriptTag = null;
- var decodeHelper = new PR_DecodeHelper();
-
- var sourceChunks = null;
-
- for (var ci = 0, nc = tokens.length; /* break below */; ++ci) {
- var tok;
-
- if (ci < nc) {
- tok = tokens[ci];
- if (null == tok.style) {
- tokens.push(tok);
- continue;
- }
- } else if (!endScriptTag) {
- break;
- } else {
- // else pretend there's an end tag so we can gracefully handle
- // unclosed source blocks
- tok = new PR_Token('', null);
- }
-
- var s = tok.token;
-
- if (null == endScriptTag) {
- if (PR_SOURCE == tok.style) {
- // split off any starting and trailing <?, <%
- if ('<' == decodeHelper.decode(s, 0)) {
- decodeHelper.decode(s, decodeHelper.next);
- if ('%' == decodeHelper.ch || '?' == decodeHelper.ch) {
- endScriptTag = decodeHelper.ch;
- tokensOut.push(new PR_Token(s.substring(0, decodeHelper.next),
- PR_TAG));
- s = s.substring(decodeHelper.next, s.length);
- }
- }
- } else if (PR_TAG == tok.style) {
- if ('<' == decodeHelper.decode(s, 0) &&
- '/' != s.charAt(decodeHelper.next)) {
- var tagContent = s.substring(decodeHelper.next).toLowerCase();
- // FIXME(msamuel): this does not mirror exactly the code in
- // in PR_splitMarkup that defers splitting tags inside script and
- // style blocks.
- if (PR_startsWith(tagContent, 'script') ||
- PR_startsWith(tagContent, 'style') ||
- PR_startsWith(tagContent, 'xmp')) {
- endScriptTag = '/';
+ if (!match) { // make sure that we make progress
+ style = PR_PLAIN;
+ }
}
- }
- }
- }
- if (null != endScriptTag) {
- var endTok = null;
- if (PR_SOURCE == tok.style) {
- if (endScriptTag == '%' || endScriptTag == '?') {
- var pos = s.lastIndexOf(endScriptTag);
- if (pos >= 0 && '>' == decodeHelper.decode(s, pos + 1) &&
- s.length == decodeHelper.next) {
- endTok = new PR_Token(s.substring(pos, s.length), PR_TAG);
- s = s.substring(0, pos);
- }
- }
- if (null == sourceChunks) { sourceChunks = []; }
- sourceChunks.push(new PR_Token(s, PR_PLAIN));
- } else if (PR_PLAIN == tok.style) {
- if (null == sourceChunks) { sourceChunks = []; }
- sourceChunks.push(tok);
- } else if (PR_TAG == tok.style) {
- // if it starts with </ then it must be the end tag.
- if ('<' == decodeHelper.decode(tok.token, 0) &&
- tok.token.length > decodeHelper.next &&
- '/' == decodeHelper.decode(tok.token, decodeHelper.next)) {
- endTok = tok;
+ isEmbedded = style.length >= 5 && 'lang-' === style.substring(0, 5);
+ if (isEmbedded && !(match && typeof match[1] === 'string')) {
+ isEmbedded = false;
+ style = PR_SOURCE;
+ }
+
+ if (!isEmbedded) { styleCache[token] = style; }
+ }
+
+ var tokenStart = pos;
+ pos += token.length;
+
+ if (!isEmbedded) {
+ decorations.push(basePos + tokenStart, style);
+ } else { // Treat group 1 as an embedded block of source code.
+ var embeddedSource = match[1];
+ var embeddedSourceStart = token.indexOf(embeddedSource);
+ var embeddedSourceEnd = embeddedSourceStart + embeddedSource.length;
+ if (match[2]) {
+ // If embeddedSource can be blank, then it would match at the
+ // beginning which would cause us to infinitely recurse on the
+ // entire token, so we catch the right context in match[2].
+ embeddedSourceEnd = token.length - match[2].length;
+ embeddedSourceStart = embeddedSourceEnd - embeddedSource.length;
+ }
+ var lang = style.substring(5);
+ // Decorate the left of the embedded source
+ appendDecorations(
+ basePos + tokenStart,
+ token.substring(0, embeddedSourceStart),
+ decorate, decorations);
+ // Decorate the embedded source
+ appendDecorations(
+ basePos + tokenStart + embeddedSourceStart,
+ embeddedSource,
+ langHandlerForExtension(lang, embeddedSource),
+ decorations);
+ // Decorate the right of the embedded section
+ appendDecorations(
+ basePos + tokenStart + embeddedSourceEnd,
+ token.substring(embeddedSourceEnd),
+ decorate, decorations);
+ }
+ }
+ job.decorations = decorations;
+ };
+ return decorate;
+ }
+
+ /** returns a function that produces a list of decorations from source text.
+ *
+ * This code treats ", ', and ` as string delimiters, and \ as a string
+ * escape. It does not recognize perl's qq() style strings.
+ * It has no special handling for double delimiter escapes as in basic, or
+ * the tripled delimiters used in python, but should work on those regardless
+ * although in those cases a single string literal may be broken up into
+ * multiple adjacent string literals.
+ *
+ * It recognizes C, C++, and shell style comments.
+ *
+ * @param {Object} options a set of optional parameters.
+ * @return {function (Object)} a function that examines the source code
+ * in the input job and builds the decoration list.
+ */
+ function sourceDecorator(options) {
+ var shortcutStylePatterns = [], fallthroughStylePatterns = [];
+ if (options['tripleQuotedStrings']) {
+ // '''multi-line-string''', 'single-line-string', and double-quoted
+ shortcutStylePatterns.push(
+ [PR_STRING, /^(?:\'\'\'(?:[^\'\\]|\\[\s\S]|\'{1,2}(?=[^\']))*(?:\'\'\'|$)|\"\"\"(?:[^\"\\]|\\[\s\S]|\"{1,2}(?=[^\"]))*(?:\"\"\"|$)|\'(?:[^\\\']|\\[\s\S])*(?:\'|$)|\"(?:[^\\\"]|\\[\s\S])*(?:\"|$))/,
+ null, '\'"']);
+ } else if (options['multiLineStrings']) {
+ // 'multi-line-string', "multi-line-string"
+ shortcutStylePatterns.push(
+ [PR_STRING, /^(?:\'(?:[^\\\']|\\[\s\S])*(?:\'|$)|\"(?:[^\\\"]|\\[\s\S])*(?:\"|$)|\`(?:[^\\\`]|\\[\s\S])*(?:\`|$))/,
+ null, '\'"`']);
+ } else {
+ // 'single-line-string', "single-line-string"
+ shortcutStylePatterns.push(
+ [PR_STRING,
+ /^(?:\'(?:[^\\\'\r\n]|\\.)*(?:\'|$)|\"(?:[^\\\"\r\n]|\\.)*(?:\"|$))/,
+ null, '"\'']);
+ }
+ if (options['verbatimStrings']) {
+ // verbatim-string-literal production from the C# grammar. See issue 93.
+ fallthroughStylePatterns.push(
+ [PR_STRING, /^@\"(?:[^\"]|\"\")*(?:\"|$)/, null]);
+ }
+ var hc = options['hashComments'];
+ if (hc) {
+ if (options['cStyleComments']) {
+ if (hc > 1) { // multiline hash comments
+ shortcutStylePatterns.push(
+ [PR_COMMENT, /^#(?:##(?:[^#]|#(?!##))*(?:###|$)|.*)/, null, '#']);
} else {
- tokensOut.push(tok);
- }
- } else if (ci >= nc) {
- // force the token to close
- endTok = tok;
+ // Stop C preprocessor declarations at an unclosed open comment
+ shortcutStylePatterns.push(
+ [PR_COMMENT, /^#(?:(?:define|e(?:l|nd)if|else|error|ifn?def|include|line|pragma|undef|warning)\b|[^\r\n]*)/,
+ null, '#']);
+ }
+ // #include <stdio.h>
+ fallthroughStylePatterns.push(
+ [PR_STRING,
+ /^<(?:(?:(?:\.\.\/)*|\/?)(?:[\w-]+(?:\/[\w-]+)+)?[\w-]+\.h(?:h|pp|\+\+)?|[a-z]\w*)>/,
+ null]);
} else {
- if (sourceChunks) {
- sourceChunks.push(tok);
- } else {
- // push remaining tag and attribute tokens from the opening tag
- tokensOut.push(tok);
- }
+ shortcutStylePatterns.push([PR_COMMENT, /^#[^\r\n]*/, null, '#']);
}
- if (endTok) {
- if (sourceChunks) {
- var sourceTokens = PR_lexSource(sourceChunks);
- tokensOut.push(new PR_Token('<span class=embsrc>', null));
- for (var si = 0, ns = sourceTokens.length; si < ns; ++si) {
- tokensOut.push(sourceTokens[si]);
- }
- tokensOut.push(new PR_Token('</span>', null));
- sourceChunks = null;
- }
- if (endTok.token) { tokensOut.push(endTok); }
- endScriptTag = null;
- }
- } else {
- tokensOut.push(tok);
- }
- }
- return tokensOut;
-}
-
-/** splits the quotes from an attribute value.
- * ['"foo"'] -> ['"', 'foo', '"']
- * @private
- */
-function PR_splitAttributeQuotes(tokens) {
- var firstPlain = null, lastPlain = null;
- for (var i = 0; i < tokens.length; ++i) {
- if (PR_PLAIN == tokens[i].style) {
- firstPlain = i;
- break;
- }
- }
- for (var i = tokens.length; --i >= 0;) {
- if (PR_PLAIN == tokens[i].style) {
- lastPlain = i;
- break;
- }
- }
- if (null == firstPlain) { return tokens; }
-
- var decodeHelper = new PR_DecodeHelper();
- var fs = tokens[firstPlain].token;
- var fc = decodeHelper.decode(fs, 0);
- if ('"' != fc && '\'' != fc) {
- return tokens;
- }
- var fpos = decodeHelper.next;
-
- var ls = tokens[lastPlain].token;
- var lpos = ls.lastIndexOf('&');
- if (lpos < 0) { lpos = ls.length - 1; }
- var lc = decodeHelper.decode(ls, lpos);
- if (lc != fc || decodeHelper.next != ls.length) {
- lc = null;
- lpos = ls.length;
- }
-
- var tokensOut = [];
- for (var i = 0; i < firstPlain; ++i) {
- tokensOut.push(tokens[i]);
- }
- tokensOut.push(new PR_Token(fs.substring(0, fpos), PR_ATTRIB_VALUE));
- if (lastPlain == firstPlain) {
- tokensOut.push(new PR_Token(fs.substring(fpos, lpos), PR_PLAIN));
- } else {
- tokensOut.push(new PR_Token(fs.substring(fpos, fs.length), PR_PLAIN));
- for (var i = firstPlain + 1; i < lastPlain; ++i) {
- tokensOut.push(tokens[i]);
}
- if (lc) {
- tokens.push(new PR_Token(ls.substring(0, lpos), PR_PLAIN));
- } else {
- tokens.push(tokens[lastPlain]);
- }
- }
- if (lc) {
- tokensOut.push(new PR_Token(ls.substring(lpos, ls.length), PR_PLAIN));
- }
- for (var i = lastPlain + 1; i < tokens.length; ++i) {
- tokensOut.push(tokens[i]);
- }
- return tokensOut;
-}
+ if (options['cStyleComments']) {
+ fallthroughStylePatterns.push([PR_COMMENT, /^\/\/[^\r\n]*/, null]);
+ fallthroughStylePatterns.push(
+ [PR_COMMENT, /^\/\*[\s\S]*?(?:\*\/|$)/, null]);
+ }
+ var regexLiterals = options['regexLiterals'];
+ if (regexLiterals) {
+ /**
+ * @const
+ */
+ var regexExcls = regexLiterals > 1
+ ? '' // Multiline regex literals
+ : '\n\r';
+ /**
+ * @const
+ */
+ var regexAny = regexExcls ? '.' : '[\\S\\s]';
+ /**
+ * @const
+ */
+ var REGEX_LITERAL = (
+ // A regular expression literal starts with a slash that is
+ // not followed by * or / so that it is not confused with
+ // comments.
+ '/(?=[^/*' + regexExcls + '])'
+ // and then contains any number of raw characters,
+ + '(?:[^/\\x5B\\x5C' + regexExcls + ']'
+ // escape sequences (\x5C),
+ + '|\\x5C' + regexAny
+ // or non-nesting character sets (\x5B\x5D);
+ + '|\\x5B(?:[^\\x5C\\x5D' + regexExcls + ']'
+ + '|\\x5C' + regexAny + ')*(?:\\x5D|$))+'
+ // finally closed by a /.
+ + '/');
+ fallthroughStylePatterns.push(
+ ['lang-regex',
+ RegExp('^' + REGEXP_PRECEDER_PATTERN + '(' + REGEX_LITERAL + ')')
+ ]);
+ }
+
+ var types = options['types'];
+ if (types) {
+ fallthroughStylePatterns.push([PR_TYPE, types]);
+ }
+
+ var keywords = ("" + options['keywords']).replace(/^ | $/g, '');
+ if (keywords.length) {
+ fallthroughStylePatterns.push(
+ [PR_KEYWORD,
+ new RegExp('^(?:' + keywords.replace(/[\s,]+/g, '|') + ')\\b'),
+ null]);
+ }
+
+ shortcutStylePatterns.push([PR_PLAIN, /^\s+/, null, ' \r\n\t\xA0']);
+
+ var punctuation =
+ // The Bash man page says
+
+ // A word is a sequence of characters considered as a single
+ // unit by GRUB. Words are separated by metacharacters,
+ // which are the following plus space, tab, and newline: { }
+ // | & $ ; < >
+ // ...
+
+ // A word beginning with # causes that word and all remaining
+ // characters on that line to be ignored.
-/** identify attribute values that really contain source code and recursively
- * lex them.
- * @private
- */
-function PR_splitSourceAttributes(tokens) {
- var tokensOut = [];
-
- var sourceChunks = null;
- var inSource = false;
- var name = '';
-
- for (var ci = 0, nc = tokens.length; ci < nc; ++ci) {
- var tok = tokens[ci];
- var outList = tokensOut;
- if (PR_TAG == tok.style) {
- if (inSource) {
- inSource = false;
- name = '';
- if (sourceChunks) {
- tokensOut.push(new PR_Token('<span class=embsrc>', null));
- var sourceTokens =
- PR_lexSource(PR_splitAttributeQuotes(sourceChunks));
- for (var si = 0, ns = sourceTokens.length; si < ns; ++si) {
- tokensOut.push(sourceTokens[si]);
- }
- tokensOut.push(new PR_Token('</span>', null));
- sourceChunks = null;
- }
- } else if (name && tok.token.indexOf('=') >= 0) {
- var nameLower = name.toLowerCase();
- if (PR_startsWith(nameLower, 'on') || 'style' == nameLower) {
- inSource = true;
+ // which means that only a '#' after /(?:^|[{}|&$;<>\s])/ starts a
+ // comment but empirically
+ // $ echo {#}
+ // {#}
+ // $ echo \$#
+ // $#
+ // $ echo }#
+ // }#
+
+ // so /(?:^|[|&;<>\s])/ is more appropriate.
+
+ // http://gcc.gnu.org/onlinedocs/gcc-2.95.3/cpp_1.html#SEC3
+ // suggests that this definition is compatible with a
+ // default mode that tries to use a single token definition
+ // to recognize both bash/python style comments and C
+ // preprocessor directives.
+
+ // This definition of punctuation does not include # in the list of
+ // follow-on exclusions, so # will not be broken before if preceeded
+ // by a punctuation character. We could try to exclude # after
+ // [|&;<>] but that doesn't seem to cause many major problems.
+ // If that does turn out to be a problem, we should change the below
+ // when hc is truthy to include # in the run of punctuation characters
+ // only when not followint [|&;<>].
+ '^.[^\\s\\w.$@\'"`/\\\\]*';
+ if (options['regexLiterals']) {
+ punctuation += '(?!\s*\/)';
+ }
+
+ fallthroughStylePatterns.push(
+ // TODO(mikesamuel): recognize non-latin letters and numerals in idents
+ [PR_LITERAL, /^@[a-z_$][a-z_$@0-9]*/i, null],
+ [PR_TYPE, /^(?:[@_]?[A-Z]+[a-z][A-Za-z_$@0-9]*|\w+_t\b)/, null],
+ [PR_PLAIN, /^[a-z_$][a-z_$@0-9]*/i, null],
+ [PR_LITERAL,
+ new RegExp(
+ '^(?:'
+ // A hex number
+ + '0x[a-f0-9]+'
+ // or an octal or decimal number,
+ + '|(?:\\d(?:_\\d+)*\\d*(?:\\.\\d*)?|\\.\\d\\+)'
+ // possibly in scientific notation
+ + '(?:e[+\\-]?\\d+)?'
+ + ')'
+ // with an optional modifier like UL for unsigned long
+ + '[a-z]*', 'i'),
+ null, '0123456789'],
+ // Don't treat escaped quotes in bash as starting strings.
+ // See issue 144.
+ [PR_PLAIN, /^\\[\s\S]?/, null],
+ [PR_PUNCTUATION, new RegExp(punctuation), null]);
+
+ return createSimpleLexer(shortcutStylePatterns, fallthroughStylePatterns);
+ }
+
+ var decorateSource = sourceDecorator({
+ 'keywords': ALL_KEYWORDS,
+ 'hashComments': true,
+ 'cStyleComments': true,
+ 'multiLineStrings': true,
+ 'regexLiterals': true
+ });
+
+ /**
+ * Given a DOM subtree, wraps it in a list, and puts each line into its own
+ * list item.
+ *
+ * @param {Node} node modified in place. Its content is pulled into an
+ * HTMLOListElement, and each line is moved into a separate list item.
+ * This requires cloning elements, so the input might not have unique
+ * IDs after numbering.
+ * @param {boolean} isPreformatted true iff white-space in text nodes should
+ * be treated as significant.
+ */
+ function numberLines(node, opt_startLineNum, isPreformatted) {
+ var nocode = /(?:^|\s)nocode(?:\s|$)/;
+ var lineBreak = /\r\n?|\n/;
+
+ var document = node.ownerDocument;
+
+ var li = document.createElement('li');
+ while (node.firstChild) {
+ li.appendChild(node.firstChild);
+ }
+ // An array of lines. We split below, so this is initialized to one
+ // un-split line.
+ var listItems = [li];
+
+ function walk(node) {
+ var type = node.nodeType;
+ if (type == 1 && !nocode.test(node.className)) { // Element
+ if ('br' === node.nodeName) {
+ breakAfter(node);
+ // Discard the <BR> since it is now flush against a </LI>.
+ if (node.parentNode) {
+ node.parentNode.removeChild(node);
+ }
+ } else {
+ for (var child = node.firstChild; child; child = child.nextSibling) {
+ walk(child);
+ }
}
+ } else if ((type == 3 || type == 4) && isPreformatted) { // Text
+ var text = node.nodeValue;
+ var match = text.match(lineBreak);
+ if (match) {
+ var firstLine = text.substring(0, match.index);
+ node.nodeValue = firstLine;
+ var tail = text.substring(match.index + match[0].length);
+ if (tail) {
+ var parent = node.parentNode;
+ parent.insertBefore(
+ document.createTextNode(tail), node.nextSibling);
+ }
+ breakAfter(node);
+ if (!firstLine) {
+ // Don't leave blank text nodes in the DOM.
+ node.parentNode.removeChild(node);
+ }
+ }
+ }
+ }
+
+ // Split a line after the given node.
+ function breakAfter(lineEndNode) {
+ // If there's nothing to the right, then we can skip ending the line
+ // here, and move root-wards since splitting just before an end-tag
+ // would require us to create a bunch of empty copies.
+ while (!lineEndNode.nextSibling) {
+ lineEndNode = lineEndNode.parentNode;
+ if (!lineEndNode) { return; }
+ }
+
+ function breakLeftOf(limit, copy) {
+ // Clone shallowly if this node needs to be on both sides of the break.
+ var rightSide = copy ? limit.cloneNode(false) : limit;
+ var parent = limit.parentNode;
+ if (parent) {
+ // We clone the parent chain.
+ // This helps us resurrect important styling elements that cross lines.
+ // E.g. in <i>Foo<br>Bar</i>
+ // should be rewritten to <li><i>Foo</i></li><li><i>Bar</i></li>.
+ var parentClone = breakLeftOf(parent, 1);
+ // Move the clone and everything to the right of the original
+ // onto the cloned parent.
+ var next = limit.nextSibling;
+ parentClone.appendChild(rightSide);
+ for (var sibling = next; sibling; sibling = next) {
+ next = sibling.nextSibling;
+ parentClone.appendChild(sibling);
+ }
+ }
+ return rightSide;
+ }
+
+ var copiedListItem = breakLeftOf(lineEndNode.nextSibling, 0);
+
+ // Walk the parent chain until we reach an unattached LI.
+ for (var parent;
+ // Check nodeType since IE invents document fragments.
+ (parent = copiedListItem.parentNode) && parent.nodeType === 1;) {
+ copiedListItem = parent;
+ }
+ // Put it on the list of lines for later processing.
+ listItems.push(copiedListItem);
+ }
+
+ // Split lines while there are lines left to split.
+ for (var i = 0; // Number of lines that have been split so far.
+ i < listItems.length; // length updated by breakAfter calls.
+ ++i) {
+ walk(listItems[i]);
+ }
+
+ // Make sure numeric indices show correctly.
+ if (opt_startLineNum === (opt_startLineNum|0)) {
+ listItems[0].setAttribute('value', opt_startLineNum);
+ }
+
+ var ol = document.createElement('ol');
+ ol.className = 'linenums';
+ var offset = Math.max(0, ((opt_startLineNum - 1 /* zero index */)) | 0) || 0;
+ for (var i = 0, n = listItems.length; i < n; ++i) {
+ li = listItems[i];
+ // Stick a class on the LIs so that stylesheets can
+ // color odd/even rows, or any other row pattern that
+ // is co-prime with 10.
+ li.className = 'L' + ((i + offset) % 10);
+ if (!li.firstChild) {
+ li.appendChild(document.createTextNode('\xA0'));
+ }
+ ol.appendChild(li);
+ }
+
+ node.appendChild(ol);
+ }
+ /**
+ * Breaks {@code job.sourceCode} around style boundaries in
+ * {@code job.decorations} and modifies {@code job.sourceNode} in place.
+ * @param {Object} job like <pre>{
+ * sourceCode: {string} source as plain text,
+ * sourceNode: {HTMLElement} the element containing the source,
+ * spans: {Array.<number|Node>} alternating span start indices into source
+ * and the text node or element (e.g. {@code <BR>}) corresponding to that
+ * span.
+ * decorations: {Array.<number|string} an array of style classes preceded
+ * by the position at which they start in job.sourceCode in order
+ * }</pre>
+ * @private
+ */
+ function recombineTagsAndDecorations(job) {
+ var isIE8OrEarlier = /\bMSIE\s(\d+)/.exec(navigator.userAgent);
+ isIE8OrEarlier = isIE8OrEarlier && +isIE8OrEarlier[1] <= 8;
+ var newlineRe = /\n/g;
+
+ var source = job.sourceCode;
+ var sourceLength = source.length;
+ // Index into source after the last code-unit recombined.
+ var sourceIndex = 0;
+
+ var spans = job.spans;
+ var nSpans = spans.length;
+ // Index into spans after the last span which ends at or before sourceIndex.
+ var spanIndex = 0;
+
+ var decorations = job.decorations;
+ var nDecorations = decorations.length;
+ // Index into decorations after the last decoration which ends at or before
+ // sourceIndex.
+ var decorationIndex = 0;
+
+ // Remove all zero-length decorations.
+ decorations[nDecorations] = sourceLength;
+ var decPos, i;
+ for (i = decPos = 0; i < nDecorations;) {
+ if (decorations[i] !== decorations[i + 2]) {
+ decorations[decPos++] = decorations[i++];
+ decorations[decPos++] = decorations[i++];
} else {
- name = '';
- }
- } else if (PR_ATTRIB_NAME == tok.style) {
- name += tok.token;
- } else if (PR_ATTRIB_VALUE == tok.style) {
- if (inSource) {
- if (null == sourceChunks) { sourceChunks = []; }
- outList = sourceChunks;
- tok = new PR_Token(tok.token, PR_PLAIN);
- }
- } else {
- if (sourceChunks) {
- outList = sourceChunks;
+ i += 2;
}
}
- outList.push(tok);
- }
- return tokensOut;
-}
-
-/** returns a list of PR_Token objects given chunks of source code.
- *
- * This code treats ", ', and ` as string delimiters, and \ as a string escape.
- * It does not recognize perl's qq() style strings. It has no special handling
- * for double delimiter escapes as in basic, or tje tripled delimiters used in
- * python, but should work on those regardless although in those cases a single
- * string literal may be broken up into multiple adjacent string literals.
- *
- * It recognizes C, C++, and shell style comments.
- *
- * @param chunks PR_Tokens with style in (null, PR_PLAIN)
- */
-function PR_lexSource(chunks) {
- // split into strings, comments, and other.
- // We do this because strings and comments are easily recognizable and can
- // contain stuff that looks like other tokens, so we want to mark those early
- // so we don't recurse into them.
- var tokens = PR_splitStringAndCommentTokens(chunks);
-
- // split non comment|string tokens on whitespace and word boundaries
- var tokensOut = [];
- for (var i = 0; i < tokens.length; ++i) {
- var tok = tokens[i];
- if (PR_PLAIN === tok.style) {
- PR_splitNonStringNonCommentToken(tok.token, tokensOut);
- continue;
- }
- tokensOut.push(tok);
- }
-
- return tokensOut;
-}
-
-/** returns a list of PR_Token objects given a string of markup.
- *
- * This code assumes that < tokens are html escaped, but " are not.
- * It will do a resonable job with <, but will not recognize an "
- * as starting a string.
- *
- * This code recognizes a number of constructs.
- * <!-- ... --> comment
- * <!\w ... > declaration
- * <\w ... > tag
- * </\w ... > tag
- * <?...?> embedded source
- * &[#\w]...; entity
- *
- * It does not recognizes %foo; entities.
- *
- * It will recurse into any <style>, <script>, and on* attributes using
- * PR_lexSource.
- */
-function PR_lexMarkup(chunks) {
- // This function works as follows:
- // 1) Start by splitting the markup into text and tag chunks
- // Input: String s
- // Output: List<PR_Token> where style in (PR_PLAIN, null)
- // 2) Then split the text chunks further into comments, declarations,
- // tags, etc.
- // After each split, consider whether the token is the start of an
- // embedded source section, i.e. is an open <script> tag. If it is,
- // find the corresponding close token, and don't bother to lex in between.
- // Input: List<String>
- // Output: List<PR_Token> with style in (PR_TAG, PR_PLAIN, PR_SOURCE, null)
- // 3) Finally go over each tag token and split out attribute names and values.
- // Input: List<PR_Token>
- // Output: List<PR_Token> where style in
- // (PR_TAG, PR_PLAIN, PR_SOURCE, NAME, VALUE, null)
- var tokensOut = PR_tokenizeMarkup(chunks);
- tokensOut = PR_splitTagAttributes(tokensOut);
- tokensOut = PR_splitSourceNodes(tokensOut);
- tokensOut = PR_splitSourceAttributes(tokensOut);
- return tokensOut;
-}
-
-/**
- * classify the string as either source or markup and lex appropriately.
- * @param {String} html
- */
-function PR_lexOne(html) {
- var chunks = PR_expandTabs(PR_chunkify(html), PR_TAB_WIDTH);
-
- // treat it as markup if the first non whitespace character is a < and the
- // last non-whitespace character is a >
- var isMarkup = false;
- for (var i = 0; i < chunks.length; ++i) {
- if (PR_PLAIN == chunks[i].style) {
- if (PR_startsWith(PR_trim(chunks[i].token), '<')) {
- for (var j = chunks.length; --j >= 0;) {
- if (PR_PLAIN == chunks[j].style) {
- isMarkup = PR_endsWith(PR_trim(chunks[j].token), '>');
- break;
+ nDecorations = decPos;
+
+ // Simplify decorations.
+ for (i = decPos = 0; i < nDecorations;) {
+ var startPos = decorations[i];
+ // Conflate all adjacent decorations that use the same style.
+ var startDec = decorations[i + 1];
+ var end = i + 2;
+ while (end + 2 <= nDecorations && decorations[end + 1] === startDec) {
+ end += 2;
+ }
+ decorations[decPos++] = startPos;
+ decorations[decPos++] = startDec;
+ i = end;
+ }
+
+ nDecorations = decorations.length = decPos;
+
+ var sourceNode = job.sourceNode;
+ var oldDisplay;
+ if (sourceNode) {
+ oldDisplay = sourceNode.style.display;
+ sourceNode.style.display = 'none';
+ }
+ try {
+ var decoration = null;
+ while (spanIndex < nSpans) {
+ var spanStart = spans[spanIndex];
+ var spanEnd = spans[spanIndex + 2] || sourceLength;
+
+ var decEnd = decorations[decorationIndex + 2] || sourceLength;
+
+ var end = Math.min(spanEnd, decEnd);
+
+ var textNode = spans[spanIndex + 1];
+ var styledText;
+ if (textNode.nodeType !== 1 // Don't muck with <BR>s or <LI>s
+ // Don't introduce spans around empty text nodes.
+ && (styledText = source.substring(sourceIndex, end))) {
+ // This may seem bizarre, and it is. Emitting LF on IE causes the
+ // code to display with spaces instead of line breaks.
+ // Emitting Windows standard issue linebreaks (CRLF) causes a blank
+ // space to appear at the beginning of every line but the first.
+ // Emitting an old Mac OS 9 line separator makes everything spiffy.
+ if (isIE8OrEarlier) {
+ styledText = styledText.replace(newlineRe, '\r');
+ }
+ textNode.nodeValue = styledText;
+ var document = textNode.ownerDocument;
+ var span = document.createElement('span');
+ span.className = decorations[decorationIndex + 1];
+ var parentNode = textNode.parentNode;
+ parentNode.replaceChild(span, textNode);
+ span.appendChild(textNode);
+ if (sourceIndex < spanEnd) { // Split off a text node.
+ spans[spanIndex + 1] = textNode
+ // TODO: Possibly optimize by using '' if there's no flicker.
[... 572 lines stripped ...]