You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucy.apache.org by nw...@apache.org on 2014/11/23 18:43:18 UTC
[06/16] lucy-clownfish git commit: Add CommonMark source files
http://git-wip-us.apache.org/repos/asf/lucy-clownfish/blob/144f0b22/compiler/modules/CommonMark/src/scanners.h
----------------------------------------------------------------------
diff --git a/compiler/modules/CommonMark/src/scanners.h b/compiler/modules/CommonMark/src/scanners.h
new file mode 100644
index 0000000..3269112
--- /dev/null
+++ b/compiler/modules/CommonMark/src/scanners.h
@@ -0,0 +1,38 @@
+#include "cmark.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int _scan_at(int (*scanner)(const unsigned char *), chunk *c, int offset);
+int _scan_autolink_uri(const unsigned char *p);
+int _scan_autolink_email(const unsigned char *p);
+int _scan_html_tag(const unsigned char *p);
+int _scan_html_block_tag(const unsigned char *p);
+int _scan_link_url(const unsigned char *p);
+int _scan_link_title(const unsigned char *p);
+int _scan_spacechars(const unsigned char *p);
+int _scan_atx_header_start(const unsigned char *p);
+int _scan_setext_header_line(const unsigned char *p);
+int _scan_hrule(const unsigned char *p);
+int _scan_open_code_fence(const unsigned char *p);
+int _scan_close_code_fence(const unsigned char *p);
+int _scan_entity(const unsigned char *p);
+
+#define scan_autolink_uri(c, n) _scan_at(&_scan_autolink_uri, c, n)
+#define scan_autolink_email(c, n) _scan_at(&_scan_autolink_email, c, n)
+#define scan_html_tag(c, n) _scan_at(&_scan_html_tag, c, n)
+#define scan_html_block_tag(c, n) _scan_at(&_scan_html_block_tag, c, n)
+#define scan_link_url(c, n) _scan_at(&_scan_link_url, c, n)
+#define scan_link_title(c, n) _scan_at(&_scan_link_title, c, n)
+#define scan_spacechars(c, n) _scan_at(&_scan_spacechars, c, n)
+#define scan_atx_header_start(c, n) _scan_at(&_scan_atx_header_start, c, n)
+#define scan_setext_header_line(c, n) _scan_at(&_scan_setext_header_line, c, n)
+#define scan_hrule(c, n) _scan_at(&_scan_hrule, c, n)
+#define scan_open_code_fence(c, n) _scan_at(&_scan_open_code_fence, c, n)
+#define scan_close_code_fence(c, n) _scan_at(&_scan_close_code_fence, c, n)
+#define scan_entity(c, n) _scan_at(&_scan_entity, c, n)
+
+#ifdef __cplusplus
+}
+#endif
http://git-wip-us.apache.org/repos/asf/lucy-clownfish/blob/144f0b22/compiler/modules/CommonMark/src/scanners.re
----------------------------------------------------------------------
diff --git a/compiler/modules/CommonMark/src/scanners.re b/compiler/modules/CommonMark/src/scanners.re
new file mode 100644
index 0000000..5e74873
--- /dev/null
+++ b/compiler/modules/CommonMark/src/scanners.re
@@ -0,0 +1,235 @@
+#include <stdlib.h>
+#include "chunk.h"
+#include "scanners.h"
+
+int _scan_at(int (*scanner)(const unsigned char *), chunk *c, int offset)
+{
+ int res;
+ unsigned char *ptr = (unsigned char *)c->data;
+ unsigned char lim = ptr[c->len];
+
+ ptr[c->len] = '\0';
+ res = scanner(ptr + offset);
+ ptr[c->len] = lim;
+
+ return res;
+}
+
+/*!re2c
+ re2c:define:YYCTYPE = "unsigned char";
+ re2c:define:YYCURSOR = p;
+ re2c:define:YYMARKER = marker;
+ re2c:define:YYCTXMARKER = marker;
+ re2c:yyfill:enable = 0;
+
+ wordchar = [^\x00-\x20];
+
+ spacechar = [ \t\n];
+
+ reg_char = [^\\()\x00-\x20];
+
+ escaped_char = [\\][!"#$%&'()*+,./:;<=>?@[\\\]^_`{|}~-];
+
+ tagname = [A-Za-z][A-Za-z0-9]*;
+
+ blocktagname = 'article'|'header'|'aside'|'hgroup'|'iframe'|'blockquote'|'hr'|'body'|'li'|'map'|'button'|'object'|'canvas'|'ol'|'caption'|'output'|'col'|'p'|'colgroup'|'pre'|'dd'|'progress'|'div'|'section'|'dl'|'table'|'td'|'dt'|'tbody'|'embed'|'textarea'|'fieldset'|'tfoot'|'figcaption'|'th'|'figure'|'thead'|'footer'|'footer'|'tr'|'form'|'ul'|'h1'|'h2'|'h3'|'h4'|'h5'|'h6'|'video'|'script'|'style';
+
+ attributename = [a-zA-Z_:][a-zA-Z0-9:._-]*;
+
+ unquotedvalue = [^\"'=<>`\x00]+;
+ singlequotedvalue = ['][^'\x00]*['];
+ doublequotedvalue = [\"][^\"\x00]*[\"];
+
+ attributevalue = unquotedvalue | singlequotedvalue | doublequotedvalue;
+
+ attributevaluespec = spacechar* [=] spacechar* attributevalue;
+
+ attribute = spacechar+ attributename attributevaluespec?;
+
+ opentag = tagname attribute* spacechar* [/]? [>];
+ closetag = [/] tagname spacechar* [>];
+
+ htmlcomment = "!--" ([^-\x00]+ | [-][^-\x00]+)* "-->";
+
+ processinginstruction = "?" ([^?>\x00]+ | [?][^>\x00] | [>])* "?>";
+
+ declaration = "!" [A-Z]+ spacechar+ [^>\x00]* ">";
+
+ cdata = "![CDATA[" ([^\]\x00]+ | "]" [^\]\x00] | "]]" [^>\x00])* "]]>";
+
+ htmltag = opentag | closetag | htmlcomment | processinginstruction |
+ declaration | cdata;
+
+ in_parens_nosp = [(] (reg_char|escaped_char)* [)];
+
+ in_double_quotes = ["] (escaped_char|[^"\x00])* ["];
+ in_single_quotes = ['] (escaped_char|[^'\x00])* ['];
+ in_parens = [(] (escaped_char|[^)\x00])* [)];
+
+ scheme = 'coap'|'doi'|'javascript'|'aaa'|'aaas'|'about'|'acap'|'cap'|'cid'|'crid'|'data'|'dav'|'dict'|'dns'|'file'|'ftp'|'geo'|'go'|'gopher'|'h323'|'http'|'https'|'iax'|'icap'|'im'|'imap'|'info'|'ipp'|'iris'|'iris.beep'|'iris.xpc'|'iris.xpcs'|'iris.lwz'|'ldap'|'mailto'|'mid'|'msrp'|'msrps'|'mtqp'|'mupdate'|'news'|'nfs'|'ni'|'nih'|'nntp'|'opaquelocktoken'|'pop'|'pres'|'rtsp'|'service'|'session'|'shttp'|'sieve'|'sip'|'sips'|'sms'|'snmp'|'soap.beep'|'soap.beeps'|'tag'|'tel'|'telnet'|'tftp'|'thismessage'|'tn3270'|'tip'|'tv'|'urn'|'vemmi'|'ws'|'wss'|'xcon'|'xcon-userid'|'xmlrpc.beep'|'xmlrpc.beeps'|'xmpp'|'z39.50r'|'z39.50s'|'adiumxtra'|'afp'|'afs'|'aim'|'apt'|'attachment'|'aw'|'beshare'|'bitcoin'|'bolo'|'callto'|'chrome'|'chrome-extension'|'com-eventbrite-attendee'|'content'|'cvs'|'dlna-playsingle'|'dlna-playcontainer'|'dtn'|'dvb'|'ed2k'|'facetime'|'feed'|'finger'|'fish'|'gg'|'git'|'gizmoproject'|'gtalk'|'hcp'|'icon'|'ipn'|'irc'|'irc6'|'ircs'|'itms'|'jar'|'jms'|'keyparc'|'lastfm'|'lda
ps'|'magnet'|'maps'|'market'|'message'|'mms'|'ms-help'|'msnim'|'mumble'|'mvn'|'notes'|'oid'|'palm'|'paparazzi'|'platform'|'proxy'|'psyc'|'query'|'res'|'resource'|'rmi'|'rsync'|'rtmp'|'secondlife'|'sftp'|'sgn'|'skype'|'smb'|'soldat'|'spotify'|'ssh'|'steam'|'svn'|'teamspeak'|'things'|'udp'|'unreal'|'ut2004'|'ventrilo'|'view-source'|'webcal'|'wtai'|'wyciwyg'|'xfire'|'xri'|'ymsgr';
+*/
+
+// Try to match URI autolink after first <, returning number of chars matched.
+int _scan_autolink_uri(const unsigned char *p)
+{
+ const unsigned char *marker = NULL;
+ const unsigned char *start = p;
+/*!re2c
+ scheme [:]([^\x00-\x20<>\\]|escaped_char)*[>] { return (p - start); }
+ .? { return 0; }
+*/
+}
+
+// Try to match email autolink after first <, returning num of chars matched.
+int _scan_autolink_email(const unsigned char *p)
+{
+ const unsigned char *marker = NULL;
+ const unsigned char *start = p;
+/*!re2c
+ [a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+
+ [@]
+ [a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?
+ ([.][a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*
+ [>] { return (p - start); }
+ .? { return 0; }
+*/
+}
+
+// Try to match an HTML tag after first <, returning num of chars matched.
+int _scan_html_tag(const unsigned char *p)
+{
+ const unsigned char *marker = NULL;
+ const unsigned char *start = p;
+/*!re2c
+ htmltag { return (p - start); }
+ .? { return 0; }
+*/
+}
+
+// Try to match an HTML block tag including first <,
+// returning num of chars matched.
+int _scan_html_block_tag(const unsigned char *p)
+{
+ const unsigned char *marker = NULL;
+ const unsigned char *start = p;
+/*!re2c
+ [<] [/] blocktagname (spacechar | [>]) { return (p - start); }
+ [<] blocktagname (spacechar | [/>]) { return (p - start); }
+ [<] [!?] { return (p - start); }
+ .? { return 0; }
+*/
+}
+
+// Try to match a URL in a link or reference, return number of chars matched.
+// This may optionally be contained in <..>; otherwise
+// whitespace and unbalanced right parentheses aren't allowed.
+// Newlines aren't ever allowed.
+int _scan_link_url(const unsigned char *p)
+{
+ const unsigned char *marker = NULL;
+ const unsigned char *start = p;
+/*!re2c
+ [ \n]* [<] ([^<>\n\\\x00] | escaped_char | [\\])* [>] { return (p - start); }
+ [ \n]* (reg_char+ | escaped_char | in_parens_nosp)* { return (p - start); }
+ .? { return 0; }
+*/
+}
+
+// Try to match a link title (in single quotes, in double quotes, or
+// in parentheses), returning number of chars matched. Allow one
+// level of internal nesting (quotes within quotes).
+int _scan_link_title(const unsigned char *p)
+{
+ const unsigned char *marker = NULL;
+ const unsigned char *start = p;
+/*!re2c
+ ["] (escaped_char|[^"\x00])* ["] { return (p - start); }
+ ['] (escaped_char|[^'\x00])* ['] { return (p - start); }
+ [(] (escaped_char|[^)\x00])* [)] { return (p - start); }
+ .? { return 0; }
+*/
+}
+
+// Match space characters, including newlines.
+int _scan_spacechars(const unsigned char *p)
+{
+ const unsigned char *start = p; \
+/*!re2c
+ [ \t\n]* { return (p - start); }
+ . { return 0; }
+*/
+}
+
+// Match ATX header start.
+int _scan_atx_header_start(const unsigned char *p)
+{
+ const unsigned char *marker = NULL;
+ const unsigned char *start = p;
+/*!re2c
+ [#]{1,6} ([ ]+|[\n]) { return (p - start); }
+ .? { return 0; }
+*/
+}
+
+// Match sexext header line. Return 1 for level-1 header,
+// 2 for level-2, 0 for no match.
+int _scan_setext_header_line(const unsigned char *p)
+{
+ const unsigned char *marker = NULL;
+/*!re2c
+ [=]+ [ ]* [\n] { return 1; }
+ [-]+ [ ]* [\n] { return 2; }
+ .? { return 0; }
+*/
+}
+
+// Scan a horizontal rule line: "...three or more hyphens, asterisks,
+// or underscores on a line by themselves. If you wish, you may use
+// spaces between the hyphens or asterisks."
+int _scan_hrule(const unsigned char *p)
+{
+ const unsigned char *marker = NULL;
+ const unsigned char *start = p;
+/*!re2c
+ ([*][ ]*){3,} [ \t]* [\n] { return (p - start); }
+ ([_][ ]*){3,} [ \t]* [\n] { return (p - start); }
+ ([-][ ]*){3,} [ \t]* [\n] { return (p - start); }
+ .? { return 0; }
+*/
+}
+
+// Scan an opening code fence.
+int _scan_open_code_fence(const unsigned char *p)
+{
+ const unsigned char *marker = NULL;
+ const unsigned char *start = p;
+/*!re2c
+ [`]{3,} / [^`\n\x00]*[\n] { return (p - start); }
+ [~]{3,} / [^~\n\x00]*[\n] { return (p - start); }
+ .? { return 0; }
+*/
+}
+
+// Scan a closing code fence with length at least len.
+int _scan_close_code_fence(const unsigned char *p)
+{
+ const unsigned char *marker = NULL;
+ const unsigned char *start = p;
+/*!re2c
+ ([`]{3,} | [~]{3,}) / spacechar* [\n] { return (p - start); }
+ .? { return 0; }
+*/
+}
+
+// Scans an entity.
+// Returns number of chars matched.
+int _scan_entity(const unsigned char *p)
+{
+ const unsigned char *marker = NULL;
+ const unsigned char *start = p;
+/*!re2c
+ [&] ([#] ([Xx][A-Fa-f0-9]{1,8}|[0-9]{1,8}) |[A-Za-z][A-Za-z0-9]{1,31} ) [;]
+ { return (p - start); }
+ .? { return 0; }
+*/
+}
http://git-wip-us.apache.org/repos/asf/lucy-clownfish/blob/144f0b22/compiler/modules/CommonMark/src/utf8.c
----------------------------------------------------------------------
diff --git a/compiler/modules/CommonMark/src/utf8.c b/compiler/modules/CommonMark/src/utf8.c
new file mode 100644
index 0000000..e144c72
--- /dev/null
+++ b/compiler/modules/CommonMark/src/utf8.c
@@ -0,0 +1,403 @@
+#include <stdlib.h>
+#include <stdint.h>
+#include <assert.h>
+
+#include "utf8.h"
+
+static const int8_t utf8proc_utf8class[256] = {
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0 };
+
+static void encode_unknown(strbuf *buf)
+{
+ static const uint8_t repl[] = {239, 191, 189};
+ strbuf_put(buf, repl, 3);
+}
+
+int utf8proc_charlen(const uint8_t *str, int str_len)
+{
+ int length, i;
+
+ if (!str_len)
+ return 0;
+
+ length = utf8proc_utf8class[str[0]];
+
+ if (!length)
+ return -1;
+
+ if (str_len >= 0 && length > str_len)
+ return -str_len;
+
+ for (i = 1; i < length; i++) {
+ if ((str[i] & 0xC0) != 0x80)
+ return -i;
+ }
+
+ return length;
+}
+
+void utf8proc_detab(strbuf *ob, const uint8_t *line, size_t size)
+{
+ static const uint8_t whitespace[] = " ";
+
+ size_t i = 0, tab = 0;
+
+ while (i < size) {
+ size_t org = i;
+
+ while (i < size && line[i] != '\t' && line[i] <= 0x80) {
+ i++; tab++;
+ }
+
+ if (i > org)
+ strbuf_put(ob, line + org, i - org);
+
+ if (i >= size)
+ break;
+
+ if (line[i] == '\t') {
+ int numspaces = 4 - (tab % 4);
+ strbuf_put(ob, whitespace, numspaces);
+ i += 1;
+ tab += numspaces;
+ } else {
+ int charlen = utf8proc_charlen(line + i, size - i);
+
+ if (charlen >= 0) {
+ strbuf_put(ob, line + i, charlen);
+ } else {
+ encode_unknown(ob);
+ charlen = -charlen;
+ }
+
+ i += charlen;
+ tab += 1;
+ }
+ }
+}
+
+int utf8proc_iterate(const uint8_t *str, int str_len, int32_t *dst)
+{
+ int length;
+ int32_t uc = -1;
+
+ *dst = -1;
+ length = utf8proc_charlen(str, str_len);
+ if (length < 0)
+ return -1;
+
+ switch (length) {
+ case 1:
+ uc = str[0];
+ break;
+ case 2:
+ uc = ((str[0] & 0x1F) << 6) + (str[1] & 0x3F);
+ if (uc < 0x80) uc = -1;
+ break;
+ case 3:
+ uc = ((str[0] & 0x0F) << 12) + ((str[1] & 0x3F) << 6)
+ + (str[2] & 0x3F);
+ if (uc < 0x800 || (uc >= 0xD800 && uc < 0xE000) ||
+ (uc >= 0xFDD0 && uc < 0xFDF0)) uc = -1;
+ break;
+ case 4:
+ uc = ((str[0] & 0x07) << 18) + ((str[1] & 0x3F) << 12)
+ + ((str[2] & 0x3F) << 6) + (str[3] & 0x3F);
+ if (uc < 0x10000 || uc >= 0x110000) uc = -1;
+ break;
+ }
+
+ if (uc < 0 || ((uc & 0xFFFF) >= 0xFFFE))
+ return -1;
+
+ *dst = uc;
+ return length;
+}
+
+void utf8proc_encode_char(int32_t uc, strbuf *buf)
+{
+ uint8_t dst[4];
+ int len = 0;
+
+ assert(uc >= 0);
+
+ if (uc < 0x80) {
+ dst[0] = uc;
+ len = 1;
+ } else if (uc < 0x800) {
+ dst[0] = 0xC0 + (uc >> 6);
+ dst[1] = 0x80 + (uc & 0x3F);
+ len = 2;
+ } else if (uc == 0xFFFF) {
+ dst[0] = 0xFF;
+ len = 1;
+ } else if (uc == 0xFFFE) {
+ dst[0] = 0xFE;
+ len = 1;
+ } else if (uc < 0x10000) {
+ dst[0] = 0xE0 + (uc >> 12);
+ dst[1] = 0x80 + ((uc >> 6) & 0x3F);
+ dst[2] = 0x80 + (uc & 0x3F);
+ len = 3;
+ } else if (uc < 0x110000) {
+ dst[0] = 0xF0 + (uc >> 18);
+ dst[1] = 0x80 + ((uc >> 12) & 0x3F);
+ dst[2] = 0x80 + ((uc >> 6) & 0x3F);
+ dst[3] = 0x80 + (uc & 0x3F);
+ len = 4;
+ } else {
+ encode_unknown(buf);
+ return;
+ }
+
+ strbuf_put(buf, dst, len);
+}
+
+void utf8proc_case_fold(strbuf *dest, const uint8_t *str, int len)
+{
+ int32_t c;
+
+#define bufpush(x) \
+ utf8proc_encode_char(x, dest)
+
+ while (len > 0) {
+ int char_len = utf8proc_iterate(str, len, &c);
+
+ if (char_len >= 0) {
+#include "case_fold_switch.inc"
+ } else {
+ encode_unknown(dest);
+ char_len = -char_len;
+ }
+
+ str += char_len;
+ len -= char_len;
+ }
+}
+
+// matches anything in the Zs class, plus LF, CR, TAB, FF.
+int utf8proc_is_space(int32_t uc)
+{
+ return (uc == 9 ||
+ uc == 10 ||
+ uc == 12 ||
+ uc == 13 ||
+ uc == 32 ||
+ uc == 160 ||
+ uc == 5760 ||
+ (uc >= 8192 && uc <= 8202) ||
+ uc == 8239 ||
+ uc == 8287 ||
+ uc == 12288);
+}
+
+// matches anything in the P[cdefios] classes.
+int utf8proc_is_punctuation(int32_t uc)
+{
+ return ((uc >= 33 && uc <= 35) ||
+ (uc >= 37 && uc <= 42) ||
+ (uc >= 44 && uc <= 47) ||
+ uc == 58 ||
+ uc == 59 ||
+ uc == 63 ||
+ uc == 64 ||
+ (uc >= 91 && uc <= 93) ||
+ uc == 95 ||
+ uc == 123 ||
+ uc == 125 ||
+ uc == 161 ||
+ uc == 167 ||
+ uc == 171 ||
+ uc == 182 ||
+ uc == 183 ||
+ uc == 187 ||
+ uc == 191 ||
+ uc == 894 ||
+ uc == 903 ||
+ (uc >= 1370 && uc <= 1375) ||
+ uc == 1417 ||
+ uc == 1418 ||
+ uc == 1470 ||
+ uc == 1472 ||
+ uc == 1475 ||
+ uc == 1478 ||
+ uc == 1523 ||
+ uc == 1524 ||
+ uc == 1545 ||
+ uc == 1546 ||
+ uc == 1548 ||
+ uc == 1549 ||
+ uc == 1563 ||
+ uc == 1566 ||
+ uc == 1567 ||
+ (uc >= 1642 && uc <= 1645) ||
+ uc == 1748 ||
+ (uc >= 1792 && uc <= 1805) ||
+ (uc >= 2039 && uc <= 2041) ||
+ (uc >= 2096 && uc <= 2110) ||
+ uc == 2142 ||
+ uc == 2404 ||
+ uc == 2405 ||
+ uc == 2416 ||
+ uc == 2800 ||
+ uc == 3572 ||
+ uc == 3663 ||
+ uc == 3674 ||
+ uc == 3675 ||
+ (uc >= 3844 && uc <= 3858) ||
+ uc == 3860 ||
+ (uc >= 3898 && uc <= 3901) ||
+ uc == 3973 ||
+ (uc >= 4048 && uc <= 4052) ||
+ uc == 4057 ||
+ uc == 4058 ||
+ (uc >= 4170 && uc <= 4175) ||
+ uc == 4347 ||
+ (uc >= 4960 && uc <= 4968) ||
+ uc == 5120 ||
+ uc == 5741 ||
+ uc == 5742 ||
+ uc == 5787 ||
+ uc == 5788 ||
+ (uc >= 5867 && uc <= 5869) ||
+ uc == 5941 ||
+ uc == 5942 ||
+ (uc >= 6100 && uc <= 6102) ||
+ (uc >= 6104 && uc <= 6106) ||
+ (uc >= 6144 && uc <= 6154) ||
+ uc == 6468 ||
+ uc == 6469 ||
+ uc == 6686 ||
+ uc == 6687 ||
+ (uc >= 6816 && uc <= 6822) ||
+ (uc >= 6824 && uc <= 6829) ||
+ (uc >= 7002 && uc <= 7008) ||
+ (uc >= 7164 && uc <= 7167) ||
+ (uc >= 7227 && uc <= 7231) ||
+ uc == 7294 ||
+ uc == 7295 ||
+ (uc >= 7360 && uc <= 7367) ||
+ uc == 7379 ||
+ (uc >= 8208 && uc <= 8231) ||
+ (uc >= 8240 && uc <= 8259) ||
+ (uc >= 8261 && uc <= 8273) ||
+ (uc >= 8275 && uc <= 8286) ||
+ uc == 8317 ||
+ uc == 8318 ||
+ uc == 8333 ||
+ uc == 8334 ||
+ (uc >= 8968 && uc <= 8971) ||
+ uc == 9001 ||
+ uc == 9002 ||
+ (uc >= 10088 && uc <= 10101) ||
+ uc == 10181 ||
+ uc == 10182 ||
+ (uc >= 10214 && uc <= 10223) ||
+ (uc >= 10627 && uc <= 10648) ||
+ (uc >= 10712 && uc <= 10715) ||
+ uc == 10748 ||
+ uc == 10749 ||
+ (uc >= 11513 && uc <= 11516) ||
+ uc == 11518 ||
+ uc == 11519 ||
+ uc == 11632 ||
+ (uc >= 11776 && uc <= 11822) ||
+ (uc >= 11824 && uc <= 11842) ||
+ (uc >= 12289 && uc <= 12291) ||
+ (uc >= 12296 && uc <= 12305) ||
+ (uc >= 12308 && uc <= 12319) ||
+ uc == 12336 ||
+ uc == 12349 ||
+ uc == 12448 ||
+ uc == 12539 ||
+ uc == 42238 ||
+ uc == 42239 ||
+ (uc >= 42509 && uc <= 42511) ||
+ uc == 42611 ||
+ uc == 42622 ||
+ (uc >= 42738 && uc <= 42743) ||
+ (uc >= 43124 && uc <= 43127) ||
+ uc == 43214 ||
+ uc == 43215 ||
+ (uc >= 43256 && uc <= 43258) ||
+ uc == 43310 ||
+ uc == 43311 ||
+ uc == 43359 ||
+ (uc >= 43457 && uc <= 43469) ||
+ uc == 43486 ||
+ uc == 43487 ||
+ (uc >= 43612 && uc <= 43615) ||
+ uc == 43742 ||
+ uc == 43743 ||
+ uc == 43760 ||
+ uc == 43761 ||
+ uc == 44011 ||
+ uc == 64830 ||
+ uc == 64831 ||
+ (uc >= 65040 && uc <= 65049) ||
+ (uc >= 65072 && uc <= 65106) ||
+ (uc >= 65108 && uc <= 65121) ||
+ uc == 65123 ||
+ uc == 65128 ||
+ uc == 65130 ||
+ uc == 65131 ||
+ (uc >= 65281 && uc <= 65283) ||
+ (uc >= 65285 && uc <= 65290) ||
+ (uc >= 65292 && uc <= 65295) ||
+ uc == 65306 ||
+ uc == 65307 ||
+ uc == 65311 ||
+ uc == 65312 ||
+ (uc >= 65339 && uc <= 65341) ||
+ uc == 65343 ||
+ uc == 65371 ||
+ uc == 65373 ||
+ (uc >= 65375 && uc <= 65381) ||
+ (uc >= 65792 && uc <= 65794) ||
+ uc == 66463 ||
+ uc == 66512 ||
+ uc == 66927 ||
+ uc == 67671 ||
+ uc == 67871 ||
+ uc == 67903 ||
+ (uc >= 68176 && uc <= 68184) ||
+ uc == 68223 ||
+ (uc >= 68336 && uc <= 68342) ||
+ (uc >= 68409 && uc <= 68415) ||
+ (uc >= 68505 && uc <= 68508) ||
+ (uc >= 69703 && uc <= 69709) ||
+ uc == 69819 ||
+ uc == 69820 ||
+ (uc >= 69822 && uc <= 69825) ||
+ (uc >= 69952 && uc <= 69955) ||
+ uc == 70004 ||
+ uc == 70005 ||
+ (uc >= 70085 && uc <= 70088) ||
+ uc == 70093 ||
+ (uc >= 70200 && uc <= 70205) ||
+ uc == 70854 ||
+ (uc >= 71105 && uc <= 71113) ||
+ (uc >= 71233 && uc <= 71235) ||
+ (uc >= 74864 && uc <= 74868) ||
+ uc == 92782 ||
+ uc == 92783 ||
+ uc == 92917 ||
+ (uc >= 92983 && uc <= 92987) ||
+ uc == 92996 ||
+ uc == 113823);
+}
http://git-wip-us.apache.org/repos/asf/lucy-clownfish/blob/144f0b22/compiler/modules/CommonMark/src/utf8.h
----------------------------------------------------------------------
diff --git a/compiler/modules/CommonMark/src/utf8.h b/compiler/modules/CommonMark/src/utf8.h
new file mode 100644
index 0000000..319e39a
--- /dev/null
+++ b/compiler/modules/CommonMark/src/utf8.h
@@ -0,0 +1,23 @@
+#ifndef CMARK_UTF8_H
+#define CMARK_UTF8_H
+
+#include <stdint.h>
+#include "buffer.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void utf8proc_case_fold(cmark_strbuf *dest, const uint8_t *str, int len);
+void utf8proc_encode_char(int32_t uc, cmark_strbuf *buf);
+int utf8proc_iterate(const uint8_t *str, int str_len, int32_t *dst);
+int utf8proc_charlen(const uint8_t *str, int str_len);
+void utf8proc_detab(cmark_strbuf *dest, const uint8_t *line, size_t size);
+int utf8proc_is_space(int32_t uc);
+int utf8proc_is_punctuation(int32_t uc);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif