You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@tapestry.apache.org by bo...@apache.org on 2013/09/22 23:33:10 UTC
svn commit: r1525452 [2/2] - in /tapestry/tapestry-site/trunk: ./
src/main/java/org/apache/cxf/cwiki/ template/
Modified: tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/SiteExporter.java
URL: http://svn.apache.org/viewvc/tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/SiteExporter.java?rev=1525452&r1=1525451&r2=1525452&view=diff
==============================================================================
--- tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/SiteExporter.java (original)
+++ tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/SiteExporter.java Sun Sep 22 21:33:09 2013
@@ -19,11 +19,11 @@
package org.apache.cxf.cwiki;
-
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
+import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
@@ -47,12 +47,16 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;
import javax.xml.datatype.DatatypeFactory;
import javax.xml.datatype.XMLGregorianCalendar;
import javax.xml.namespace.QName;
+import javax.xml.parsers.ParserConfigurationException;
import javax.xml.ws.AsyncHandler;
import javax.xml.ws.Dispatch;
import javax.xml.ws.Response;
@@ -81,7 +85,7 @@ import org.apache.cxf.transport.http.HTT
import org.apache.cxf.transports.http.configuration.HTTPClientPolicy;
import org.apache.velocity.Template;
import org.apache.velocity.VelocityContext;
-import org.apache.velocity.app.Velocity;
+import org.apache.velocity.app.VelocityEngine;
import org.apache.velocity.runtime.resource.loader.URLResourceLoader;
import org.ccil.cowan.tagsoup.Parser;
import org.ccil.cowan.tagsoup.XMLWriter;
@@ -93,18 +97,19 @@ public class SiteExporter implements Run
static final String HOST = "https://cwiki.apache.org";
static final String ROOT = HOST + "/confluence";
- static final String RPC_ROOT = "/rpc/soap-axis/confluenceservice-v1";
+ static final String RPC_ROOT = "/rpc/soap-axis/confluenceservice-v";
static final String SOAPNS = "http://soap.rpc.confluence.atlassian.com";
+ static final String SEPARATOR = " > ";
- static final String CONFLUENCE_IMAGES = "images/confluence";
-
static boolean debug;
static String userName = "cxf-export-user";
static String password;
+ static int apiVersion = 1;
+
static boolean svn;
static boolean commit;
static StringBuilder svnCommitMessage = new StringBuilder();
@@ -113,11 +118,16 @@ public class SiteExporter implements Run
static String loginToken;
static Dispatch<Document> dispatch;
static AtomicInteger asyncCount = new AtomicInteger();
+ static Map<String, Space> spaces = new ConcurrentHashMap<String, Space>();
+ static List<SiteExporter> siteExporters;
Map<String, Page> pages = new ConcurrentHashMap<String, Page>();
Collection<Page> modifiedPages = new ConcurrentLinkedQueue<Page>();
Set<String> globalPages = new CopyOnWriteArraySet<String>();
- Set<String> blogPages = new CopyOnWriteArraySet<String>();
+
+ Map<String, BlogEntrySummary> blog = new ConcurrentHashMap<String, BlogEntrySummary>();
+ Set<BlogEntrySummary> modifiedBlog = new CopyOnWriteArraySet<BlogEntrySummary>();
+
String spaceKey = "CXF";
String pageCacheFile = "pagesConfig.obj";
@@ -129,7 +139,9 @@ public class SiteExporter implements Run
File outputDir = rootOutputDir;
Template template;
+ Space space;
+
public SiteExporter(String fileName, boolean force) throws Exception {
forceAll = force;
@@ -159,29 +171,25 @@ public class SiteExporter implements Run
String[] pgs = globals.split(",");
globalPages.addAll(Arrays.asList(pgs));
}
- if (props.containsKey("blogPages")) {
- String blogpages = props.getProperty("blogPages");
- String[] pgs = blogpages.split(",");
- blogPages.addAll(Arrays.asList(pgs));
- }
props = new Properties();
String clzName = URLResourceLoader.class.getName();
props.put("resource.loader", "url");
props.put("url.resource.loader.class", clzName);
props.put("url.resource.loader.root", "");
- synchronized (Velocity.class) {
- Velocity.init(props);
+
+ VelocityEngine engine = new VelocityEngine();
+ engine.init(props);
- URL url = ClassLoaderUtils.getResource(templateName, this.getClass());
- if (url == null) {
- File file = new File(templateName);
- if (file.exists()) {
- url = file.toURI().toURL();
- }
+ URL url = ClassLoaderUtils.getResource(templateName, this.getClass());
+ if (url == null) {
+ File file = new File(templateName);
+ if (file.exists()) {
+ url = file.toURI().toURL();
}
- template = Velocity.getTemplate(url.toURI().toString());
}
+ template = engine.getTemplate(url.toURI().toString());
+
outputDir.mkdirs();
}
@@ -190,7 +198,7 @@ public class SiteExporter implements Run
Service service = Service.create(new QName(SOAPNS, "Service"));
service.addPort(new QName(SOAPNS, "Port"),
SOAPBinding.SOAP11HTTP_BINDING,
- ROOT + RPC_ROOT);
+ ROOT + RPC_ROOT + apiVersion);
dispatch = service.createDispatch(new QName(SOAPNS, "Port"),
Document.class, Service.Mode.PAYLOAD);
@@ -216,7 +224,7 @@ public class SiteExporter implements Run
public void run() {
try {
- doExport();
+ render();
} catch (Exception e) {
e.printStackTrace();
}
@@ -226,29 +234,70 @@ public class SiteExporter implements Run
Page p = findPage(s);
if (p != null) {
pages.remove(p.getId());
- modifiedPages.add(p);
+ if (!modifiedPages.contains(p)) {
+ modifiedPages.add(p);
+ }
}
}
- public void doExport() throws Exception {
+ /**
+ * @return true if some pages have changed - rendering is needed
+ * @throws Exception
+ */
+ public boolean initialize() throws Exception {
if (!forceAll) {
- loadPagesCache();
+ loadCache();
}
-
+
// debug stuff, force regen of a page
//forcePage("Navigation");
//forcePage("Index");
//forcePage("JavaDoc");
//forcePage("DOSGi Architecture");
+ //forcePage("Book In One Page");
if (modifiedPages.isEmpty() && checkRSS()) {
System.out.println("(" + spaceKey + ") No changes detected from RSS");
- return;
+ return false;
}
-
+
doLogin();
+ checkVersion();
+ getSpace();
+ if ("-space-".equals(breadCrumbRoot)) {
+ breadCrumbRoot = space.getName();
+ }
+ loadBlog();
loadPages();
+ return true;
+ }
+
+ private void checkVersion() throws ParserConfigurationException, IOException {
+ Document doc = DOMUtils.createDocument();
+ Element el = doc.createElementNS(SOAPNS, "ns1:getServerInfo");
+ Element el2 = doc.createElement("in0");
+ el.appendChild(el2);
+ el2.setTextContent(loginToken);
+ doc.appendChild(el);
+
+ doc = getDispatch().invoke(doc);
+ el = DOMUtils.getFirstElement(DOMUtils.getFirstElement(doc.getDocumentElement()));
+ while (el != null) {
+ if ("majorVersion".equals(el.getLocalName())) {
+ String major = DOMUtils.getContent(el);
+ if (Integer.parseInt(major) >= 5) {
+ apiVersion = 2;
+ ((java.io.Closeable)dispatch).close();
+ dispatch = null;
+ }
+ }
+
+ el = DOMUtils.getNextElement(el);
+ }
+ }
+
+ protected void render() throws Exception {
for (Page p : modifiedPages) {
if (globalPages.contains(p.getTitle())) {
modifiedPages.clear();
@@ -256,18 +305,27 @@ public class SiteExporter implements Run
break;
}
}
+
if (forceAll) {
modifiedPages.clear();
modifiedPages.addAll(pages.values());
+
+ modifiedBlog.clear();
+ modifiedBlog.addAll(blog.values());
}
-
-
- if (!modifiedPages.isEmpty()) {
+ if (!modifiedBlog.isEmpty()) {
+ //blogs changed, see if any pages have blogs
+ for (Page p : pages.values()) {
+ if (p.hasBlog() && !modifiedPages.contains(p)) {
+ modifiedPages.add(p);
+ }
+ }
+ }
+ if (!modifiedPages.isEmpty() || !modifiedBlog.isEmpty()) {
+ renderBlog();
renderPages();
- savePages();
+ saveCache();
}
-
-
}
@@ -286,15 +344,15 @@ public class SiteExporter implements Run
List<Element> els = DOMUtils.getChildrenWithName(doc.getDocumentElement(),
"http://www.w3.org/2005/Atom",
"entry");
- //XMLUtils.printDOM(doc);
+ // XMLUtils.printDOM(doc);
for (Element el : els) {
Element e2 = DOMUtils.getFirstChildWithName(el, "http://www.w3.org/2005/Atom", "updated");
String val = DOMUtils.getContent(e2);
XMLGregorianCalendar cal = DatatypeFactory.newInstance().newXMLGregorianCalendar(val);
e2 = DOMUtils.getFirstChildWithName(el, "http://www.w3.org/2005/Atom", "title");
String title = DOMUtils.getContent(e2);
- Page p = findPage(title);
+ Page p = findPage(title);
if (p != null) {
//found a modified page - need to rebuild
if (cal.compare(p.getModifiedTime()) > 0) {
@@ -302,24 +360,36 @@ public class SiteExporter implements Run
return false;
}
} else {
- System.out.println("(" + spaceKey + ") Did not find page for: " + title);
- return false;
+ BlogEntrySummary entry = findBlogEntry(title);
+ if (entry != null) {
+ // we don't have modified date so just assume it's modified
+ // we'll use version number to actually figure out if page is modified or not
+ System.out.println("(" + spaceKey + ") Possible changed blog page found: " + title);
+ return false;
+ } else {
+ System.out.println("(" + spaceKey + ") Did not find page for: " + title);
+ return false;
+ }
}
}
return true;
}
- private void savePages() throws Exception {
+ private void saveCache() throws Exception {
File file = new File(rootOutputDir, pageCacheFile);
file.getParentFile().mkdirs();
FileOutputStream fout = new FileOutputStream(file);
ObjectOutputStream oout = new ObjectOutputStream(fout);
oout.writeObject(pages);
+ oout.writeObject(blog);
oout.close();
}
private void renderPages() throws Exception {
+ PageManager pageManager = new PageManager(this);
+ Renderer renderer = new Renderer(this);
+
int total = modifiedPages.size();
int count = 0;
for (Page p : modifiedPages) {
@@ -330,9 +400,13 @@ public class SiteExporter implements Run
loadPageContent(p, null, null);
VelocityContext ctx = new VelocityContext();
- ctx.put("exporter", this);
+ ctx.put("autoexport", this);
ctx.put("page", p);
+ ctx.put("body", p.getContent());
ctx.put("confluenceUri", ROOT);
+ ctx.put("pageManager", pageManager);
+ ctx.put("renderer", renderer);
+ ctx.put("exporter", this);
File file = new File(outputDir, p.createFileName());
boolean isNew = !file.exists();
@@ -348,8 +422,56 @@ public class SiteExporter implements Run
} else {
svnCommitMessage.append("Modified: " + file.getName() + "\n");
}
+
+ p.setContent(null);
+ }
+ }
+
+ private void renderBlog() throws Exception {
+ PageManager pageManager = new PageManager(this);
+ Renderer renderer = new Renderer(this);
+
+ int total = modifiedBlog.size();
+ int count = 0;
+ for (BlogEntrySummary entry : modifiedBlog) {
+ count++;
+ System.out.println("(" + spaceKey + ") Rendering Blog Entry " + entry.getTitle()
+ + " (" + count + "/" + total + ")");
+
+ loadAttachments(entry);
+ String body = renderPage(entry);
+ body = updateContentLinks(entry, body, null, mainDivClass);
+
+ pageManager.setDirectory(entry.getDirectory());
+
+ VelocityContext ctx = new VelocityContext();
+ ctx.put("autoexport", this);
+ ctx.put("page", entry);
+ ctx.put("body", body);
+ ctx.put("confluenceUri", ROOT);
+ ctx.put("pageManager", pageManager);
+ ctx.put("renderer", renderer);
+ ctx.put("exporter", this);
+ ctx.put("isBlogEntry", Boolean.TRUE);
+
+ File file = new File(outputDir, entry.getPath());
+ file.getParentFile().mkdirs();
+ boolean isNew = !file.exists();
+
+ FileWriter writer = new FileWriter(file);
+ ctx.put("out", writer);
+ template.merge(ctx, writer);
+ writer.close();
+ if (isNew) {
+ //call "svn add"
+ callSvn("add", file.getAbsolutePath());
+ svnCommitMessage.append("Adding: " + file.getName() + "\n");
+ } else {
+ svnCommitMessage.append("Modified: " + file.getName() + "\n");
+ }
}
}
+
void callSvn(String ... commands) throws Exception {
callSvn(outputDir, commands);
}
@@ -367,8 +489,8 @@ public class SiteExporter implements Run
}
}
- private void loadAttachments(Page p) throws Exception {
- Document doc = XMLUtils.newDocument();
+ private void loadAttachments(AbstractPage p) throws Exception {
+ Document doc = DOMUtils.createDocument();
Element el = doc.createElementNS(SOAPNS, "ns1:getAttachments");
Element el2 = doc.createElement("in0");
el.appendChild(el2);
@@ -389,7 +511,7 @@ public class SiteExporter implements Run
p.addAttachment(aid, filename);
- String dirName = p.createFileName();
+ String dirName = p.getPath();
dirName = dirName.substring(0, dirName.lastIndexOf(".")) + ".data";
File file = new File(outputDir, dirName);
if (!file.exists()) {
@@ -417,37 +539,20 @@ public class SiteExporter implements Run
el = DOMUtils.getNextElement(el);
}
}
- String loadIcon(String href) throws Exception {
-
- String filename = href.replace("/confluence/images", "images/confluence");
- File file = new File(outputDir + filename);
- if(file.exists())
- return file.getName();
- if(!file.getParentFile().exists()) {
- file.getParentFile().mkdirs();
- }
- FileOutputStream out = new FileOutputStream(file);
- URL url = new URL(HOST + href);
- InputStream ins = url.openStream();
- IOUtils.copy(ins, out);
- out.close();
- ins.close();
- return file.getName();
- }
- String loadUserImage(Page p, String href) throws Exception {
+ String loadUserImage(AbstractPage p, String href) throws Exception {
return loadPageBinaryData(p, href, "userimage", true);
}
- String loadThumbnail(Page p, String href) throws Exception {
+ String loadThumbnail(AbstractPage p, String href) throws Exception {
return loadPageBinaryData(p, href, "thumbs", false);
}
- String loadPageBinaryData(Page p, String href, String type, boolean auth) throws Exception {
+ String loadPageBinaryData(AbstractPage p, String href, String type, boolean auth) throws Exception {
String filename = href.substring(href.lastIndexOf('/') + 1);
filename = filename.replace(' ', '_');
if (filename.indexOf('?') != -1) {
filename = filename.substring(0, filename.indexOf('?'));
}
- String dirName = p.createFileName();
+ String dirName = p.getPath();
dirName = dirName.substring(0, dirName.lastIndexOf(".")) + "." + type;
File file = new File(outputDir, dirName);
if (!file.exists()) {
@@ -490,21 +595,13 @@ public class SiteExporter implements Run
}
}
public Page findPage(String title) throws Exception {
- for (Page p : pages.values()) {
- if (title.equals(p.getTitle())) {
- return p;
- }
- }
- return null;
+ return (Page) findByTitle(title, pages.values());
}
+
public Page findPageByURL(String url) throws Exception {
- for (Page p : pages.values()) {
- if (p.getURL().endsWith(url)) {
- return p;
- }
- }
- return null;
+ return (Page) findByURL(url, pages.values());
}
+
public Page findPageByID(String id) {
for (Page p : pages.values()) {
if (p.getId().equals(id)) {
@@ -513,14 +610,46 @@ public class SiteExporter implements Run
}
return null;
}
- public String breadcrumbs(Page page) {
- String separator = ">";
- String s = " " + separator + " ";
+ public String breadcrumbs(BlogEntrySummary page) {
+ StringBuffer buffer = new StringBuffer();
+ if (breadCrumbRoot != null) {
+ buffer.append("<a href=\"");
+ buffer.append("../../../index.html");
+ buffer.append("\">");
+ buffer.append(breadCrumbRoot);
+ buffer.append("</a>");
+ buffer.append(SEPARATOR);
+ } else {
+ buffer.append("<a href=\"../../../index.html\">Index</a>");
+ buffer.append(SEPARATOR);
+ }
+ XMLGregorianCalendar published = page.getPublished();
+ buffer.append(String.valueOf(published.getYear()));
+ buffer.append(SEPARATOR);
+ if (published.getMonth() < 10) {
+ buffer.append("0");
+ }
+ buffer.append(String.valueOf(published.getMonth()));
+ buffer.append(SEPARATOR);
+ if (published.getDay() < 10) {
+ buffer.append("0");
+ }
+ buffer.append(String.valueOf(published.getDay()));
+ buffer.append(SEPARATOR);
+ buffer.append("<a href=\"");
+ buffer.append(page.createFileName());
+ buffer.append("\">");
+ buffer.append(page.getTitle());
+ buffer.append("</a>");
+ return buffer.toString();
+ }
+
+ public String breadcrumbs(Page page) {
StringBuffer buffer = new StringBuffer();
List<Page> p = new LinkedList<Page>();
String parentId = page.getParentId();
- Page parent = parentId == null ? null : pages.get(parentId);
+ Page parent = pages.get(parentId);
while (parent != null) {
p.add(0, parent);
parentId = parent.getParentId();
@@ -532,7 +661,7 @@ public class SiteExporter implements Run
buffer.append("\">");
buffer.append(breadCrumbRoot);
buffer.append("</a>");
- buffer.append(s);
+ buffer.append(SEPARATOR);
}
for (Page p2 : p) {
buffer.append("<a href=\"");
@@ -540,7 +669,7 @@ public class SiteExporter implements Run
buffer.append("\">");
buffer.append(p2.getTitle());
buffer.append("</a>");
- buffer.append(s);
+ buffer.append(SEPARATOR);
}
buffer.append("<a href=\"");
buffer.append(page.createFileName());
@@ -575,7 +704,19 @@ public class SiteExporter implements Run
}
return p.getContent();
}
- private String loadPageContent(Page p, String divId, String divCls) throws Exception {
+ protected String loadPageContent(Page p, String divId, String divCls) throws Exception {
+ String content = renderPage(p);
+ content = updateContentLinks(p, content, divId,
+ divCls == null && divId == null ? mainDivClass : divCls);
+ if (divId == null) {
+ p.setContent(content);
+ } else {
+ p.setContentForDivId(divId, content);
+ }
+ return content;
+ }
+
+ private String renderPage(AbstractPage p) throws ParserConfigurationException {
Document doc = XMLUtils.newDocument();
Element el = doc.createElementNS(SOAPNS, "ns1:renderContent");
Element el2 = doc.createElement("in0");
@@ -609,24 +750,19 @@ public class SiteExporter implements Run
doc.appendChild(el);
doc = getDispatch().invoke(doc);
- String content = doc.getDocumentElement().getFirstChild().getTextContent().trim();
- content = updateContentLinks(p, content, divId,
- divCls == null && divId == null ? mainDivClass : divCls);
- if (divId == null) {
- p.setContent(content);
- } else {
- p.setContentForDivId(divId, content);
- }
- return content;
+ return doc.getDocumentElement().getFirstChild().getTextContent().trim();
}
public String unwrap(String v) throws Exception {
+ if (v == null) {
+ return null;
+ }
return v.trim().replaceFirst("^<div[^>]*>", "").replaceFirst("</div>$", "");
}
private static synchronized void doLogin() throws Exception {
if (loginToken == null) {
- Document doc = XMLUtils.newDocument();
+ Document doc = DOMUtils.createDocument();
Element el = doc.createElementNS(SOAPNS, "ns1:login");
Element el2 = doc.createElement("in0");
@@ -651,85 +787,138 @@ public class SiteExporter implements Run
}
}
- public void loadPagesCache() throws Exception {
+ public void loadCache() throws Exception {
File file = new File(rootOutputDir, pageCacheFile);
if (file.exists()) {
- FileInputStream fin = new FileInputStream(file);
- ObjectInputStream oin = new ObjectInputStream(fin);
- pages = CastUtils.cast((Map<?, ?>)oin.readObject());
- oin.close();
+ try {
+ FileInputStream fin = new FileInputStream(file);
+ ObjectInputStream oin = new ObjectInputStream(fin);
+ pages = CastUtils.cast((Map<?, ?>)oin.readObject());
+ blog = CastUtils.cast((Map<?, ?>)oin.readObject());
+ oin.close();
+
+ for (Page p : pages.values()) {
+ p.setExporter(this);
+ }
+ } catch (Throwable t) {
+ //invalid cache, punt
+ pages.clear();
+ blog.clear();
+ }
}
}
- private Document getPagesDocument() throws Exception {
- return getElementsDocument("ns1:getPages");
- }
-
- private Document getBlogEntriesDocument() throws Exception {
- return getElementsDocument("ns1:getBlogEntries");
- }
-
- private Document getElementsDocument(String function) throws Exception {
+ public int getBlogVersion(String pageId) throws Exception {
Document doc = XMLUtils.newDocument();
- Element el = doc.createElementNS(SOAPNS, function);
+ Element el = doc.createElementNS(SOAPNS, "ns1:getBlogEntry");
Element el2 = doc.createElement("in0");
el.appendChild(el2);
el2.setTextContent(loginToken);
el2 = doc.createElement("in1");
el.appendChild(el2);
- el2.setTextContent(spaceKey);
+ el2.setTextContent(pageId);
doc.appendChild(el);
doc = getDispatch().invoke(doc);
- return doc;
- }
-
- private void loadAndAddPages(List<Future<?>> futures, Set<String> allPages, Set<Page> newPages) throws Exception {
- Document doc = getPagesDocument();
- ElementLoader loader = new ElementLoader()
- {
- public Future<?> loadElement(Element element, Set<String> allPages, Set<Page> newPages) throws Exception
- {
- return loadPage(element, allPages, newPages);
- }
- };
- loadAndAddElements(futures, loader, doc, allPages, newPages);
- }
-
- private void loadAndAddBlogEntries(List<Future<?>> futures, Set<String> allPages, Set<Page> newPages) throws Exception {
- Document doc = getBlogEntriesDocument();
- ElementLoader loader = new ElementLoader()
- {
- public Future<?> loadElement(Element element, Set<String> allPages, Set<Page> newPages) throws Exception
- {
- return loadBlogEntry(element, allPages, newPages);
- }
- };
- loadAndAddElements(futures, loader, doc, allPages, newPages);
+
+ Node nd = doc.getDocumentElement().getFirstChild();
+
+ String version = DOMUtils.getChildContent(nd, "version");
+ return Integer.parseInt(version);
}
- private void loadAndAddElements(List<Future<?>> futures, ElementLoader loader, Document doc,
- Set<String> allPages, Set<Page> newPages) throws Exception {
+ public void loadBlog() throws Exception {
+ Document doc = DOMUtils.createDocument();
+ Element el = doc.createElementNS(SOAPNS, "ns1:getBlogEntries");
+ Element el2 = doc.createElement("in0");
+ el.appendChild(el2);
+ el2.setTextContent(loginToken);
+ el2 = doc.createElement("in1");
+ el.appendChild(el2);
+ el2.setTextContent(spaceKey);
+ doc.appendChild(el);
+ doc = getDispatch().invoke(doc);
+
+ Map<String, BlogEntrySummary> oldBlog = new ConcurrentHashMap<String, BlogEntrySummary>(blog);
+
Node nd = doc.getDocumentElement().getFirstChild().getFirstChild();
while (nd != null) {
if (nd instanceof Element) {
- futures.add(loader.loadElement((Element)nd, allPages, newPages));
+ BlogEntrySummary entry = new BlogEntrySummary((Element)nd);
+ entry.setVersion(getBlogVersion(entry.id));
+ BlogEntrySummary oldEntry = blog.put(entry.getId(), entry);
+ if (oldEntry == null || oldEntry.getVersion() != entry.getVersion()) {
+ modifiedBlog.add(entry);
+ }
+ oldBlog.remove(entry.getId());
}
nd = nd.getNextSibling();
}
+
+ for (String id : oldBlog.keySet()) {
+ //these pages have been deleted
+ BlogEntrySummary p = blog.remove(id);
+ File file = new File(outputDir, p.getPath());
+ if (file.exists()) {
+ callSvn("rm", file.getAbsolutePath());
+ svnCommitMessage.append("Deleted: " + file.getName() + "\n");
+ }
+ if (file.exists()) {
+ file.delete();
+ }
+ }
+ }
+
+ public BlogEntrySummary findBlogEntry(String title) throws Exception {
+ return (BlogEntrySummary) findByTitle(title, blog.values());
}
- private interface ElementLoader {
- Future<?> loadElement(Element element, Set<String> allPages, Set<Page> newPages) throws Exception;
+ public BlogEntrySummary findBlogEntryByURL(String url) throws Exception {
+ return (BlogEntrySummary) findByURL(url, blog.values());
+ }
+
+ private static AbstractPage findByURL(String url, Collection<? extends AbstractPage> pages) throws Exception {
+ for (AbstractPage p : pages) {
+ if (p.getURL().endsWith(url)) {
+ return p;
+ }
+ }
+ return null;
+ }
+
+ private static AbstractPage findByTitle(String title, Collection<? extends AbstractPage> pages) throws Exception {
+ for (AbstractPage p : pages) {
+ if (title.equals(p.getTitle())) {
+ return p;
+ }
+ }
+ return null;
}
public void loadPages() throws Exception {
+ Document doc = XMLUtils.newDocument();
+ Element el = doc.createElementNS(SOAPNS, "ns1:getPages");
+ Element el2 = doc.createElement("in0");
+ el.appendChild(el2);
+ el2.setTextContent(loginToken);
+ el2 = doc.createElement("in1");
+ el.appendChild(el2);
+ el2.setTextContent(spaceKey);
+ doc.appendChild(el);
+ doc = getDispatch().invoke(doc);
+
Set<String> allPages = new CopyOnWriteArraySet<String>(pages.keySet());
Set<Page> newPages = new CopyOnWriteArraySet<Page>();
List<Future<?>> futures = new ArrayList<Future<?>>(allPages.size());
-
- loadAndAddPages(futures, allPages, newPages);
- loadAndAddBlogEntries(futures, allPages, newPages);
+ // XMLUtils.printDOM(doc.getDocumentElement());
+
+ Node nd = doc.getDocumentElement().getFirstChild().getFirstChild();
+ while (nd != null) {
+ if (nd instanceof Element) {
+ futures.add(loadPage((Element)nd, allPages, newPages));
+ }
+ nd = nd.getNextSibling();
+ }
for (Future<?> f : futures) {
//wait for all the pages to be done
f.get();
@@ -755,6 +944,7 @@ public class SiteExporter implements Run
while (checkIncludes()) {
// nothing
}
+
}
public boolean checkIncludes() {
@@ -777,8 +967,7 @@ public class SiteExporter implements Run
return false;
}
public void checkForChildren(Page p) {
- String parentId = p.getParentId();
- Page parent = parentId == null ? null : pages.get(parentId);
+ Page parent = pages.get(p.getParentId());
int d = 1;
while (parent != null) {
for (Page p2 : pages.values()) {
@@ -792,6 +981,32 @@ public class SiteExporter implements Run
}
}
+ public static synchronized Space getSpace(String key) {
+ Space space = spaces.get(key);
+ if (space == null) {
+ try {
+ doLogin();
+
+ Document doc = XMLUtils.newDocument();
+ Element el = doc.createElementNS(SOAPNS, "ns1:getSpace");
+ Element el2 = doc.createElement("in0");
+ el.appendChild(el2);
+ el2.setTextContent(loginToken);
+ el2 = doc.createElement("in1");
+ el.appendChild(el2);
+ el2.setTextContent(key);
+ doc.appendChild(el);
+
+ Document out = getDispatch().invoke(doc);
+ space = new Space(out);
+ spaces.put(key, space);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ return space;
+ }
+
public Future<?> loadPage(Element pageSumEl,
final Set<String> allPages,
final Set<Page> newPages) throws Exception {
@@ -805,45 +1020,20 @@ public class SiteExporter implements Run
el2.setTextContent(DOMUtils.getChildContent(pageSumEl, "id"));
doc.appendChild(el);
- return getResponseHandler(doc, allPages, newPages, false);
- }
-
- public Future<?> loadBlogEntry(Element pageSumEl, final Set<String> allPages,
- final Set<Page> newPages) throws Exception
- {
- Document doc = XMLUtils.newDocument();
- Element el = doc.createElementNS(SOAPNS, "ns1:getBlogEntry");
- Element el2 = doc.createElement("in0");
- el.appendChild(el2);
- el2.setTextContent(loginToken);
- el2 = doc.createElement("in1");
- el.appendChild(el2);
- el2.setTextContent(DOMUtils.getChildContent(pageSumEl, "id"));
- doc.appendChild(el);
-
- return getResponseHandler(doc, allPages, newPages, true);
- }
-
- private Future<?> getResponseHandler(Document doc, final Set<String> allPages, final Set<Page> newPages, final boolean blogPost) throws Exception
- {
- // make sure we only fire off about 15-20 or confluence may get a bit overloaded
- while (asyncCount.get() > 15)
- {
+ //make sure we only fire off about 15-20 or confluence may get a bit overloaded
+ while (asyncCount.get() > 15) {
Thread.sleep(10);
}
asyncCount.incrementAndGet();
- return getDispatch().invokeAsync(doc, new AsyncHandler<Document>() {
+ Future<?> f = getDispatch().invokeAsync(doc, new AsyncHandler<Document>() {
public void handleResponse(Response<Document> doc) {
try {
- Page page = new Page(doc.get());
+ Page page = new Page(doc.get(), SiteExporter.this);
+ page.setExporter(SiteExporter.this);
Page oldPage = pages.put(page.getId(), page);
if (oldPage == null || page.getModifiedTime().compare(oldPage.getModifiedTime()) > 0) {
- modifiedPages.add(page);
- if(blogPost) {
- // if we are adding or modifying a blog post, make sure the pages containing them get re-rendered
- for (String title : blogPages) {
- modifiedPages.add(findPage(title));
- }
+ if (!modifiedPages.contains(page)) {
+ modifiedPages.add(page);
}
if (oldPage == null) {
//need to check parents to see if it has a {children} tag so we can re-render
@@ -860,9 +1050,10 @@ public class SiteExporter implements Run
}
}
});
- }
+ return f;
+ }
- private String updateContentLinks(Page page, String content,
+ private String updateContentLinks(AbstractPage page, String content,
String id, String divCls) throws Exception {
XMLReader parser = createTagSoupParser();
StringWriter w = new StringWriter();
@@ -871,6 +1062,14 @@ public class SiteExporter implements Run
content = w.toString();
content = content.substring("<html><body>".length());
content = content.substring(0, content.lastIndexOf("</body></html>"));
+
+ int idx = content.indexOf('>');
+ if (idx != -1
+ && content.substring(idx + 1).startsWith("<p></p>")) {
+ //new confluence tends to stick an empty paragraph at the beginning for some pages (like Banner)
+ //that causes major formatting issues. Strip it.
+ content = content.substring(0, idx + 1) + content.substring(idx + 8);
+ }
return content;
}
protected XMLReader createTagSoupParser() throws Exception {
@@ -886,7 +1085,7 @@ public class SiteExporter implements Run
return reader;
}
- protected ContentHandler createContentHandler(final Page page, Writer w,
+ protected ContentHandler createContentHandler(AbstractPage page, Writer w,
String id, String divCls) {
XMLWriter xmlWriter = new ConfluenceCleanupWriter(this, w, page, id, divCls);
xmlWriter.setOutputProperty(XMLWriter.OMIT_XML_DECLARATION, "yes");
@@ -903,6 +1102,7 @@ public class SiteExporter implements Run
ListIterator<String> it = Arrays.asList(args).listIterator();
List<String> files = new ArrayList<String>();
boolean forceAll = false;
+ int maxThreads = -1;
while (it.hasNext()) {
String s = it.next();
if ("-debug".equals(s)) {
@@ -919,21 +1119,49 @@ public class SiteExporter implements Run
svn = true;
} else if ("-commit".equals(s)) {
commit = true;
+ } else if ("-maxThreads".equals(s)) {
+ maxThreads = Integer.parseInt(it.next());
} else if (s != null && s.length() > 0) {
files.add(s);
}
}
- List<Thread> threads = new ArrayList<Thread>(files.size());
+
+ List<SiteExporter> exporters = new ArrayList<SiteExporter>();
for (String file : files) {
- Thread t = new Thread(new SiteExporter(file, forceAll));
- threads.add(t);
- t.start();
+ exporters.add(new SiteExporter(file, forceAll));
}
- for (Thread t : threads) {
- t.join();
+ List<SiteExporter> modified = new ArrayList<SiteExporter>();
+ for (SiteExporter exporter : exporters) {
+ if (exporter.initialize()) {
+ modified.add(exporter);
+ }
}
+ // render stuff only if needed
+ if (!modified.isEmpty()) {
+ setSiteExporters(exporters);
+
+ if (maxThreads <= 0) {
+ maxThreads = modified.size();
+ }
+
+ ExecutorService executor = Executors.newFixedThreadPool(maxThreads, new ThreadFactory() {
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(r);
+ t.setDaemon(true);
+ return t;
+ }
+ });
+ List<Future<?>> futures = new ArrayList<Future<?>>(modified.size());
+ for (SiteExporter exporter : modified) {
+ futures.add(executor.submit(exporter));
+ }
+ for (Future<?> t : futures) {
+ t.get();
+ }
+ }
+
if (commit) {
File file = FileUtils.createTempFile("svncommit", "txt");
FileWriter writer = new FileWriter(file);
@@ -943,4 +1171,56 @@ public class SiteExporter implements Run
svnCommitMessage.setLength(0);
}
}
+
+ public boolean hasChildren(Page page) {
+ for (Page p : pages.values()) {
+ if (p == page) {
+ continue;
+ }
+ if (page.getId().equals(p.getParentId())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public List<Page> getChildren(Page page) {
+ List<Page> children = new ArrayList<Page>();
+ for (Page p : pages.values()) {
+ if (p == page) {
+ continue;
+ }
+ if (page.getId().equals(p.getParentId())) {
+ children.add(p);
+ }
+ }
+ return children;
+ }
+
+ public String link(Page page) {
+ return page.getLink();
+ }
+
+ public Space getSpace() {
+ if (space == null) {
+ space = getSpace(spaceKey);
+ }
+ return space;
+ }
+
+ private static void setSiteExporters(List<SiteExporter> exporters) {
+ siteExporters = exporters;
+ }
+
+ public int getAPIVersion() {
+ return apiVersion;
+ }
+
+ public String stripHost(String value) {
+ if (value.startsWith(HOST)) {
+ value = value.substring(HOST.length());
+ }
+ return value;
+ }
+
}
Added: tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/Space.java
URL: http://svn.apache.org/viewvc/tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/Space.java?rev=1525452&view=auto
==============================================================================
--- tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/Space.java (added)
+++ tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/Space.java Sun Sep 22 21:33:09 2013
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cxf.cwiki;
+
+import org.w3c.dom.Document;
+
+import org.apache.cxf.helpers.DOMUtils;
+
+/**
+ *
+ */
+public class Space {
+
+ final String key;
+ final String name;
+ final String url;
+
+ public Space(Document doc) throws Exception {
+ // org.apache.cxf.helpers.XMLUtils.printDOM(doc.getDocumentElement());
+
+ key = DOMUtils.getChildContent(doc.getDocumentElement().getFirstChild(), "key");
+ name = DOMUtils.getChildContent(doc.getDocumentElement().getFirstChild(), "name");
+ url = DOMUtils.getChildContent(doc.getDocumentElement().getFirstChild(), "url");
+ }
+
+ public String getKey() {
+ return key;
+ }
+ public String getName() {
+ return name;
+ }
+ public String getURL() {
+ return url;
+ }
+
+}
Propchange: tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/Space.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: tapestry/tapestry-site/trunk/template/template.vm
URL: http://svn.apache.org/viewvc/tapestry/tapestry-site/trunk/template/template.vm?rev=1525452&r1=1525451&r2=1525452&view=diff
==============================================================================
--- tapestry/tapestry-site/trunk/template/template.vm (original)
+++ tapestry/tapestry-site/trunk/template/template.vm Sun Sep 22 21:33:09 2013
@@ -28,7 +28,23 @@
#end
</title>
<link type="text/css" rel="stylesheet" href="/resources/space.css">
+
+#if($page.hasCode)
+ ## Link directly to Apache CXF site's CSS files for SyntaxHighlighter:
+ <link href='http://cxf.apache.org/resources/highlighter/styles/shCoreCXF.css' rel='stylesheet' type='text/css' />
+ <link href='http://cxf.apache.org/resources/highlighter/styles/shThemeCXF.css' rel='stylesheet' type='text/css' />
+ <script src='http://cxf.apache.org/resources/highlighter/scripts/shCore.js' type='text/javascript'></script>
+#foreach ($hscript in $page.CodeScripts)
+ <script src='http://cxf.apache.org/resources/highlighter/scripts/$hscript' type='text/javascript'></script>
+#end
+ <script type="text/javascript">
+ SyntaxHighlighter.defaults['toolbar'] = false;
+ SyntaxHighlighter.all();
+ </script>
+#end
+
<link href="/styles/style.css" rel="stylesheet" type="text/css"/>
+
</head>
<body>
<div class="wrapper bs">