You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/06/20 09:22:45 UTC
svn commit: r1494879 - in
/manifoldcf/branches/CONNECTORS-723/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira:
JiraRepositoryConnector.java JiraSession.java
Author: kwright
Date: Thu Jun 20 07:22:45 2013
New Revision: 1494879
URL: http://svn.apache.org/r1494879
Log:
More cleanup.
Modified:
manifoldcf/branches/CONNECTORS-723/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java
manifoldcf/branches/CONNECTORS-723/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java
Modified: manifoldcf/branches/CONNECTORS-723/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-723/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java?rev=1494879&r1=1494878&r2=1494879&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-723/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-723/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java Thu Jun 20 07:22:45 2013
@@ -924,100 +924,73 @@ public class JiraRepositoryConnector ext
Logging.connectors.debug("JIRA: have this file:\t" + jiraFile.get("key"));
}
- // its a file
- if (!scanOnly[i]) {
- doLog = true;
-
-
-
- // Unpack the version string
- ArrayList acls = new ArrayList();
- StringBuilder denyAclBuffer = new StringBuilder();
- int index = unpackList(acls,version,0,'+');
- if (index < version.length() && version.charAt(index++) == '+') {
- index = unpack(denyAclBuffer,version,index,'+');
- }
+ // its a file
+ if (!scanOnly[i]) {
+ doLog = true;
+
+ // Unpack the version string
+ ArrayList acls = new ArrayList();
+ StringBuilder denyAclBuffer = new StringBuilder();
+ int index = unpackList(acls,version,0,'+');
+ if (index < version.length() && version.charAt(index++) == '+') {
+ index = unpack(denyAclBuffer,version,index,'+');
+ }
- //otherwise process
- RepositoryDocument rd = new RepositoryDocument();
-
+ //otherwise process
+ RepositoryDocument rd = new RepositoryDocument();
- // Turn into acls and add into description
- String[] aclArray = new String[acls.size()];
- for (int j = 0; j < aclArray.length; j++) {
- aclArray[j] = (String)acls.get(j);
- }
- rd.setACL(aclArray);
- if (denyAclBuffer.length() > 0) {
- String[] denyAclArray = new String[]{denyAclBuffer.toString()};
- rd.setDenyACL(denyAclArray);
- }
+ // Turn into acls and add into description
+ String[] aclArray = new String[acls.size()];
+ for (int j = 0; j < aclArray.length; j++) {
+ aclArray[j] = (String)acls.get(j);
+ }
+ rd.setACL(aclArray);
+ if (denyAclBuffer.length() > 0) {
+ String[] denyAclArray = new String[]{denyAclBuffer.toString()};
+ rd.setDenyACL(denyAclArray);
+ }
- // Now do standard stuff
+ // Now do standard stuff
- String mimeType = "Jira Ticket";
- String createdDate = ((JSONObject)jiraFile.get("fields")).get("created").toString();
- String modifiedDate = ((JSONObject)jiraFile.get("fields")).get("updated").toString();
- String title = ((JSONObject)jiraFile.get("fields")).get("summary").toString();
-
- // Get general document metadata
- HashMap<String,String> metadataMap=new HashMap<String, String>();
+ String mimeType = "text/plain";
+ JSONObject fields = (JSONObject)jiraFile.get("fields");
+ JSONObject createdDate = (JSONObject)fields.get("created");
+ JSONObject modifiedDate = (JSONObject)fields.get("updated");
+
+ rd.setMimeType(mimeType);
+ if (createdDate != null)
+ rd.setCreatedDate(DateParser.parseISO8601Date(createdDate.toString()));
+ if (modifiedDate != null)
+ rd.setModifiedDate(DateParser.parseISO8601Date(modifiedDate.toString()));
+
+ // Get general document metadata
+ HashMap<String,String> metadataMap=new HashMap<String, String>();
- metadataMap = JiraSession.addMetaDataToMap("", jiraFile, metadataMap);
+ metadataMap = addMetaDataToMap("", jiraFile, metadataMap);
- for (Entry<String, String> entry : metadataMap.entrySet()) {
- rd.addField(entry.getKey(), entry.getValue());
- }
- String documentURI = jiraFile.get("self").toString();
- // Fire up the document reading thread
- BackgroundStreamThread t = new BackgroundStreamThread(jiraFile);
- try {
- t.start();
- boolean wasInterrupted = false;
- try {
- InputStream is = t.getSafeInputStream();
- try {
- rd.setBinary(is, session.getJiraBody(jiraFile).length());
- activities.ingestDocument(nodeId, version, documentURI, rd);
- } finally {
- is.close();
- }
- } catch (ManifoldCFException e) {
- if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
- wasInterrupted = true;
- throw e;
- } catch (java.net.SocketTimeoutException e) {
- throw e;
- } catch (InterruptedIOException e) {
- wasInterrupted = true;
- throw e;
- } finally {
- if (!wasInterrupted)
- t.finishUp();
- }
+ for (Entry<String, String> entry : metadataMap.entrySet()) {
+ rd.addField(entry.getKey(), entry.getValue());
+ }
+ String documentURI = jiraFile.get("self").toString();
+ String document = getJiraBody(jiraFile);
+ try {
+ byte[] documentBytes = document.getBytes("UTF-8");
+ InputStream is = new ByteArrayInputStream(documentBytes);
+ try {
+ rd.setBinary(is, documentBytes.length);
+ activities.ingestDocument(nodeId, version, documentURI, rd);
// No errors. Record the fact that we made it.
errorCode = "OK";
- } catch (InterruptedException e) {
- t.interrupt();
- throw new ManifoldCFException("Interrupted: " + e.getMessage(), e,
- ManifoldCFException.INTERRUPTED);
- } catch (java.net.SocketTimeoutException e) {
- Logging.connectors.warn("JIRA: Socket timeout reading document: " + e.getMessage(), e);
- handleIOException(e);
- } catch (InterruptedIOException e) {
- t.interrupt();
- throw new ManifoldCFException("Interrupted: " + e.getMessage(), e,
- ManifoldCFException.INTERRUPTED);
- } catch (IOException e) {
- errorCode = "IO ERROR";
- errorDesc = e.getMessage();
- Logging.connectors.warn("JIRA: Error reading document: " + e.getMessage(), e);
- handleIOException(e);
+ } finally {
+ is.close();
}
+ } catch (java.io.IOException e) {
+ throw new RuntimeException("UTF-8 encoding unknown!!");
}
+
}
- finally {
+ } finally {
if (doLog)
activities.recordActivity(new Long(startTime), ACTIVITY_READ,
fileSize, nodeId, errorCode, errorDesc, null);
@@ -1025,101 +998,43 @@ public class JiraRepositoryConnector ext
}
}
-protected class BackgroundStreamThread extends Thread
- {
- protected final JSONObject jiraissue;
-
- protected boolean abortThread = false;
- protected Throwable responseException = null;
- protected InputStream sourceStream = null;
- protected XThreadInputStream threadStream = null;
-
- public BackgroundStreamThread(JSONObject jiraissue)
- {
- super();
- setDaemon(true);
- this.jiraissue = jiraissue;
- }
-
- public void run()
- {
- try {
- try {
- synchronized (this) {
- if (!abortThread) {
- sourceStream = session.getJiraInputStream(jiraissue);
- threadStream = new XThreadInputStream(sourceStream);
- this.notifyAll();
- }
+ protected static HashMap<String, String> addMetaDataToMap(String parent, JSONObject cval, HashMap<String, String> currentMap) {
+ String append="";
+ if (parent.length() > 0) {
+ append=parent+"_";
+ }
+ for (Object key : cval.keySet()) {
+ Object value = cval.get(key);
+ if (value == null) {
+ continue;
+ }
+ //System.out.println(key);
+ if (JSONObject.class.isInstance(value)) {
+ currentMap = addMetaDataToMap(append + key, (JSONObject) value, currentMap);
+ } else if (JSONArray.class.isInstance(value)) {
+ for (Object subpiece : (JSONArray) (value)) {
+ if (String.class.isInstance(subpiece)) {
+ currentMap.put(append + key, subpiece.toString());
+ } else {
+ currentMap = addMetaDataToMap(append + key, (JSONObject) subpiece, currentMap);
}
-
- if (threadStream != null)
- {
- // Stuff the content until we are done
- threadStream.stuffQueue();
- }
- } finally {
- if (sourceStream != null)
- sourceStream.close();
- }
- } catch (Throwable e) {
- responseException = e;
- }
- }
-
- public InputStream getSafeInputStream()
- throws InterruptedException, IOException
- {
- // Must wait until stream is created, or until we note an exception was thrown.
- while (true)
- {
- synchronized (this)
- {
- if (responseException != null)
- throw new IllegalStateException("Check for response before getting stream");
- checkException(responseException);
- if (threadStream != null)
- return threadStream;
- wait();
}
+ } else {
+ currentMap.put(append+key + "", value.toString());
}
}
-
- public void finishUp()
- throws InterruptedException, IOException
- {
- // This will be called during the finally
- // block in the case where all is well (and
- // the stream completed) and in the case where
- // there were exceptions.
- synchronized (this) {
- if (threadStream != null) {
- threadStream.abort();
- }
- abortThread = true;
- }
-
- join();
+ return currentMap;
+ }
- checkException(responseException);
- }
-
- protected synchronized void checkException(Throwable exception)
- throws IOException
- {
- if (exception != null)
- {
- Throwable e = exception;
- if (e instanceof IOException)
- throw (IOException)e;
- else if (e instanceof RuntimeException)
- throw (RuntimeException)e;
- else if (e instanceof Error)
- throw (Error)e;
- else
- throw new RuntimeException("Unhandled exception of type: "+e.getClass().getName(),e);
- }
+ protected static String getJiraBody(JSONObject jiraFile){
+ String body;
+ Object possibleBody = ((JSONObject) jiraFile.get("fields")).get("description");
+ if (possibleBody == null) {
+ body = ((JSONObject) jiraFile.get("fields")).get("summary").toString();
+ } else {
+ body = possibleBody.toString();
}
+ return body;
}
/**
Modified: manifoldcf/branches/CONNECTORS-723/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-723/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java?rev=1494879&r1=1494878&r2=1494879&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-723/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java (original)
+++ manifoldcf/branches/CONNECTORS-723/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java Thu Jun 20 07:22:45 2013
@@ -70,6 +70,9 @@ public class JiraSession {
}
private JSONObject getRest(String rightside) throws IOException {
+ // This should use HttpComponents HttpClient instead; the java implementation is
+ // hopelessly primitive. Also, the encoding is not dealt with properly
+ // MHL
URL url = new URL(URLbase + apiPost + rightside);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("GET");
@@ -127,46 +130,5 @@ public class JiraSession {
return getRest("issue/" + id);
}
- static public HashMap<String, String> addMetaDataToMap(String parent, JSONObject cval, HashMap<String, String> currentMap) {
- String append="";
- if (parent.length() > 0) {
- append=parent+"_";
- }
- for (Object key : cval.keySet()) {
- Object value = cval.get(key);
- if (value == null) {
- continue;
- }
- //System.out.println(key);
- if (JSONObject.class.isInstance(value)) {
- currentMap = addMetaDataToMap(append + key, (JSONObject) value, currentMap);
- } else if (JSONArray.class.isInstance(value)) {
- for (Object subpiece : (JSONArray) (value)) {
- if (String.class.isInstance(subpiece)) {
- currentMap.put(append + key, subpiece.toString());
- } else {
- currentMap = addMetaDataToMap(append + key, (JSONObject) subpiece, currentMap);
- }
- }
- } else {
- currentMap.put(append+key + "", value.toString());
- }
- }
- return currentMap;
- }
- public InputStream getJiraInputStream(JSONObject jiraFile) {
- return new ByteArrayInputStream(getJiraBody(jiraFile).getBytes());
- }
-
- static public String getJiraBody(JSONObject jiraFile){
- String body;
- Object possibleBody = ((JSONObject) jiraFile.get("fields")).get("description");
- if (possibleBody == null) {
- body = ((JSONObject) jiraFile.get("fields")).get("summary").toString();
- } else {
- body = possibleBody.toString();
- }
- return body;
- }
}