You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by mi...@apache.org on 2013/06/24 05:26:58 UTC
svn commit: r1495913 [2/5] - in /manifoldcf/branches/CONNECTORS-728: ./
connectors/
connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/
connectors/filesystem/connector/src/main/native2ascii/org/apache/mani...
Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputSpecs.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputSpecs.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputSpecs.java (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputSpecs.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,153 @@
+/* $Id: FileOutputSpecs.java 1299512 2013-05-31 22:58:38Z minoru $ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.manifoldcf.agents.output.hdfs;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.manifoldcf.core.interfaces.ConfigurationNode;
+import org.apache.manifoldcf.core.interfaces.IPostParameters;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+public class HDFSOutputSpecs extends HDFSOutputParam {
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1145652730572662025L;
+
+ final public static ParameterEnum[] SPECIFICATIONLIST = {
+ ParameterEnum.ROOTPATH
+ };
+
+ private String rootPath;
+
+ /** Build a set of ElasticSearch parameters by reading an JSON object
+ *
+ * @param json
+ * @throws JSONException
+ * @throws ManifoldCFException
+ */
+ public HDFSOutputSpecs(String json) throws JSONException, ManifoldCFException {
+ this(new JSONObject(json));
+ }
+
+ /** Build a set of ElasticSearch parameters by reading an JSON object
+ *
+ * @param json
+ * @throws JSONException
+ * @throws ManifoldCFException
+ */
+ public HDFSOutputSpecs(JSONObject json) throws JSONException, ManifoldCFException {
+ super(SPECIFICATIONLIST);
+ rootPath = null;
+ for (ParameterEnum param : SPECIFICATIONLIST) {
+ String value = null;
+ value = json.getString(param.name());
+ if (value == null) {
+ value = param.defaultValue;
+ }
+ put(param, value);
+ }
+ rootPath = getRootPath();
+ }
+
+ /** Build a set of ElasticSearch parameters by reading an instance of
+ * SpecificationNode.
+ *
+ * @param node
+ * @throws ManifoldCFException
+ */
+ public HDFSOutputSpecs(ConfigurationNode node) throws ManifoldCFException {
+ super(SPECIFICATIONLIST);
+ rootPath = null;
+ for (ParameterEnum param : SPECIFICATIONLIST) {
+ String value = null;
+ if (node != null) {
+ value = node.getAttributeValue(param.name());
+ }
+ if (value == null) {
+ value = param.defaultValue;
+ }
+ put(param, value);
+ }
+ rootPath = getRootPath();
+ }
+
+ /**
+ * @param variableContext
+ * @param specNode
+ */
+ public static void contextToSpecNode(IPostParameters variableContext, ConfigurationNode specNode) {
+ for (ParameterEnum param : SPECIFICATIONLIST) {
+ String p = variableContext.getParameter(param.name().toLowerCase());
+ if (p != null) {
+ specNode.setAttribute(param.name(), p);
+ }
+ }
+ }
+
+ /** @return a JSON representation of the parameter list */
+ public JSONObject toJson() {
+ return new JSONObject(this);
+ }
+
+ /**
+ * @return
+ */
+ public String getRootPath() {
+ return get(ParameterEnum.ROOTPATH);
+ }
+
+ /**
+ * @param content
+ * @return
+ * @throws ManifoldCFException
+ */
+ private final static TreeSet<String> createStringSet(String content) throws ManifoldCFException {
+ TreeSet<String> set = new TreeSet<String>();
+ BufferedReader br = null;
+ StringReader sr = null;
+ try {
+ sr = new StringReader(content);
+ br = new BufferedReader(sr);
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ line = line.trim();
+ if (line.length() > 0) {
+ set.add(line);
+ }
+ }
+ return set;
+ } catch (IOException e) {
+ throw new ManifoldCFException(e);
+ } finally {
+ if (br != null) {
+ IOUtils.closeQuietly(br);
+ }
+ }
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputSpecs.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/Messages.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/Messages.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/Messages.java (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/Messages.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,141 @@
+/* $Id: Messages.java 1295926 2013-05-31 23:00:00Z minoru $ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.manifoldcf.agents.output.hdfs;
+
+import java.util.Locale;
+import java.util.Map;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+
+public class Messages extends org.apache.manifoldcf.ui.i18n.Messages
+{
+ public static final String DEFAULT_BUNDLE_NAME="org.apache.manifoldcf.agents.output.hdfs.common";
+ public static final String DEFAULT_PATH_NAME="org.apache.manifoldcf.agents.output.hdfs";
+
+ /** Constructor - do no instantiate
+ */
+ protected Messages()
+ {
+ }
+
+ public static String getString(Locale locale, String messageKey)
+ {
+ return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getAttributeString(Locale locale, String messageKey)
+ {
+ return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getBodyString(Locale locale, String messageKey)
+ {
+ return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getAttributeJavascriptString(Locale locale, String messageKey)
+ {
+ return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getBodyJavascriptString(Locale locale, String messageKey)
+ {
+ return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getString(Locale locale, String messageKey, Object[] args)
+ {
+ return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getAttributeString(Locale locale, String messageKey, Object[] args)
+ {
+ return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getBodyString(Locale locale, String messageKey, Object[] args)
+ {
+ return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getAttributeJavascriptString(Locale locale, String messageKey, Object[] args)
+ {
+ return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getBodyJavascriptString(Locale locale, String messageKey, Object[] args)
+ {
+ return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ // More general methods which allow bundlenames and class loaders to be specified.
+
+ public static String getString(String bundleName, Locale locale, String messageKey, Object[] args)
+ {
+ return getString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getAttributeString(String bundleName, Locale locale, String messageKey, Object[] args)
+ {
+ return getAttributeString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getBodyString(String bundleName, Locale locale, String messageKey, Object[] args)
+ {
+ return getBodyString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getAttributeJavascriptString(String bundleName, Locale locale, String messageKey, Object[] args)
+ {
+ return getAttributeJavascriptString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getBodyJavascriptString(String bundleName, Locale locale, String messageKey, Object[] args)
+ {
+ return getBodyJavascriptString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ // Resource output
+
+ public static void outputResource(IHTTPOutput output, Locale locale, String resourceKey,
+ Map<String,String> substitutionParameters, boolean mapToUpperCase)
+ throws ManifoldCFException
+ {
+ outputResource(output,Messages.class,DEFAULT_PATH_NAME,locale,resourceKey,
+ substitutionParameters,mapToUpperCase);
+ }
+
+ public static void outputResourceWithVelocity(IHTTPOutput output, Locale locale, String resourceKey,
+ Map<String,String> substitutionParameters, boolean mapToUpperCase)
+ throws ManifoldCFException
+ {
+ outputResourceWithVelocity(output,Messages.class,DEFAULT_BUNDLE_NAME,DEFAULT_PATH_NAME,locale,resourceKey,
+ substitutionParameters,mapToUpperCase);
+ }
+
+ public static void outputResourceWithVelocity(IHTTPOutput output, Locale locale, String resourceKey,
+ Map<String,Object> contextObjects)
+ throws ManifoldCFException
+ {
+ outputResourceWithVelocity(output,Messages.class,DEFAULT_BUNDLE_NAME,DEFAULT_PATH_NAME,locale,resourceKey,
+ contextObjects);
+ }
+
+}
+
Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/Messages.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,1503 @@
+/* $Id: FileConnector.java 995085 2010-09-08 15:13:38Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.connectors.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.Logging;
+import org.apache.manifoldcf.core.extmimemap.ExtensionMimeMap;
+import java.util.*;
+import java.io.*;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/** This is the "repository connector" for a file system. It's a relative of the share crawler, and should have
+* comparable basic functionality, with the exception of the ability to use ActiveDirectory and look at other shares.
+*/
+public class HDFSRepositoryConnector extends org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector
+{
+ public static final String _rcsid = "@(#)$Id: FileConnector.java 995085 2010-09-08 15:13:38Z kwright $";
+
+ // Activities that we know about
+ protected final static String ACTIVITY_READ = "read document";
+
+ // Relationships we know about
+ protected static final String RELATIONSHIP_CHILD = "child";
+
+ // Activities list
+ protected static final String[] activitiesList = new String[]{ACTIVITY_READ};
+
+ // Parameters that this connector cares about
+ // public final static String ROOTDIRECTORY = "rootdirectory";
+
+ // Local data
+ // protected File rootDirectory = null;
+
+ protected Configuration config = null;
+ protected FileSystem fileSystem = null;
+
+ /*
+ * Constructor.
+ */
+ public HDFSRepositoryConnector()
+ {
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.manifoldcf.core.connector.BaseConnector#connect(org.apache.manifoldcf.core.interfaces.ConfigParams)
+ */
+ @Override
+ public void connect(ConfigParams configParams) {
+ super.connect(configParams);
+
+ String nameNode = configParams.getParameter("namenode");
+
+ String user = configParams.getParameter("user");
+
+ /*
+ * make Configuration
+ */
+ ClassLoader ocl = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(org.apache.hadoop.conf.Configuration.class.getClassLoader());
+ config = new Configuration();
+ config.set("fs.default.name", nameNode);
+ } finally {
+ Thread.currentThread().setContextClassLoader(ocl);
+ }
+
+ /*
+ * get connection to HDFS
+ */
+ try {
+ fileSystem = FileSystem.get(new URI(nameNode), config, user);
+ } catch (URISyntaxException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.manifoldcf.core.connector.BaseConnector#disconnect()
+ */
+ @Override
+ public void disconnect() throws ManifoldCFException {
+ try {
+ fileSystem.close();
+ } catch(IOException ex) {
+ throw new ManifoldCFException(ex);
+ }
+ config.clear();
+ super.disconnect();
+ }
+
+ /** Tell the world what model this connector uses for getDocumentIdentifiers().
+ * This must return a model value as specified above.
+ *@return the model type value.
+ */
+ @Override
+ public int getConnectorModel()
+ {
+ return MODEL_CHAINED_ADD_CHANGE;
+ }
+
+ /** Return the list of relationship types that this connector recognizes.
+ *@return the list.
+ */
+ @Override
+ public String[] getRelationshipTypes()
+ {
+ return new String[]{RELATIONSHIP_CHILD};
+ }
+
+ /** List the activities we might report on.
+ */
+ @Override
+ public String[] getActivitiesList()
+ {
+ return activitiesList;
+ }
+
+ /** For any given document, list the bins that it is a member of.
+ */
+ @Override
+ public String[] getBinNames(String documentIdentifier)
+ {
+ /*
+ // Note: This code is for testing, so we can see how documents behave when they are in various kinds of bin situations.
+ // The testing model is that there are documents belonging to "SLOW", to "FAST", or both to "SLOW" and "FAST" bins.
+ // The connector chooses which bins to assign a document to based on the identifier (which is the document's path), so
+ // this is something that should NOT be duplicated by other connector implementers.
+ if (documentIdentifier.indexOf("/BOTH/") != -1 || (documentIdentifier.indexOf("/SLOW/") != -1 && documentIdentifier.indexOf("/FAST/") != -1))
+ return new String[]{"SLOW","FAST"};
+ if (documentIdentifier.indexOf("/SLOW/") != -1)
+ return new String[]{"SLOW"};
+ if (documentIdentifier.indexOf("/FAST/") != -1)
+ return new String[]{"FAST"};
+ */
+ return new String[]{""};
+ }
+
+ /** Convert a document identifier to a URI. The URI is the URI that will be the unique key from
+ * the search index, and will be presented to the user as part of the search results.
+ *@param filePath is the document filePath.
+ *@param repositoryPath is the document repositoryPath.
+ *@return the document uri.
+ */
+ protected String convertToURI(String documentIdentifier, String[] repositoryPaths)
+ throws ManifoldCFException
+ {
+ //
+ // Note well: This MUST be a legal URI!!!
+ try
+ {
+ String path = new Path(documentIdentifier).toString();
+ for (String repositoryPath : repositoryPaths) {
+ if (path.startsWith(repositoryPath)) {
+ StringBuffer sb = new StringBuffer();
+ path = path.replaceFirst(repositoryPath, "");
+ if (path.startsWith("/")) {
+ path = path.replaceFirst("/", "");
+ }
+ String[] tmp = path.split("/", 3);
+ String scheme = "";
+ String host = "";
+ String other = "";
+ try {
+ scheme = tmp[0];
+ } catch (ArrayIndexOutOfBoundsException e) {
+ scheme = "hdfs";
+ }
+ try {
+ host = tmp[1];
+ } catch (ArrayIndexOutOfBoundsException e) {
+ host = "localhost:9000";
+ }
+ try {
+ other = "/" + tmp[2];
+ } catch (ArrayIndexOutOfBoundsException e) {
+ other = "/";
+ }
+ return new URI(scheme + "://" + host + other).toURL().toString();
+ }
+ }
+ return convertToURI(documentIdentifier);
+ }
+ catch (URISyntaxException e)
+ {
+ throw new ManifoldCFException("Bad url",e);
+ }
+ catch (IOException e)
+ {
+ throw new ManifoldCFException("Bad url",e);
+ }
+ }
+
+ /** Convert a document identifier to a URI. The URI is the URI that will be the unique key from
+ * the search index, and will be presented to the user as part of the search results.
+ *@param documentIdentifier is the document identifier.
+ *@return the document uri.
+ */
+ protected String convertToURI(String documentIdentifier)
+ throws ManifoldCFException
+ {
+ //
+ // Note well: This MUST be a legal URI!!!
+ return new Path(documentIdentifier).toUri().toString();
+ }
+
+
+ /** Given a document specification, get either a list of starting document identifiers (seeds),
+ * or a list of changes (deltas), depending on whether this is a "crawled" connector or not.
+ * These document identifiers will be loaded into the job's queue at the beginning of the
+ * job's execution.
+ * This method can return changes only (because it is provided a time range). For full
+ * recrawls, the start time is always zero.
+ * Note that it is always ok to return MORE documents rather than less with this method.
+ *@param spec is a document specification (that comes from the job).
+ *@param startTime is the beginning of the time range to consider, inclusive.
+ *@param endTime is the end of the time range to consider, exclusive.
+ *@return the stream of local document identifiers that should be added to the queue.
+ */
+ @Override
+ public IDocumentIdentifierStream getDocumentIdentifiers(DocumentSpecification spec, long startTime, long endTime)
+ throws ManifoldCFException
+ {
+ return new IdentifierStream(spec);
+ }
+
+
+ /** Get document versions given an array of document identifiers.
+ * This method is called for EVERY document that is considered. It is therefore important to perform
+ * as little work as possible here.
+ * The connector will be connected before this method can be called.
+ *@param documentIdentifiers is the array of local document identifiers, as understood by this connector.
+ *@param oldVersions is the corresponding array of version strings that have been saved for the document identifiers.
+ * A null value indicates that this is a first-time fetch, while an empty string indicates that the previous document
+ * had an empty version string.
+ *@param activities is the interface this method should use to perform whatever framework actions are desired.
+ *@param spec is the current document specification for the current job. If there is a dependency on this
+ * specification, then the version string should include the pertinent data, so that reingestion will occur
+ * when the specification changes. This is primarily useful for metadata.
+ *@param jobMode is an integer describing how the job is being run, whether continuous or once-only.
+ *@param usesDefaultAuthority will be true only if the authority in use for these documents is the default one.
+ *@return the corresponding version strings, with null in the places where the document no longer exists.
+ * Empty version strings indicate that there is no versioning ability for the corresponding document, and the document
+ * will always be processed.
+ */
+ public String[] getDocumentVersions(String[] documentIdentifiers, String[] oldVersions, IVersionActivity activities,
+ DocumentSpecification spec, int jobMode, boolean usesDefaultAuthority)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ int i = 0;
+
+ /*
+ * get filepathtouri value
+ */
+ boolean filePathToUri = false;
+ i = 0;
+ while (i < spec.getChildCount()) {
+ SpecificationNode sn = spec.getChild(i++);
+ if (sn.getType().equals("filepathtouri")) {
+ filePathToUri = Boolean.valueOf(sn.getValue());
+ }
+ }
+
+ String[] rval = new String[documentIdentifiers.length];
+ try
+ {
+ i = 0;
+ while (i < rval.length)
+ {
+ Path path = new Path(documentIdentifiers[i]);
+ if (fileSystem.exists(path))
+ {
+ if (fileSystem.getFileStatus(path).isDir())
+ {
+ // It's a directory. The version ID will be the
+ // last modified date.
+ long lastModified = fileSystem.getFileStatus(path).getModificationTime();
+ rval[i] = new Long(lastModified).toString();
+
+ // Signal that we don't have any versioning.
+ // rval[i] = "";
+ }
+ else
+ {
+ // It's a file
+ long fileLength = fileSystem.getFileStatus(path).getLen();
+ if (activities.checkLengthIndexable(fileLength))
+ {
+ // Get the file's modified date.
+ long lastModified = fileSystem.getFileStatus(path).getModificationTime();
+ StringBuilder sb = new StringBuilder();
+ if (filePathToUri)
+ {
+ sb.append("+");
+ }
+ else
+ {
+ sb.append("-");
+ }
+ sb.append(new Long(lastModified).toString()).append(":").append(new Long(fileLength).toString());
+ rval[i] = sb.toString();
+ }
+ else
+ {
+ rval[i] = null;
+ }
+ }
+ }
+ else
+ {
+ rval[i] = null;
+ }
+ i++;
+ }
+ }
+ catch (IOException e)
+ {
+ throw new ManifoldCFException(e);
+ }
+ return rval;
+ }
+
+
+ /** Process a set of documents.
+ * This is the method that should cause each document to be fetched, processed, and the results either added
+ * to the queue of documents for the current job, and/or entered into the incremental ingestion manager.
+ * The document specification allows this class to filter what is done based on the job.
+ *@param documentIdentifiers is the set of document identifiers to process.
+ *@param activities is the interface this method should use to queue up new document references
+ * and ingest documents.
+ *@param spec is the document specification.
+ *@param scanOnly is an array corresponding to the document identifiers. It is set to true to indicate when the processing
+ * should only find other references, and should not actually call the ingestion methods.
+ */
+ @Override
+ public void processDocuments(String[] documentIdentifiers, String[] versions, IProcessActivity activities, DocumentSpecification spec, boolean[] scanOnly)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ try
+ {
+ int i = 0;
+ while (i < documentIdentifiers.length)
+ {
+ String version = versions[i];
+ String documentIdentifier = documentIdentifiers[i];
+ Path path = new Path(documentIdentifier);
+ FileStatus fileStatus = fileSystem.getFileStatus(path);
+ if (fileSystem.exists(path))
+ {
+ if (fileStatus.isDir())
+ {
+ // Queue up stuff for directory
+ long startTime = System.currentTimeMillis();
+ String errorCode = "OK";
+ String errorDesc = null;
+ String entityReference = documentIdentifier;
+ try
+ {
+ try
+ {
+ FileStatus[] fileStatuses = fileSystem.listStatus(path);
+ if (fileStatuses != null)
+ {
+ int j = 0;
+ while (j < fileStatuses.length)
+ {
+ FileStatus fs = fileStatuses[j++];
+ String canonicalPath = fs.getPath().toString();
+ if (checkInclude(fileSystem.getUri().toString(),fs,canonicalPath,spec))
+ activities.addDocumentReference(canonicalPath,documentIdentifier,RELATIONSHIP_CHILD);
+ }
+ }
+ }
+ catch (IOException e)
+ {
+ errorCode = "IO ERROR";
+ errorDesc = e.getMessage();
+ throw new ManifoldCFException("IO Error: "+e.getMessage(),e);
+ }
+ }
+ finally
+ {
+ activities.recordActivity(new Long(startTime),ACTIVITY_READ,null,entityReference,errorCode,errorDesc,null);
+ }
+ }
+ else
+ {
+ if (!scanOnly[i])
+ {
+ // We've already avoided queuing documents that we don't want, based on file specifications.
+ // We still need to check based on file data.
+ if (checkIngest(fileSystem.getUri().toString(),fileStatus,spec))
+ {
+ int j = 0;
+
+ /*
+ * get repository paths
+ */
+ j = 0;
+ List<String> repositoryPaths = new ArrayList<String>();
+ while ( j < spec.getChildCount())
+ {
+ SpecificationNode sn = spec.getChild(j++);
+ if (sn.getType().equals("startpoint"))
+ {
+ if (sn.getAttributeValue("path").length() > 0) {
+ repositoryPaths.add(fileSystem.getUri().resolve(sn.getAttributeValue("path")).toString());
+ }
+ }
+ }
+
+ /*
+ * get filepathtouri value
+ */
+ boolean filePathToUri = false;
+ if (version.length() > 0 && version.startsWith("+")) {
+ filePathToUri = true;
+ }
+
+ long startTime = System.currentTimeMillis();
+ String errorCode = "OK";
+ String errorDesc = null;
+ Long fileLength = null;
+ String entityDescription = documentIdentifier;
+ try
+ {
+ // Ingest the document.
+ try
+ {
+ FSDataInputStream is = fileSystem.open(path);
+ try
+ {
+ long fileBytes = fileStatus.getLen();
+ RepositoryDocument data = new RepositoryDocument();
+ data.setBinary(is,fileBytes);
+ String fileName = path.getName();
+ data.setFileName(fileName);
+ data.setMimeType(mapExtensionToMimeType(fileName));
+ data.setModifiedDate(new Date(fileStatus.getModificationTime()));
+ if (filePathToUri) {
+ data.addField("uri",convertToURI(documentIdentifier,repositoryPaths.toArray(new String[0])));
+ // MHL for other metadata
+ activities.ingestDocument(documentIdentifier,version,convertToURI(documentIdentifier,repositoryPaths.toArray(new String[0])),data);
+ } else {
+ data.addField("uri",path.toString());
+ // MHL for other metadata
+ activities.ingestDocument(documentIdentifier,version,convertToURI(documentIdentifier),data);
+ }
+ fileLength = new Long(fileBytes);
+ }
+ finally
+ {
+ is.close();
+ }
+ }
+ catch (IOException e)
+ {
+ errorCode = "IO ERROR";
+ errorDesc = e.getMessage();
+ throw new ManifoldCFException("IO Error: "+e.getMessage(),e);
+ }
+ }
+ finally
+ {
+ activities.recordActivity(new Long(startTime),ACTIVITY_READ,fileLength,entityDescription,errorCode,errorDesc,null);
+ }
+ }
+ }
+ }
+ }
+ i++;
+ }
+ }
+ catch(IOException e)
+ {
+ throw new ManifoldCFException(e);
+ }
+ }
+
+ /** Map an extension to a mime type */
+ protected static String mapExtensionToMimeType(String fileName)
+ {
+ int slashIndex = fileName.lastIndexOf("/");
+ if (slashIndex != -1)
+ fileName = fileName.substring(slashIndex+1);
+ int dotIndex = fileName.lastIndexOf(".");
+ if (dotIndex == -1)
+ return null;
+ return ExtensionMimeMap.mapToMimeType(fileName.substring(dotIndex+1).toLowerCase(java.util.Locale.ROOT));
+ }
+
+ // UI support methods.
+ //
+ // These support methods come in two varieties. The first bunch is involved in setting up connection configuration information. The second bunch
+ // is involved in presenting and editing document specification information for a job. The two kinds of methods are accordingly treated differently,
+ // in that the first bunch cannot assume that the current connector object is connected, while the second bunch can. That is why the first bunch
+ // receives a thread context argument for all UI methods, while the second bunch does not need one (since it has already been applied via the connect()
+ // method, above).
+
+ /** Output the configuration header section.
+ * This method is called in the head section of the connector's configuration page. Its purpose is to add the required tabs to the list, and to output any
+ * javascript methods that might be needed by the configuration editing HTML.
+ *@param threadContext is the local thread context.
+ *@param out is the output to which any HTML should be sent.
+ *@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
+ *@param tabsArray is an array of tab names. Add to this array any tab names that are specific to the connector.
+ */
+ @Override
+ public void outputConfigurationHeader(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters, List<String> tabsArray) throws ManifoldCFException, IOException
+ {
+ tabsArray.add(Messages.getString(locale,"HDFSRepositoryConnector.ServerTabName"));
+
+ out.print(
+"<script type=\"text/javascript\">\n"+
+"<!--\n"+
+"function checkConfigForSave()\n"+
+"{\n"+
+" return true;\n"+
+"}\n"+
+"\n"+
+"//-->\n"+
+"</script>\n"
+ );
+ }
+
+ /** Output the configuration body section.
+ * This method is called in the body section of the connector's configuration page. Its purpose is to present the required form elements for editing.
+ * The coder can presume that the HTML that is output from this configuration will be within appropriate <html>, <body>, and <form> tags. The name of the
+ * form is "editconnection".
+ *@param threadContext is the local thread context.
+ *@param out is the output to which any HTML should be sent.
+ *@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
+ *@param tabName is the current tab name.
+ */
+ public void outputConfigurationBody(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters, String tabName)
+ throws ManifoldCFException, IOException
+ {
+ String nameNode = parameters.getParameter("namenode");
+ if (nameNode == null) {
+ nameNode = "hdfs://localhost:9000";
+ }
+ String user = parameters.getParameter("user");
+ if (user == null) {
+ user = "";
+ }
+
+ if (tabName.equals(Messages.getString(locale,"HDFSRepositoryConnector.ServerTabName")))
+ {
+ out.print(
+"<table class=\"displaytable\">\n"+
+" <tr>\n"+
+" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.NameNode") + "</nobr></td>\n"+
+" <td class=\"value\">\n"+
+" <input name=\"namenode\" type=\"text\" size=\"48\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(nameNode)+"\"/>\n"+
+" </td>\n"+
+" </tr>\n"+
+" <tr>\n"+
+" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.User") + "</nobr></td>\n"+
+" <td class=\"value\">\n"+
+" <input name=\"user\" type=\"text\" size=\"48\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(user)+"\"/>\n"+
+" </td>\n"+
+" </tr>\n"+
+"</table>\n"
+ );
+ }
+ else
+ {
+ // Server tab hiddens
+ out.print(
+"<input type=\"hidden\" name=\"namenode\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(nameNode)+"\"/>\n"+
+"<input type=\"hidden\" name=\"user\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(user)+"\"/>\n"
+ );
+ }
+ }
+
+ /** Process a configuration post.
+ * This method is called at the start of the connector's configuration page, whenever there is a possibility that form data for a connection has been
+ * posted. Its purpose is to gather form information and modify the configuration parameters accordingly.
+ * The name of the posted form is "editconnection".
+ *@param threadContext is the local thread context.
+ *@param variableContext is the set of variables available from the post, including binary file post information.
+ *@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
+ *@return null if all is well, or a string error message if there is an error that should prevent saving of the connection (and cause a redirection to an error page).
+ */
+ @Override
+ public String processConfigurationPost(IThreadContext threadContext, IPostParameters variableContext, ConfigParams parameters)
+ throws ManifoldCFException
+ {
+ String nameNode = variableContext.getParameter("namenode");
+ if (nameNode != null) {
+ parameters.setParameter("namenode", nameNode);
+ }
+
+ String user = variableContext.getParameter("user");
+ if (user != null) {
+ parameters.setParameter("user", user);
+ }
+
+ return null;
+ }
+
+ /** View configuration.
+ * This method is called in the body section of the connector's view configuration page. Its purpose is to present the connection information to the user.
+ * The coder can presume that the HTML that is output from this configuration will be within appropriate <html> and <body> tags.
+ *@param threadContext is the local thread context.
+ *@param out is the output to which any HTML should be sent.
+ *@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
+ */
+ @Override
+ public void viewConfiguration(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters)
+ throws ManifoldCFException, IOException
+ {
+ String nameNode = parameters.getParameter("namenode");
+ if (nameNode == null) {
+ nameNode = "hdfs://localhost:9000";
+ }
+
+ String user = parameters.getParameter("user");
+ if (user == null) {
+ user = "user";
+ }
+
+ out.print(
+"<table class=\"displaytable\">\n"+
+" <tr>\n"+
+" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.NameNode") + "</nobr></td>\n"+
+" <td class=\"value\">\n"+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(nameNode)+"</td>\n"+
+" </tr>\n"+
+" <tr>\n"+
+" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.User") + "</nobr></td>\n"+
+" <td class=\"value\">\n"+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(user)+"</td>\n"+
+" </tr>\n"+
+"</table>\n"
+ );
+ }
+
+ /** Output the specification header section.
+ * This method is called in the head section of a job page which has selected a repository connection of the current type. Its purpose is to add the required tabs
+ * to the list, and to output any javascript methods that might be needed by the job editing HTML.
+ *@param out is the output to which any HTML should be sent.
+ *@param ds is the current document specification for this job.
+ *@param tabsArray is an array of tab names. Add to this array any tab names that are specific to the connector.
+ */
+ @Override
+ public void outputSpecificationHeader(IHTTPOutput out, Locale locale, DocumentSpecification ds, List<String> tabsArray)
+ throws ManifoldCFException, IOException
+ {
+ tabsArray.add(Messages.getString(locale,"HDFSRepositoryConnector.Paths"));
+ tabsArray.add(Messages.getString(locale,"HDFSRepositoryConnector.FilePathToURITab"));
+
+ out.print(
+"<script type=\"text/javascript\">\n"+
+"<!--\n"+
+"function checkSpecification()\n"+
+"{\n"+
+" // Does nothing right now.\n"+
+" return true;\n"+
+"}\n"+
+"\n"+
+"function SpecOp(n, opValue, anchorvalue)\n"+
+"{\n"+
+" eval(\"editjob.\"+n+\".value = \\\"\"+opValue+\"\\\"\");\n"+
+" postFormSetAnchor(anchorvalue);\n"+
+"}\n"+
+"//-->\n"+
+"</script>\n"
+ );
+ }
+
+ /** Output the specification body section.
+ * This method is called in the body section of a job page which has selected a repository connection of the current type. Its purpose is to present the required form elements for editing.
+ * The coder can presume that the HTML that is output from this configuration will be within appropriate <html>, <body>, and <form> tags. The name of the
+ * form is "editjob".
+ *@param out is the output to which any HTML should be sent.
+ *@param ds is the current document specification for this job.
+ *@param tabName is the current tab name.
+ */
+ @Override
+ public void outputSpecificationBody(IHTTPOutput out, Locale locale, DocumentSpecification ds, String tabName)
+ throws ManifoldCFException, IOException
+ {
+ int i;
+ int k;
+
+ // Paths tab
+ if (tabName.equals(Messages.getString(locale,"HDFSRepositoryConnector.Paths")))
+ {
+ out.print(
+"<table class=\"displaytable\">\n"+
+" <tr><td class=\"separator\" colspan=\"3\"><hr/></td></tr>\n"+
+" <tr>\n"+
+" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.Paths2") + "</nobr></td>\n"+
+" <td class=\"boxcell\">\n"+
+" <table class=\"formtable\">\n"+
+" <tr class=\"formheaderrow\">\n"+
+" <td class=\"formcolumnheader\"></td>\n"+
+" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.RootPath") + "</nobr></td>\n"+
+" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.Rules") + "</nobr></td>\n"+
+" </tr>\n"
+ );
+ i = 0;
+ k = 0;
+ while (i < ds.getChildCount())
+ {
+ SpecificationNode sn = ds.getChild(i++);
+ if (sn.getType().equals("startpoint"))
+ {
+ String pathDescription = "_"+Integer.toString(k);
+ String pathOpName = "specop"+pathDescription;
+ out.print(
+" <tr class=\""+(((k % 2)==0)?"evenformrow":"oddformrow")+"\">\n"+
+" <td class=\"formcolumncell\">\n"+
+" <input type=\"hidden\" name=\""+pathOpName+"\" value=\"\"/>\n"+
+" <input type=\"hidden\" name=\""+"specpath"+pathDescription+"\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(sn.getAttributeValue("path"))+"\"/>\n"+
+" <a name=\""+"path_"+Integer.toString(k)+"\">\n"+
+" <input type=\"button\" value=\"" + Messages.getAttributeString(locale,"HDFSRepositoryConnector.Delete") + "\" onClick='Javascript:SpecOp(\""+pathOpName+"\",\"Delete\",\"path_"+Integer.toString(k)+"\")' alt=\""+Messages.getAttributeString(locale,"HDFSRepositoryConnector.DeletePath")+Integer.toString(k)+"\"/>\n"+
+" </a>\n"+
+" </td>\n"+
+" <td class=\"formcolumncell\">\n"+
+" <nobr>\n"+
+" "+org.apache.manifoldcf.ui.util.Encoder.bodyEscape(sn.getAttributeValue("path"))+" \n"+
+" </nobr>\n"+
+" </td>\n"+
+" <td class=\"boxcell\">\n"+
+" <input type=\"hidden\" name=\""+"specchildcount"+pathDescription+"\" value=\""+Integer.toString(sn.getChildCount())+"\"/>\n"+
+" <table class=\"formtable\">\n"+
+" <tr class=\"formheaderrow\">\n"+
+" <td class=\"formcolumnheader\"></td>\n"+
+" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.IncludeExclude") + "</nobr></td>\n"+
+" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.FileDirectory") + "</nobr></td>\n"+
+" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.Match") + "</nobr></td>\n"+
+" </tr>\n"
+ );
+ int j = 0;
+ while (j < sn.getChildCount())
+ {
+ SpecificationNode excludeNode = sn.getChild(j);
+ String instanceDescription = "_"+Integer.toString(k)+"_"+Integer.toString(j);
+ String instanceOpName = "specop" + instanceDescription;
+
+ String nodeFlavor = excludeNode.getType();
+ String nodeType = excludeNode.getAttributeValue("type");
+ String nodeMatch = excludeNode.getAttributeValue("match");
+ out.print(
+" <tr class=\"evenformrow\">\n"+
+" <td class=\"formcolumncell\">\n"+
+" <nobr>\n"+
+" <input type=\"button\" value=\"" + Messages.getAttributeString(locale,"HDFSRepositoryConnector.InsertHere") + "\" onClick='Javascript:SpecOp(\"specop"+instanceDescription+"\",\"Insert Here\",\"match_"+Integer.toString(k)+"_"+Integer.toString(j+1)+"\")' alt=\""+Messages.getAttributeString(locale,"HDFSRepositoryConnector.InsertNewMatchForPath")+Integer.toString(k)+" before position #"+Integer.toString(j)+"\"/>\n"+
+" </nobr>\n"+
+" </td>\n"+
+" <td class=\"formcolumncell\">\n"+
+" <nobr>\n"+
+" <select name=\""+"specflavor"+instanceDescription+"\">\n"+
+" <option value=\"include\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.include") + "</option>\n"+
+" <option value=\"exclude\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.exclude") + "</option>\n"+
+" </select>\n"+
+" </nobr>\n"+
+" </td>\n"+
+" <td class=\"formcolumncell\">\n"+
+" <nobr>\n"+
+" <select name=\""+"spectype"+instanceDescription+"\">\n"+
+" <option value=\"file\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.File") + "</option>\n"+
+" <option value=\"directory\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.Directory") + "</option>\n"+
+" </select>\n"+
+" </nobr>\n"+
+" </td>\n"+
+" <td class=\"formcolumncell\">\n"+
+" <nobr>\n"+
+" <input type=\"text\" size=\"10\" name=\""+"specmatch"+instanceDescription+"\" value=\"\"/>\n"+
+" </nobr>\n"+
+" </td>\n"+
+" </tr>\n"+
+" <tr class=\"oddformrow\">\n"+
+" <td class=\"formcolumncell\">\n"+
+" <nobr>\n"+
+" <input type=\"hidden\" name=\""+"specop"+instanceDescription+"\" value=\"\"/>\n"+
+" <input type=\"hidden\" name=\""+"specfl"+instanceDescription+"\" value=\""+nodeFlavor+"\"/>\n"+
+" <input type=\"hidden\" name=\""+"specty"+instanceDescription+"\" value=\""+nodeType+"\"/>\n"+
+" <input type=\"hidden\" name=\""+"specma"+instanceDescription+"\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(nodeMatch)+"\"/>\n"+
+" <a name=\""+"match_"+Integer.toString(k)+"_"+Integer.toString(j)+"\">\n"+
+" <input type=\"button\" value=\"" + Messages.getAttributeString(locale,"HDFSRepositoryConnector.Delete") + "\" onClick='Javascript:SpecOp(\"specop"+instanceDescription+"\",\"Delete\",\"match_"+Integer.toString(k)+"_"+Integer.toString(j)+"\")' alt=\""+Messages.getAttributeString(locale,"HDFSRepositoryConnector.DeletePath")+Integer.toString(k)+", match spec #"+Integer.toString(j)+"\"/>\n"+
+" </a>\n"+
+" </nobr>\n"+
+" </td>\n"+
+" <td class=\"formcolumncell\">\n"+
+" <nobr>\n"+
+" "+nodeFlavor+"\n"+
+" </nobr>\n"+
+" </td>\n"+
+" <td class=\"formcolumncell\">\n"+
+" <nobr>\n"+
+" "+nodeType+"\n"+
+" </nobr>\n"+
+" </td>\n"+
+" <td class=\"formcolumncell\">\n"+
+" <nobr>\n"+
+" "+org.apache.manifoldcf.ui.util.Encoder.bodyEscape(nodeMatch)+"\n"+
+" </nobr>\n"+
+" </td>\n"+
+" </tr>\n"
+ );
+ j++;
+ }
+ if (j == 0)
+ {
+ out.print(
+" <tr class=\"formrow\"><td class=\"message\" colspan=\"4\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.NoRulesDefined") + "</td></tr>\n"
+ );
+ }
+ out.print(
+" <tr class=\"formrow\"><td class=\"lightseparator\" colspan=\"4\"><hr/></td></tr>\n"+
+" <tr class=\"formrow\">\n"+
+" <td class=\"formcolumncell\">\n"+
+" <a name=\""+"match_"+Integer.toString(k)+"_"+Integer.toString(j)+"\">\n"+
+" <input type=\"button\" value=\"" + Messages.getAttributeString(locale,"HDFSRepositoryConnector.Add") + "\" onClick='Javascript:SpecOp(\""+pathOpName+"\",\"Add\",\"match_"+Integer.toString(k)+"_"+Integer.toString(j+1)+"\")' alt=\""+Messages.getAttributeString(locale,"HDFSRepositoryConnector.AddNewMatchForPath")+Integer.toString(k)+"\"/>\n"+
+" </a>\n"+
+" </td>\n"+
+" <td class=\"formcolumncell\">\n"+
+" <nobr>\n"+
+" <select name=\""+"specflavor"+pathDescription+"\">\n"+
+" <option value=\"include\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.include") + "</option>\n"+
+" <option value=\"exclude\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.exclude") + "</option>\n"+
+" </select>\n"+
+" </nobr>\n"+
+" </td>\n"+
+" <td class=\"formcolumncell\">\n"+
+" <nobr>\n"+
+" <select name=\""+"spectype"+pathDescription+"\">\n"+
+" <option value=\"file\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.File") + "</option>\n"+
+" <option value=\"directory\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.Directory") + "</option>\n"+
+" </select>\n"+
+" </nobr>\n"+
+" </td>\n"+
+" <td class=\"formcolumncell\">\n"+
+" <nobr>\n"+
+" <input type=\"text\" size=\"10\" name=\""+"specmatch"+pathDescription+"\" value=\"\"/>\n"+
+" </nobr>\n"+
+" </td>\n"+
+" </tr>\n"+
+" </table>\n"+
+" </td>\n"+
+" </tr>\n"
+ );
+ k++;
+ }
+ }
+ if (k == 0)
+ {
+ out.print(
+" <tr class=\"formrow\"><td class=\"message\" colspan=\"3\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.NoDocumentsSpecified") + "</td></tr>\n"
+ );
+ }
+ out.print(
+" <tr class=\"formrow\"><td class=\"lightseparator\" colspan=\"3\"><hr/></td></tr>\n"+
+" <tr class=\"formrow\">\n"+
+" <td class=\"formcolumncell\">\n"+
+" <nobr>\n"+
+" <a name=\""+"path_"+Integer.toString(k)+"\">\n"+
+" <input type=\"button\" value=\"" + Messages.getAttributeString(locale,"HDFSRepositoryConnector.Add") + "\" onClick='Javascript:SpecOp(\"specop\",\"Add\",\"path_"+Integer.toString(i+1)+"\")' alt=\"" + Messages.getAttributeString(locale,"HDFSRepositoryConnector.AddNewPath") + "\"/>\n"+
+" <input type=\"hidden\" name=\"pathcount\" value=\""+Integer.toString(k)+"\"/>\n"+
+" <input type=\"hidden\" name=\"specop\" value=\"\"/>\n"+
+" </a>\n"+
+" </nobr>\n"+
+" </td>\n"+
+" <td class=\"formcolumncell\">\n"+
+" <nobr>\n"+
+" <input type=\"text\" size=\"80\" name=\"specpath\" value=\"\"/>\n"+
+" </nobr>\n"+
+" </td>\n"+
+" <td class=\"formcolumncell\">\n"+
+" </td>\n"+
+" </tr>\n"+
+" </table>\n"+
+" </td>\n"+
+" </tr>\n"+
+"</table>\n"
+ );
+ }
+ else
+ {
+ i = 0;
+ k = 0;
+ while (i < ds.getChildCount())
+ {
+ SpecificationNode sn = ds.getChild(i++);
+ if (sn.getType().equals("startpoint"))
+ {
+ String pathDescription = "_"+Integer.toString(k);
+ out.print(
+"<input type=\"hidden\" name=\"specpath"+pathDescription+"\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(sn.getAttributeValue("path"))+"\"/>\n"+
+"<input type=\"hidden\" name=\"specchildcount"+pathDescription+"\" value=\""+Integer.toString(sn.getChildCount())+"\"/>\n"
+ );
+
+ int j = 0;
+ while (j < sn.getChildCount())
+ {
+ SpecificationNode excludeNode = sn.getChild(j);
+ String instanceDescription = "_"+Integer.toString(k)+"_"+Integer.toString(j);
+
+ String nodeFlavor = excludeNode.getType();
+ String nodeType = excludeNode.getAttributeValue("type");
+ String nodeMatch = excludeNode.getAttributeValue("match");
+ out.print(
+"<input type=\"hidden\" name=\"specfl"+instanceDescription+"\" value=\""+nodeFlavor+"\"/>\n"+
+"<input type=\"hidden\" name=\"specty"+instanceDescription+"\" value=\""+nodeType+"\"/>\n"+
+"<input type=\"hidden\" name=\"specma"+instanceDescription+"\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(nodeMatch)+"\"/>\n"
+ );
+ j++;
+ }
+ k++;
+ }
+ }
+ out.print(
+"<input type=\"hidden\" name=\"pathcount\" value=\""+Integer.toString(k)+"\"/>\n"
+ );
+ }
+
+
+ /*
+ * get filepathtouri value
+ */
+ boolean filePathToUri = false;
+ i = 0;
+ while (i < ds.getChildCount()) {
+ SpecificationNode sn = ds.getChild(i++);
+ if (sn.getType().equals("filepathtouri")) {
+ filePathToUri = Boolean.valueOf(sn.getValue());
+ }
+ }
+
+ /*
+ * File path to URI tab
+ */
+ if (tabName.equals(Messages.getString(locale,"HDFSRepositoryConnector.FilePathToURITab"))) {
+ out.print(
+"<table class=\"displaytable\">\n"+
+" <tr><td colspan=\"2\" class=\"separator\"><hr/></td></tr>\n"+
+" <tr>\n"+
+" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.FilePathToURI") + "</nobr></td>\n"+
+" <td class=\"value\">\n"+
+" <input name=\"filepathtouri\" type=\"checkbox\" value=\"true\"" + (filePathToUri ? "checked" : "") +"/> " + Messages.getBodyString(locale,"HDFSRepositoryConnector.FilePathToURIExample") + "\n" +
+" </td>\n"+
+" </tr>\n"+
+"</table>\n"
+ );
+ } else {
+ /*
+ * File path to URI tab hiddens
+ */
+ out.print(
+"<input type=\"hidden\" name=\"filepathtouri\" value=\"" + org.apache.manifoldcf.ui.util.Encoder.attributeEscape(Boolean.toString(filePathToUri)) + "\"/>\n"
+ );
+ }
+
+ }
+
+ /** Process a specification post.
+ * This method is called at the start of job's edit or view page, whenever there is a possibility that form data for a connection has been
+ * posted. Its purpose is to gather form information and modify the document specification accordingly.
+ * The name of the posted form is "editjob".
+ *@param variableContext contains the post data, including binary file-upload information.
+ *@param ds is the current document specification for this job.
+ *@return null if all is well, or a string error message if there is an error that should prevent saving of the job (and cause a redirection to an error page).
+ */
+ @Override
+ public String processSpecificationPost(IPostParameters variableContext, Locale locale, DocumentSpecification ds)
+ throws ManifoldCFException
+ {
+ String x = variableContext.getParameter("pathcount");
+ if (x != null)
+ {
+ ds.clearChildren();
+ // Find out how many children were sent
+ int pathCount = Integer.parseInt(x);
+ // Gather up these
+ int i = 0;
+ int k = 0;
+ while (i < pathCount)
+ {
+ String pathDescription = "_"+Integer.toString(i);
+ String pathOpName = "specop"+pathDescription;
+ x = variableContext.getParameter(pathOpName);
+ if (x != null && x.equals("Delete"))
+ {
+ // Skip to the next
+ i++;
+ continue;
+ }
+ // Path inserts won't happen until the very end
+ String path = variableContext.getParameter("specpath"+pathDescription);
+ SpecificationNode node = new SpecificationNode("startpoint");
+ node.setAttribute("path",path);
+ // Now, get the number of children
+ String y = variableContext.getParameter("specchildcount"+pathDescription);
+ int childCount = Integer.parseInt(y);
+ int j = 0;
+ int w = 0;
+ while (j < childCount)
+ {
+ String instanceDescription = "_"+Integer.toString(i)+"_"+Integer.toString(j);
+ // Look for an insert or a delete at this point
+ String instanceOp = "specop"+instanceDescription;
+ String z = variableContext.getParameter(instanceOp);
+ String flavor;
+ String type;
+ String match;
+ SpecificationNode sn;
+ if (z != null && z.equals("Delete"))
+ {
+ // Process the deletion as we gather
+ j++;
+ continue;
+ }
+ if (z != null && z.equals("Insert Here"))
+ {
+ // Process the insertion as we gather.
+ flavor = variableContext.getParameter("specflavor"+instanceDescription);
+ type = variableContext.getParameter("spectype"+instanceDescription);
+ match = variableContext.getParameter("specmatch"+instanceDescription);
+ sn = new SpecificationNode(flavor);
+ sn.setAttribute("type",type);
+ sn.setAttribute("match",match);
+ node.addChild(w++,sn);
+ }
+ flavor = variableContext.getParameter("specfl"+instanceDescription);
+ type = variableContext.getParameter("specty"+instanceDescription);
+ match = variableContext.getParameter("specma"+instanceDescription);
+ sn = new SpecificationNode(flavor);
+ sn.setAttribute("type",type);
+ sn.setAttribute("match",match);
+ node.addChild(w++,sn);
+ j++;
+ }
+ if (x != null && x.equals("Add"))
+ {
+ // Process adds to the end of the rules in-line
+ String match = variableContext.getParameter("specmatch"+pathDescription);
+ String type = variableContext.getParameter("spectype"+pathDescription);
+ String flavor = variableContext.getParameter("specflavor"+pathDescription);
+ SpecificationNode sn = new SpecificationNode(flavor);
+ sn.setAttribute("type",type);
+ sn.setAttribute("match",match);
+ node.addChild(w,sn);
+ }
+ ds.addChild(k++,node);
+ i++;
+ }
+
+ // See if there's a global add operation
+ String op = variableContext.getParameter("specop");
+ if (op != null && op.equals("Add"))
+ {
+ String path = variableContext.getParameter("specpath");
+ SpecificationNode node = new SpecificationNode("startpoint");
+ node.setAttribute("path",path);
+
+ // Now add in the defaults; these will be "include all directories" and "include all files".
+ SpecificationNode sn = new SpecificationNode("include");
+ sn.setAttribute("type","file");
+ sn.setAttribute("match","*");
+ node.addChild(node.getChildCount(),sn);
+ sn = new SpecificationNode("include");
+ sn.setAttribute("type","directory");
+ sn.setAttribute("match","*");
+ node.addChild(node.getChildCount(),sn);
+
+ ds.addChild(k,node);
+ }
+ }
+
+ /*
+ * "filepathtouri"
+ */
+ String filepathtouri = variableContext.getParameter("filepathtouri");
+ if (filepathtouri != null) {
+ SpecificationNode sn;
+ int i = 0;
+ while (i < ds.getChildCount()) {
+ if (ds.getChild(i).getType().equals("filepathtouri")) {
+ ds.removeChild(i);
+ } else {
+ i++;
+ }
+ }
+ sn = new SpecificationNode("filepathtouri");
+ sn.setValue(filepathtouri);
+ ds.addChild(ds.getChildCount(),sn);
+ }
+
+ return null;
+ }
+
+ /** View specification.
+ * This method is called in the body section of a job's view page. Its purpose is to present the document specification information to the user.
+ * The coder can presume that the HTML that is output from this configuration will be within appropriate <html> and <body> tags.
+ *@param out is the output to which any HTML should be sent.
+ *@param ds is the current document specification for this job.
+ */
+ @Override
+ public void viewSpecification(IHTTPOutput out, Locale locale, DocumentSpecification ds)
+ throws ManifoldCFException, IOException
+ {
+ int i = 0;
+
+ out.print(
+"<table class=\"displaytable\">\n"+
+" <tr>\n"+
+" <td colspan=\"2\" class=\"message\">" + Messages.getAttributeString(locale,"HDFSRepositoryConnector.Paths") + "</td>\n"+
+" </tr>\n"
+ );
+
+ i = 0;
+ boolean seenAny = false;
+ while (i < ds.getChildCount())
+ {
+ SpecificationNode sn = ds.getChild(i++);
+ if (sn.getType().equals("startpoint"))
+ {
+ if (seenAny == false)
+ {
+ seenAny = true;
+ }
+ out.print(
+" <tr>\n"+
+" <td class=\"description\">"+org.apache.manifoldcf.ui.util.Encoder.bodyEscape(sn.getAttributeValue("path"))+":"+"</td>\n"+
+" <td class=\"value\">\n"
+ );
+ int j = 0;
+ while (j < sn.getChildCount())
+ {
+ SpecificationNode excludeNode = sn.getChild(j++);
+ out.print(
+" "+(excludeNode.getType().equals("include")?"Include ":"")+"\n"+
+" "+(excludeNode.getType().equals("exclude")?"Exclude ":"")+"\n"+
+" "+(excludeNode.getAttributeValue("type").equals("file")?"file ":"")+"\n"+
+" "+(excludeNode.getAttributeValue("type").equals("directory")?"directory ":"")+"\n"+
+" "+org.apache.manifoldcf.ui.util.Encoder.bodyEscape(excludeNode.getAttributeValue("match"))+"<br/>\n"
+ );
+ }
+ out.print(
+" </td>\n"+
+" </tr>\n"
+ );
+ }
+ }
+ if (seenAny == false)
+ {
+ out.print(
+" <tr><td class=\"message\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.NoDocumentsSpecified") + "</td></tr>\n"
+ );
+ }
+ out.print(
+"</table>\n"
+ );
+
+ /*
+ * get filepathtouri value
+ */
+ boolean filePathToUri = false;
+ i = 0;
+ while (i < ds.getChildCount()) {
+ SpecificationNode sn = ds.getChild(i++);
+ if (sn.getType().equals("filepathtouri")) {
+ filePathToUri = Boolean.valueOf(sn.getValue());
+ }
+ }
+
+ out.print(
+"<table class=\"displaytable\">\n"+
+" <tr><td colspan=\"2\" class=\"separator\"><hr/></td></tr>\n"+
+" <tr>\n"+
+" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.FilePathToURI") + "</nobr></td>\n"+
+" <td class=\"value\">" + org.apache.manifoldcf.ui.util.Encoder.bodyEscape(Boolean.toString(filePathToUri)) + "</td>\n"+
+" </tr>\n"+
+"</table>\n"
+ );
+
+ }
+
+ // Protected static methods
+
+ /** Check if a file or directory should be included, given a document specification.
+ *@param fileName is the canonical file name.
+ *@param documentSpecification is the specification.
+ *@return true if it should be included.
+ */
+ protected static boolean checkInclude(String nameNode, FileStatus fileStatus, String fileName, DocumentSpecification documentSpecification)
+ throws ManifoldCFException
+ {
+ /*
+ * TODO:
+ * fileName = hdfs://localhost:9000/user/minoru/KEN_ALL_UTF-8_UNIX_SHRINK.CSV
+ * pathPart = hdfs://localhost:9000/user/minoru
+ * fliePart = KEN_ALL_UTF-8_UNIX_SHRINK.CSV
+ * path = /user/minoru => hdfs://localhost:9000/user/minoru
+ */
+ if (Logging.connectors.isDebugEnabled())
+ {
+ Logging.connectors.debug("Checking whether to include file '"+fileName+"'");
+ }
+
+ String pathPart;
+ String filePart;
+ if (fileStatus.isDir())
+ {
+ pathPart = fileName;
+ filePart = null;
+ }
+ else
+ {
+ pathPart = fileStatus.getPath().getParent().toString();
+ filePart = fileStatus.getPath().getName();
+ }
+
+ // Scan until we match a startpoint
+ int i = 0;
+ while (i < documentSpecification.getChildCount())
+ {
+ SpecificationNode sn = documentSpecification.getChild(i++);
+ if (sn.getType().equals("startpoint"))
+ {
+ String path = null;
+ try {
+ path = new URI(nameNode).resolve(sn.getAttributeValue("path")).toString();
+ } catch (URISyntaxException e) {
+ e.printStackTrace();
+ }
+ if (Logging.connectors.isDebugEnabled())
+ {
+ Logging.connectors.debug("Checking path '"+path+"' against canonical '"+pathPart+"'");
+ }
+ // Compare with filename
+ int matchEnd = matchSubPath(path,pathPart);
+ if (matchEnd == -1)
+ {
+ if (Logging.connectors.isDebugEnabled())
+ {
+ Logging.connectors.debug("Match check '"+path+"' against canonical '"+pathPart+"' failed");
+ }
+
+ continue;
+ }
+ // matchEnd is the start of the rest of the path (after the match) in fileName.
+ // We need to walk through the rules and see whether it's in or out.
+ int j = 0;
+ while (j < sn.getChildCount())
+ {
+ SpecificationNode node = sn.getChild(j++);
+ String flavor = node.getType();
+ String match = node.getAttributeValue("match");
+ String type = node.getAttributeValue("type");
+ // If type is "file", then our match string is against the filePart.
+ // If filePart is null, then this rule is simply skipped.
+ String sourceMatch;
+ int sourceIndex;
+ if (type.equals("file"))
+ {
+ if (filePart == null)
+ {
+ continue;
+ }
+ sourceMatch = filePart;
+ sourceIndex = 0;
+ }
+ else
+ {
+ if (filePart != null)
+ {
+ continue;
+ }
+ sourceMatch = pathPart;
+ sourceIndex = matchEnd;
+ }
+
+ if (flavor.equals("include"))
+ {
+ if (checkMatch(sourceMatch,sourceIndex,match))
+ {
+ return true;
+ }
+ }
+ else if (flavor.equals("exclude"))
+ {
+ if (checkMatch(sourceMatch,sourceIndex,match))
+ {
+ return false;
+ }
+ }
+ }
+ }
+ }
+ if (Logging.connectors.isDebugEnabled())
+ {
+ Logging.connectors.debug("Not including '"+fileName+"' because no matching rules");
+ }
+
+ return false;
+ }
+
+ /** Check if a file should be ingested, given a document specification. It is presumed that
+ * documents that do not pass checkInclude() will be checked with this method.
+ *@param file is the file.
+ *@param documentSpecification is the specification.
+ */
+ protected static boolean checkIngest(String nameNode, FileStatus fileStatus, DocumentSpecification documentSpecification)
+ throws ManifoldCFException
+ {
+ // Since the only exclusions at this point are not based on file contents, this is a no-op.
+ // MHL
+ return true;
+ }
+
+ /** Match a sub-path. The sub-path must match the complete starting part of the full path, in a path
+ * sense. The returned value should point into the file name beyond the end of the matched path, or
+ * be -1 if there is no match.
+ *@param subPath is the sub path.
+ *@param fullPath is the full path.
+ *@return the index of the start of the remaining part of the full path, or -1.
+ */
+ protected static int matchSubPath(String subPath, String fullPath)
+ {
+ if (subPath.length() > fullPath.length())
+ return -1;
+ if (fullPath.startsWith(subPath) == false)
+ return -1;
+ int rval = subPath.length();
+ if (fullPath.length() == rval)
+ return rval;
+ char x = fullPath.charAt(rval);
+ if (x == Path.SEPARATOR_CHAR)
+ rval++;
+ return rval;
+ }
+
+ /** Check a match between two strings with wildcards.
+ *@param sourceMatch is the expanded string (no wildcards)
+ *@param sourceIndex is the starting point in the expanded string.
+ *@param match is the wildcard-based string.
+ *@return true if there is a match.
+ */
+ protected static boolean checkMatch(String sourceMatch, int sourceIndex, String match)
+ {
+ // Note: The java regex stuff looks pretty heavyweight for this purpose.
+ // I've opted to try and do a simple recursive version myself, which is not compiled.
+ // Basically, the match proceeds by recursive descent through the string, so that all *'s cause
+ // recursion.
+ boolean caseSensitive = true;
+
+ return processCheck(caseSensitive, sourceMatch, sourceIndex, match, 0);
+ }
+
+ /** Recursive worker method for checkMatch. Returns 'true' if there is a path that consumes both
+ * strings in their entirety in a matched way.
+ *@param caseSensitive is true if file names are case sensitive.
+ *@param sourceMatch is the source string (w/o wildcards)
+ *@param sourceIndex is the current point in the source string.
+ *@param match is the match string (w/wildcards)
+ *@param matchIndex is the current point in the match string.
+ *@return true if there is a match.
+ */
+ protected static boolean processCheck(boolean caseSensitive, String sourceMatch, int sourceIndex,
+ String match, int matchIndex)
+ {
+ // Logging.connectors.debug("Matching '"+sourceMatch+"' position "+Integer.toString(sourceIndex)+
+ // " against '"+match+"' position "+Integer.toString(matchIndex));
+
+ // Match up through the next * we encounter
+ while (true)
+ {
+ // If we've reached the end, it's a match.
+ if (sourceMatch.length() == sourceIndex && match.length() == matchIndex)
+ return true;
+ // If one has reached the end but the other hasn't, no match
+ if (match.length() == matchIndex)
+ return false;
+ if (sourceMatch.length() == sourceIndex)
+ {
+ if (match.charAt(matchIndex) != '*')
+ return false;
+ matchIndex++;
+ continue;
+ }
+ char x = sourceMatch.charAt(sourceIndex);
+ char y = match.charAt(matchIndex);
+ if (!caseSensitive)
+ {
+ if (x >= 'A' && x <= 'Z')
+ x -= 'A'-'a';
+ if (y >= 'A' && y <= 'Z')
+ y -= 'A'-'a';
+ }
+ if (y == '*')
+ {
+ // Wildcard!
+ // We will recurse at this point.
+ // Basically, we want to combine the results for leaving the "*" in the match string
+ // at this point and advancing the source index, with skipping the "*" and leaving the source
+ // string alone.
+ return processCheck(caseSensitive,sourceMatch,sourceIndex+1,match,matchIndex) ||
+ processCheck(caseSensitive,sourceMatch,sourceIndex,match,matchIndex+1);
+ }
+ if (y == '?' || x == y)
+ {
+ sourceIndex++;
+ matchIndex++;
+ }
+ else
+ return false;
+ }
+ }
+
+ /** Document identifier stream.
+ */
+ protected static class IdentifierStream implements IDocumentIdentifierStream
+ {
+ protected String[] ids = null;
+ protected int currentIndex = 0;
+
+ public IdentifierStream(DocumentSpecification spec)
+ throws ManifoldCFException
+ {
+ // Walk the specification for the "startpoint" types. Amalgamate these into a list of strings.
+ // Presume that all roots are startpoint nodes
+ int i = 0;
+ int j = 0;
+ while (i < spec.getChildCount())
+ {
+ SpecificationNode n = spec.getChild(i);
+ if (n.getType().equals("startpoint"))
+ {
+ j++;
+ }
+ i++;
+ }
+ ids = new String[j];
+ i = 0;
+ j = 0;
+ while (i < ids.length)
+ {
+ SpecificationNode n = spec.getChild(i);
+ if (n.getType().equals("startpoint"))
+ {
+ // The id returned MUST be in canonical form!!!
+ ids[j] = new Path(n.getAttributeValue("path")).toString();
+ if (Logging.connectors.isDebugEnabled())
+ {
+ Logging.connectors.debug("Seed = '"+ids[j]+"'");
+ }
+ j++;
+ }
+ i++;
+ }
+ }
+
+ /** Get the next identifier.
+ *@return the next document identifier, or null if there are no more.
+ */
+ public String getNextIdentifier()
+ throws ManifoldCFException, ServiceInterruption
+ {
+ if (currentIndex == ids.length)
+ return null;
+ return ids[currentIndex++];
+ }
+
+ /** Close the stream.
+ */
+ public void close()
+ throws ManifoldCFException
+ {
+ ids = null;
+ }
+
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/Messages.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/Messages.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/Messages.java (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/Messages.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,141 @@
+/* $Id: Messages.java 1295926 2012-03-01 21:56:27Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.connectors.hdfs;
+
+import java.util.Locale;
+import java.util.Map;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+
+public class Messages extends org.apache.manifoldcf.ui.i18n.Messages
+{
+ public static final String DEFAULT_BUNDLE_NAME="org.apache.manifoldcf.crawler.connectors.hdfs.common";
+ public static final String DEFAULT_PATH_NAME="org.apache.manifoldcf.crawler.connectors.hdfs";
+
+ /** Constructor - do no instantiate
+ */
+ protected Messages()
+ {
+ }
+
+ public static String getString(Locale locale, String messageKey)
+ {
+ return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getAttributeString(Locale locale, String messageKey)
+ {
+ return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getBodyString(Locale locale, String messageKey)
+ {
+ return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getAttributeJavascriptString(Locale locale, String messageKey)
+ {
+ return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getBodyJavascriptString(Locale locale, String messageKey)
+ {
+ return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getString(Locale locale, String messageKey, Object[] args)
+ {
+ return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getAttributeString(Locale locale, String messageKey, Object[] args)
+ {
+ return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getBodyString(Locale locale, String messageKey, Object[] args)
+ {
+ return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getAttributeJavascriptString(Locale locale, String messageKey, Object[] args)
+ {
+ return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getBodyJavascriptString(Locale locale, String messageKey, Object[] args)
+ {
+ return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ // More general methods which allow bundlenames and class loaders to be specified.
+
+ public static String getString(String bundleName, Locale locale, String messageKey, Object[] args)
+ {
+ return getString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getAttributeString(String bundleName, Locale locale, String messageKey, Object[] args)
+ {
+ return getAttributeString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getBodyString(String bundleName, Locale locale, String messageKey, Object[] args)
+ {
+ return getBodyString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getAttributeJavascriptString(String bundleName, Locale locale, String messageKey, Object[] args)
+ {
+ return getAttributeJavascriptString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getBodyJavascriptString(String bundleName, Locale locale, String messageKey, Object[] args)
+ {
+ return getBodyJavascriptString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ // Resource output
+
+ public static void outputResource(IHTTPOutput output, Locale locale, String resourceKey,
+ Map<String,String> substitutionParameters, boolean mapToUpperCase)
+ throws ManifoldCFException
+ {
+ outputResource(output,Messages.class,DEFAULT_PATH_NAME,locale,resourceKey,
+ substitutionParameters,mapToUpperCase);
+ }
+
+ public static void outputResourceWithVelocity(IHTTPOutput output, Locale locale, String resourceKey,
+ Map<String,String> substitutionParameters, boolean mapToUpperCase)
+ throws ManifoldCFException
+ {
+ outputResourceWithVelocity(output,Messages.class,DEFAULT_BUNDLE_NAME,DEFAULT_PATH_NAME,locale,resourceKey,
+ substitutionParameters,mapToUpperCase);
+ }
+
+ public static void outputResourceWithVelocity(IHTTPOutput output, Locale locale, String resourceKey,
+ Map<String,Object> contextObjects)
+ throws ManifoldCFException
+ {
+ outputResourceWithVelocity(output,Messages.class,DEFAULT_BUNDLE_NAME,DEFAULT_PATH_NAME,locale,resourceKey,
+ contextObjects);
+ }
+
+}
+
Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/Messages.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_en_US.properties?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_en_US.properties (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_en_US.properties Mon Jun 24 03:26:56 2013
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDFSOutputConnector.ServerTabName=Server
+HDFSOutputConnector.NameNode=Name Node:
+HDFSOutputConnector.User=User:
+
+HDFSOutputConnector.PathTabName=Output Path
+HDFSOutputConnector.Path=Output Path:
+HDFSOutputConnector.RootPath=Root path:
+
Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_en_US.properties
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_ja_JP.properties?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_ja_JP.properties (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_ja_JP.properties Mon Jun 24 03:26:56 2013
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDFSOutputConnector.ServerTabName=ãµã¼ãã¼
+HDFSOutputConnector.NameNode=ãã¼ã ãã¼ãï¼
+HDFSOutputConnector.User=ã¦ã¼ã¶ã¼:
+
+HDFSOutputConnector.PathTabName=åºåãã¹
+HDFSOutputConnector.Path=åºåãã¹ï¼
+HDFSOutputConnector.RootPath=ã«ã¼ããã¹ï¼
Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_ja_JP.properties
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/hdfs/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/hdfs/common_en_US.properties?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/hdfs/common_en_US.properties (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/hdfs/common_en_US.properties Mon Jun 24 03:26:56 2013
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDFSRepositoryConnector.ServerTabName=Server
+HDFSRepositoryConnector.NameNode=Name Node:
+HDFSRepositoryConnector.User=User:
+
+HDFSRepositoryConnector.Paths=Repository Paths
+HDFSRepositoryConnector.Paths2=Repository Paths:
+HDFSRepositoryConnector.RootPath=Root path
+HDFSRepositoryConnector.Rules=Rules
+HDFSRepositoryConnector.Delete=Delete
+HDFSRepositoryConnector.DeletePath=Delete path #
+HDFSRepositoryConnector.IncludeExclude=Include/exclude
+HDFSRepositoryConnector.FileDirectory=File/directory
+HDFSRepositoryConnector.Match=Match
+HDFSRepositoryConnector.include=include
+HDFSRepositoryConnector.exclude=exclude
+HDFSRepositoryConnector.File=File
+HDFSRepositoryConnector.Directory=Directory
+HDFSRepositoryConnector.NoDocumentsSpecified=No documents specified
+HDFSRepositoryConnector.Add=Add
+HDFSRepositoryConnector.InsertHere=Insert Here
+HDFSRepositoryConnector.include=include
+HDFSRepositoryConnector.NoRulesDefined=No rules defined
+HDFSRepositoryConnector.InsertNewMatchForPath=Insert new match for path #
+HDFSRepositoryConnector.DeletePath=Delete path #
+HDFSRepositoryConnector.AddNewMatchForPath=Add new match for path #
+HDFSRepositoryConnector.AddNewPath=Add new path
+
+HDFSRepositoryConnector.FilePathToURITab=Convert file path to URI
+HDFSRepositoryConnector.FilePathToURI=Convert file path to URI:
+HDFSRepositoryConnector.FilePathToURIExample=EX) $REPOSITORY_PATH/http/localhost/index.html => http://localhost/index.html
Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/hdfs/common_en_US.properties
------------------------------------------------------------------------------
svn:mime-type = text/plain