You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2013/09/27 13:12:45 UTC
svn commit: r1526848 [5/8] - in /hadoop/common/branches/branch-2:
hadoop-project/ hadoop-tools/ hadoop-tools/hadoop-openstack/
hadoop-tools/hadoop-openstack/src/ hadoop-tools/hadoop-openstack/src/main/
hadoop-tools/hadoop-openstack/src/main/java/ hadoo...
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.snative;
+
+import java.util.Date;
+
+/**
+ * Java mapping of Swift JSON file status.
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ */
+
+class SwiftObjectFileStatus {
+ private long bytes;
+ private String content_type;
+ private String hash;
+ private Date last_modified;
+ private String name;
+ private String subdir;
+
+ SwiftObjectFileStatus() {
+ }
+
+ SwiftObjectFileStatus(long bytes, String content_type, String hash,
+ Date last_modified, String name) {
+ this.bytes = bytes;
+ this.content_type = content_type;
+ this.hash = hash;
+ this.last_modified = last_modified;
+ this.name = name;
+ }
+
+ public long getBytes() {
+ return bytes;
+ }
+
+ public void setBytes(long bytes) {
+ this.bytes = bytes;
+ }
+
+ public String getContent_type() {
+ return content_type;
+ }
+
+ public void setContent_type(String content_type) {
+ this.content_type = content_type;
+ }
+
+ public String getHash() {
+ return hash;
+ }
+
+ public void setHash(String hash) {
+ this.hash = hash;
+ }
+
+ public Date getLast_modified() {
+ return last_modified;
+ }
+
+ public void setLast_modified(Date last_modified) {
+ this.last_modified = last_modified;
+ }
+
+ public String getName() {
+ return pathToRootPath(name);
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getSubdir() {
+ return pathToRootPath(subdir);
+ }
+
+ public void setSubdir(String subdir) {
+ this.subdir = subdir;
+ }
+
+ /**
+ * If path doesn't starts with '/'
+ * method will concat '/'
+ *
+ * @param path specified path
+ * @return root path string
+ */
+ private String pathToRootPath(String path) {
+ if (path == null) {
+ return null;
+ }
+
+ if (path.startsWith("/")) {
+ return path;
+ }
+
+ return "/".concat(path);
+ }
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/Duration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/Duration.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/Duration.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/Duration.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+public class Duration {
+
+ private final long started;
+ private long finished;
+
+ public Duration() {
+ started = time();
+ finished = started;
+ }
+
+ private long time() {
+ return System.currentTimeMillis();
+ }
+
+ public void finished() {
+ finished = time();
+ }
+
+ public String getDurationString() {
+ return humanTime(value());
+ }
+
+ public static String humanTime(long time) {
+ long seconds = (time / 1000);
+ long minutes = (seconds / 60);
+ return String.format("%d:%02d:%03d", minutes, seconds % 60, time % 1000);
+ }
+
+ @Override
+ public String toString() {
+ return getDurationString();
+ }
+
+ public long value() {
+ return finished -started;
+ }
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStats.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStats.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStats.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStats.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+/**
+ * Build ongoing statistics from duration data
+ */
+public class DurationStats {
+
+ final String operation;
+ int n;
+ long sum;
+ long min;
+ long max;
+ double mean, m2;
+
+ /**
+ * Construct statistics for a given operation.
+ * @param operation operation
+ */
+ public DurationStats(String operation) {
+ this.operation = operation;
+ reset();
+ }
+
+ /**
+ * construct from anothr stats entry;
+ * all value are copied.
+ * @param that the source statistics
+ */
+ public DurationStats(DurationStats that) {
+ operation = that.operation;
+ n = that.n;
+ sum = that.sum;
+ min = that.min;
+ max = that.max;
+ mean = that.mean;
+ m2 = that.m2;
+ }
+
+ /**
+ * Add a duration
+ * @param duration the new duration
+ */
+ public void add(Duration duration) {
+ add(duration.value());
+ }
+
+ /**
+ * Add a number
+ * @param x the number
+ */
+ public void add(long x) {
+ n++;
+ sum += x;
+ double delta = x - mean;
+ mean += delta / n;
+ m2 += delta * (x - mean);
+ if (x < min) {
+ min = x;
+ }
+ if (x > max) {
+ max = x;
+ }
+ }
+
+ /**
+ * Reset the data
+ */
+ public void reset() {
+ n = 0;
+ sum = 0;
+ sum = 0;
+ min = 10000000;
+ max = 0;
+ mean = 0;
+ m2 = 0;
+ }
+
+ /**
+ * Get the number of entries sampled
+ * @return the number of durations added
+ */
+ public int getCount() {
+ return n;
+ }
+
+ /**
+ * Get the sum of all durations
+ * @return all the durations
+ */
+ public long getSum() {
+ return sum;
+ }
+
+ /**
+ * Get the arithmetic mean of the aggregate statistics
+ * @return the arithmetic mean
+ */
+ public double getArithmeticMean() {
+ return mean;
+ }
+
+ /**
+ * Variance, sigma^2
+ * @return variance, or, if no samples are there, 0.
+ */
+ public double getVariance() {
+ return n > 0 ? (m2 / (n - 1)) : 0;
+ }
+
+ /**
+ * Get the std deviation, sigma
+ * @return the stddev, 0 may mean there are no samples.
+ */
+ public double getDeviation() {
+ double variance = getVariance();
+ return (variance > 0) ? Math.sqrt(variance) : 0;
+ }
+
+ /**
+ * Covert to a useful string
+ * @return a human readable summary
+ */
+ @Override
+ public String toString() {
+ return String.format(
+ "%s count=%d total=%.3fs mean=%.3fs stddev=%.3fs min=%.3fs max=%.3fs",
+ operation,
+ n,
+ sum / 1000.0,
+ mean / 1000.0,
+ getDeviation() / 1000000.0,
+ min / 1000.0,
+ max / 1000.0);
+ }
+
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStatsTable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStatsTable.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStatsTable.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStatsTable.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift.util;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Build a duration stats table to which you can add statistics.
+ * Designed to be multithreaded
+ */
+public class DurationStatsTable {
+
+ private Map<String,DurationStats> statsTable
+ = new HashMap<String, DurationStats>(6);
+
+ /**
+ * Add an operation
+ * @param operation operation name
+ * @param duration duration
+ */
+ public void add(String operation, Duration duration, boolean success) {
+ DurationStats durationStats;
+ String key = operation;
+ if (!success) {
+ key += "-FAIL";
+ }
+ synchronized (this) {
+ durationStats = statsTable.get(key);
+ if (durationStats == null) {
+ durationStats = new DurationStats(key);
+ statsTable.put(key, durationStats);
+ }
+ }
+ synchronized (durationStats) {
+ durationStats.add(duration);
+ }
+ }
+
+ /**
+ * Get the current duration statistics
+ * @return a snapshot of the statistics
+ */
+ public synchronized List<DurationStats> getDurationStatistics() {
+ List<DurationStats> results = new ArrayList<DurationStats>(statsTable.size());
+ for (DurationStats stat: statsTable.values()) {
+ results.add(new DurationStats(stat));
+ }
+ return results;
+ }
+
+ /**
+ * reset the values of the statistics. This doesn't delete them, merely zeroes them.
+ */
+ public synchronized void reset() {
+ for (DurationStats stat : statsTable.values()) {
+ stat.reset();
+ }
+ }
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/JSONUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/JSONUtil.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/JSONUtil.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/JSONUtil.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+import org.apache.hadoop.fs.swift.exceptions.SwiftJsonMarshallingException;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.type.CollectionType;
+import org.codehaus.jackson.type.TypeReference;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.io.Writer;
+
+
+public class JSONUtil {
+ private static ObjectMapper jsonMapper = new ObjectMapper();
+
+ /**
+ * Private constructor.
+ */
+ private JSONUtil() {
+ }
+
+ /**
+ * Converting object to JSON string. If errors appears throw
+ * MeshinException runtime exception.
+ *
+ * @param object The object to convert.
+ * @return The JSON string representation.
+ * @throws IOException IO issues
+ * @throws SwiftJsonMarshallingException failure to generate JSON
+ */
+ public static String toJSON(Object object) throws
+ IOException {
+ Writer json = new StringWriter();
+ try {
+ jsonMapper.writeValue(json, object);
+ return json.toString();
+ } catch (JsonGenerationException e) {
+ throw new SwiftJsonMarshallingException(e.toString(), e);
+ } catch (JsonMappingException e) {
+ throw new SwiftJsonMarshallingException(e.toString(), e);
+ }
+ }
+
+ /**
+ * Convert string representation to object. If errors appears throw
+ * Exception runtime exception.
+ *
+ * @param value The JSON string.
+ * @param klazz The class to convert.
+ * @return The Object of the given class.
+ */
+ public static <T> T toObject(String value, Class<T> klazz) throws
+ IOException {
+ try {
+ return jsonMapper.readValue(value, klazz);
+ } catch (JsonGenerationException e) {
+ throw new SwiftJsonMarshallingException(e.toString()
+ + " source: " + value,
+ e);
+ } catch (JsonMappingException e) {
+ throw new SwiftJsonMarshallingException(e.toString()
+ + " source: " + value,
+ e);
+ }
+ }
+
+ /**
+ * @param value json string
+ * @param typeReference class type reference
+ * @param <T> type
+ * @return deserialized T object
+ */
+ public static <T> T toObject(String value,
+ final TypeReference<T> typeReference)
+ throws IOException {
+ try {
+ return jsonMapper.readValue(value, typeReference);
+ } catch (JsonGenerationException e) {
+ throw new SwiftJsonMarshallingException("Error generating response", e);
+ } catch (JsonMappingException e) {
+ throw new SwiftJsonMarshallingException("Error generating response", e);
+ }
+ }
+
+ /**
+ * @param value json string
+ * @param collectionType class describing how to deserialize collection of objects
+ * @param <T> type
+ * @return deserialized T object
+ */
+ public static <T> T toObject(String value,
+ final CollectionType collectionType)
+ throws IOException {
+ try {
+ return jsonMapper.readValue(value, collectionType);
+ } catch (JsonGenerationException e) {
+ throw new SwiftJsonMarshallingException(e.toString()
+ + " source: " + value,
+ e);
+ } catch (JsonMappingException e) {
+ throw new SwiftJsonMarshallingException(e.toString()
+ + " source: " + value,
+ e);
+ }
+ }
+
+ public static ObjectMapper getJsonMapper() {
+ return jsonMapper;
+ }
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftObjectPath.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftObjectPath.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftObjectPath.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftObjectPath.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
+import org.apache.hadoop.fs.swift.http.RestClientBindings;
+
+import java.net.URI;
+import java.util.regex.Pattern;
+
+/**
+ * Swift hierarchy mapping of (container, path)
+ */
+public final class SwiftObjectPath {
+ private static final Pattern PATH_PART_PATTERN = Pattern.compile(".*/AUTH_\\w*/");
+
+ /**
+ * Swift container
+ */
+ private final String container;
+
+ /**
+ * swift object
+ */
+ private final String object;
+
+ private final String uriPath;
+
+ /**
+ * Build an instance from a (host, object) pair
+ *
+ * @param container container name
+ * @param object object ref underneath the container
+ */
+ public SwiftObjectPath(String container, String object) {
+
+ this.container = container;
+ this.object = object;
+ uriPath = buildUriPath();
+ }
+
+ public String getContainer() {
+ return container;
+ }
+
+ public String getObject() {
+ return object;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof SwiftObjectPath)) return false;
+ final SwiftObjectPath that = (SwiftObjectPath) o;
+ return this.toUriPath().equals(that.toUriPath());
+ }
+
+ @Override
+ public int hashCode() {
+ int result = container.hashCode();
+ result = 31 * result + object.hashCode();
+ return result;
+ }
+
+ private String buildUriPath() {
+ return SwiftUtils.joinPaths(container, object);
+ }
+
+ public String toUriPath() {
+ return uriPath;
+ }
+
+ @Override
+ public String toString() {
+ return toUriPath();
+ }
+
+ /**
+ * Test for the object matching a path, ignoring the container
+ * value.
+ *
+ * @param path path string
+ * @return true iff the object's name matches the path
+ */
+ public boolean objectMatches(String path) {
+ return object.equals(path);
+ }
+
+
+ /**
+ * Query to see if the possibleChild object is a child path of this.
+ * object.
+ *
+ * The test is done by probing for the path of the this object being
+ * at the start of the second -with a trailing slash, and both
+ * containers being equal
+ *
+ * @param possibleChild possible child dir
+ * @return true iff the possibleChild is under this object
+ */
+ public boolean isEqualToOrParentOf(SwiftObjectPath possibleChild) {
+ String origPath = toUriPath();
+ String path = origPath;
+ if (!path.endsWith("/")) {
+ path = path + "/";
+ }
+ String childPath = possibleChild.toUriPath();
+ return childPath.equals(origPath) || childPath.startsWith(path);
+ }
+
+ /**
+ * Create a path tuple of (container, path), where the container is
+ * chosen from the host of the URI.
+ *
+ * @param uri uri to start from
+ * @param path path underneath
+ * @return a new instance.
+ * @throws SwiftConfigurationException if the URI host doesn't parse into
+ * container.service
+ */
+ public static SwiftObjectPath fromPath(URI uri,
+ Path path)
+ throws SwiftConfigurationException {
+ return fromPath(uri, path, false);
+ }
+
+ /**
+ * Create a path tuple of (container, path), where the container is
+ * chosen from the host of the URI.
+ * A trailing slash can be added to the path. This is the point where
+ * these /-es need to be appended, because when you construct a {@link Path}
+ * instance, {@link Path#normalizePath(String, String)} is called
+ * -which strips off any trailing slash.
+ *
+ * @param uri uri to start from
+ * @param path path underneath
+ * @param addTrailingSlash should a trailing slash be added if there isn't one.
+ * @return a new instance.
+ * @throws SwiftConfigurationException if the URI host doesn't parse into
+ * container.service
+ */
+ public static SwiftObjectPath fromPath(URI uri,
+ Path path,
+ boolean addTrailingSlash)
+ throws SwiftConfigurationException {
+
+ String url =
+ path.toUri().getPath().replaceAll(PATH_PART_PATTERN.pattern(), "");
+ //add a trailing slash if needed
+ if (addTrailingSlash && !url.endsWith("/")) {
+ url += "/";
+ }
+
+ String container = uri.getHost();
+ if (container == null) {
+ //no container, not good: replace with ""
+ container = "";
+ } else if (container.contains(".")) {
+ //its a container.service URI. Strip the container
+ container = RestClientBindings.extractContainerName(container);
+ }
+ return new SwiftObjectPath(container, url);
+ }
+
+
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,544 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
+import org.junit.internal.AssumptionViolatedException;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Properties;
+
+/**
+ * Utilities used across test cases
+ */
+public class SwiftTestUtils extends org.junit.Assert {
+
+ private static final Log LOG =
+ LogFactory.getLog(SwiftTestUtils.class);
+
+ public static final String TEST_FS_SWIFT = "test.fs.swift.name";
+ public static final String IO_FILE_BUFFER_SIZE = "io.file.buffer.size";
+
+ /**
+ * Get the test URI
+ * @param conf configuration
+ * @throws SwiftConfigurationException missing parameter or bad URI
+ */
+ public static URI getServiceURI(Configuration conf) throws
+ SwiftConfigurationException {
+ String instance = conf.get(TEST_FS_SWIFT);
+ if (instance == null) {
+ throw new SwiftConfigurationException(
+ "Missing configuration entry " + TEST_FS_SWIFT);
+ }
+ try {
+ return new URI(instance);
+ } catch (URISyntaxException e) {
+ throw new SwiftConfigurationException("Bad URI: " + instance);
+ }
+ }
+
+ public static boolean hasServiceURI(Configuration conf) {
+ String instance = conf.get(TEST_FS_SWIFT);
+ return instance != null;
+ }
+
+ /**
+ * Assert that a property in the property set matches the expected value
+ * @param props property set
+ * @param key property name
+ * @param expected expected value. If null, the property must not be in the set
+ */
+ public static void assertPropertyEquals(Properties props,
+ String key,
+ String expected) {
+ String val = props.getProperty(key);
+ if (expected == null) {
+ assertNull("Non null property " + key + " = " + val, val);
+ } else {
+ assertEquals("property " + key + " = " + val,
+ expected,
+ val);
+ }
+ }
+
+ /**
+ *
+ * Write a file and read it in, validating the result. Optional flags control
+ * whether file overwrite operations should be enabled, and whether the
+ * file should be deleted afterwards.
+ *
+ * If there is a mismatch between what was written and what was expected,
+ * a small range of bytes either side of the first error are logged to aid
+ * diagnosing what problem occurred -whether it was a previous file
+ * or a corrupting of the current file. This assumes that two
+ * sequential runs to the same path use datasets with different character
+ * moduli.
+ *
+ * @param fs filesystem
+ * @param path path to write to
+ * @param len length of data
+ * @param overwrite should the create option allow overwrites?
+ * @param delete should the file be deleted afterwards? -with a verification
+ * that it worked. Deletion is not attempted if an assertion has failed
+ * earlier -it is not in a <code>finally{}</code> block.
+ * @throws IOException IO problems
+ */
+ public static void writeAndRead(FileSystem fs,
+ Path path,
+ byte[] src,
+ int len,
+ int blocksize,
+ boolean overwrite,
+ boolean delete) throws IOException {
+ fs.mkdirs(path.getParent());
+
+ writeDataset(fs, path, src, len, blocksize, overwrite);
+
+ byte[] dest = readDataset(fs, path, len);
+
+ compareByteArrays(src, dest, len);
+
+ if (delete) {
+ boolean deleted = fs.delete(path, false);
+ assertTrue("Deleted", deleted);
+ assertPathDoesNotExist(fs, "Cleanup failed", path);
+ }
+ }
+
+ /**
+ * Write a file.
+ * Optional flags control
+ * whether file overwrite operations should be enabled
+ * @param fs filesystem
+ * @param path path to write to
+ * @param len length of data
+ * @param overwrite should the create option allow overwrites?
+ * @throws IOException IO problems
+ */
+ public static void writeDataset(FileSystem fs,
+ Path path,
+ byte[] src,
+ int len,
+ int blocksize,
+ boolean overwrite) throws IOException {
+ assertTrue(
+ "Not enough data in source array to write " + len + " bytes",
+ src.length >= len);
+ FSDataOutputStream out = fs.create(path,
+ overwrite,
+ fs.getConf()
+ .getInt(IO_FILE_BUFFER_SIZE,
+ 4096),
+ (short) 1,
+ blocksize);
+ out.write(src, 0, len);
+ out.close();
+ assertFileHasLength(fs, path, len);
+ }
+
+ /**
+ * Read the file and convert to a byte dataaset
+ * @param fs filesystem
+ * @param path path to read from
+ * @param len length of data to read
+ * @return the bytes
+ * @throws IOException IO problems
+ */
+ public static byte[] readDataset(FileSystem fs, Path path, int len)
+ throws IOException {
+ FSDataInputStream in = fs.open(path);
+ byte[] dest = new byte[len];
+ try {
+ in.readFully(0, dest);
+ } finally {
+ in.close();
+ }
+ return dest;
+ }
+
+ /**
+ * Assert that tthe array src[0..len] and dest[] are equal
+ * @param src source data
+ * @param dest actual
+ * @param len length of bytes to compare
+ */
+ public static void compareByteArrays(byte[] src,
+ byte[] dest,
+ int len) {
+ assertEquals("Number of bytes read != number written",
+ len, dest.length);
+ int errors = 0;
+ int first_error_byte = -1;
+ for (int i = 0; i < len; i++) {
+ if (src[i] != dest[i]) {
+ if (errors == 0) {
+ first_error_byte = i;
+ }
+ errors++;
+ }
+ }
+
+ if (errors > 0) {
+ String message = String.format(" %d errors in file of length %d",
+ errors, len);
+ LOG.warn(message);
+ // the range either side of the first error to print
+ // this is a purely arbitrary number, to aid user debugging
+ final int overlap = 10;
+ for (int i = Math.max(0, first_error_byte - overlap);
+ i < Math.min(first_error_byte + overlap, len);
+ i++) {
+ byte actual = dest[i];
+ byte expected = src[i];
+ String letter = toChar(actual);
+ String line = String.format("[%04d] %2x %s\n", i, actual, letter);
+ if (expected != actual) {
+ line = String.format("[%04d] %2x %s -expected %2x %s\n",
+ i,
+ actual,
+ letter,
+ expected,
+ toChar(expected));
+ }
+ LOG.warn(line);
+ }
+ fail(message);
+ }
+ }
+
+ /**
+ * Convert a byte to a character for printing. If the
+ * byte value is < 32 -and hence unprintable- the byte is
+ * returned as a two digit hex value
+ * @param b byte
+ * @return the printable character string
+ */
+ public static String toChar(byte b) {
+ if (b >= 0x20) {
+ return Character.toString((char) b);
+ } else {
+ return String.format("%02x", b);
+ }
+ }
+
+ public static String toChar(byte[] buffer) {
+ StringBuilder builder = new StringBuilder(buffer.length);
+ for (byte b : buffer) {
+ builder.append(toChar(b));
+ }
+ return builder.toString();
+ }
+
+ public static byte[] toAsciiByteArray(String s) {
+ char[] chars = s.toCharArray();
+ int len = chars.length;
+ byte[] buffer = new byte[len];
+ for (int i = 0; i < len; i++) {
+ buffer[i] = (byte) (chars[i] & 0xff);
+ }
+ return buffer;
+ }
+
+ public static void cleanupInTeardown(FileSystem fileSystem,
+ String cleanupPath) {
+ cleanup("TEARDOWN", fileSystem, cleanupPath);
+ }
+
+ public static void cleanup(String action,
+ FileSystem fileSystem,
+ String cleanupPath) {
+ noteAction(action);
+ try {
+ if (fileSystem != null) {
+ fileSystem.delete(new Path(cleanupPath).makeQualified(fileSystem),
+ true);
+ }
+ } catch (Exception e) {
+ LOG.error("Error deleting in "+ action + " - " + cleanupPath + ": " + e, e);
+ }
+ }
+
+ public static void noteAction(String action) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("============== "+ action +" =============");
+ }
+ }
+
+ /**
+ * downgrade a failure to a message and a warning, then an
+ * exception for the Junit test runner to mark as failed
+ * @param message text message
+ * @param failure what failed
+ * @throws AssumptionViolatedException always
+ */
+ public static void downgrade(String message, Throwable failure) {
+ LOG.warn("Downgrading test " + message, failure);
+ AssumptionViolatedException ave =
+ new AssumptionViolatedException(failure, null);
+ throw ave;
+ }
+
+ /**
+ * report an overridden test as unsupported
+ * @param message message to use in the text
+ * @throws AssumptionViolatedException always
+ */
+ public static void unsupported(String message) {
+ throw new AssumptionViolatedException(message);
+ }
+
+ /**
+ * report a test has been skipped for some reason
+ * @param message message to use in the text
+ * @throws AssumptionViolatedException always
+ */
+ public static void skip(String message) {
+ throw new AssumptionViolatedException(message);
+ }
+
+
+ /**
+ * Make an assertion about the length of a file
+ * @param fs filesystem
+ * @param path path of the file
+ * @param expected expected length
+ * @throws IOException on File IO problems
+ */
+ public static void assertFileHasLength(FileSystem fs, Path path,
+ int expected) throws IOException {
+ FileStatus status = fs.getFileStatus(path);
+ assertEquals(
+ "Wrong file length of file " + path + " status: " + status,
+ expected,
+ status.getLen());
+ }
+
+ /**
+ * Assert that a path refers to a directory
+ * @param fs filesystem
+ * @param path path of the directory
+ * @throws IOException on File IO problems
+ */
+ public static void assertIsDirectory(FileSystem fs,
+ Path path) throws IOException {
+ FileStatus fileStatus = fs.getFileStatus(path);
+ assertIsDirectory(fileStatus);
+ }
+
+ /**
+ * Assert that a path refers to a directory
+ * @param fileStatus stats to check
+ */
+ public static void assertIsDirectory(FileStatus fileStatus) {
+ assertTrue("Should be a dir -but isn't: " + fileStatus,
+ fileStatus.isDirectory());
+ }
+
+ /**
+ * Write the text to a file, returning the converted byte array
+ * for use in validating the round trip
+ * @param fs filesystem
+ * @param path path of file
+ * @param text text to write
+ * @param overwrite should the operation overwrite any existing file?
+ * @return the read bytes
+ * @throws IOException on IO problems
+ */
+ public static byte[] writeTextFile(FileSystem fs,
+ Path path,
+ String text,
+ boolean overwrite) throws IOException {
+ FSDataOutputStream stream = fs.create(path, overwrite);
+ byte[] bytes = new byte[0];
+ if (text != null) {
+ bytes = toAsciiByteArray(text);
+ stream.write(bytes);
+ }
+ stream.close();
+ return bytes;
+ }
+
+ /**
+ * Touch a file: fails if it is already there
+ * @param fs filesystem
+ * @param path path
+ * @throws IOException IO problems
+ */
+ public static void touch(FileSystem fs,
+ Path path) throws IOException {
+ fs.delete(path, true);
+ writeTextFile(fs, path, null, false);
+ }
+
+ public static void assertDeleted(FileSystem fs,
+ Path file,
+ boolean recursive) throws IOException {
+ assertPathExists(fs, "about to be deleted file", file);
+ boolean deleted = fs.delete(file, recursive);
+ String dir = ls(fs, file.getParent());
+ assertTrue("Delete failed on " + file + ": " + dir, deleted);
+ assertPathDoesNotExist(fs, "Deleted file", file);
+ }
+
+ /**
+ * Read in "length" bytes, convert to an ascii string
+ * @param fs filesystem
+ * @param path path to read
+ * @param length #of bytes to read.
+ * @return the bytes read and converted to a string
+ * @throws IOException
+ */
+ public static String readBytesToString(FileSystem fs,
+ Path path,
+ int length) throws IOException {
+ FSDataInputStream in = fs.open(path);
+ try {
+ byte[] buf = new byte[length];
+ in.readFully(0, buf);
+ return toChar(buf);
+ } finally {
+ in.close();
+ }
+ }
+
+ public static String getDefaultWorkingDirectory() {
+ return "/user/" + System.getProperty("user.name");
+ }
+
+ public static String ls(FileSystem fileSystem, Path path) throws IOException {
+ return SwiftUtils.ls(fileSystem, path);
+ }
+
+ public static String dumpStats(String pathname, FileStatus[] stats) {
+ return pathname + SwiftUtils.fileStatsToString(stats,"\n");
+ }
+
+ /**
+ /**
+ * Assert that a file exists and whose {@link FileStatus} entry
+ * declares that this is a file and not a symlink or directory.
+ * @param fileSystem filesystem to resolve path against
+ * @param filename name of the file
+ * @throws IOException IO problems during file operations
+ */
+ public static void assertIsFile(FileSystem fileSystem, Path filename) throws
+ IOException {
+ assertPathExists(fileSystem, "Expected file", filename);
+ FileStatus status = fileSystem.getFileStatus(filename);
+ String fileInfo = filename + " " + status;
+ assertFalse("File claims to be a directory " + fileInfo,
+ status.isDirectory());
+/* disabled for Hadoop v1 compatibility
+ assertFalse("File claims to be a symlink " + fileInfo,
+ status.isSymlink());
+*/
+ }
+
+ /**
+ * Create a dataset for use in the tests; all data is in the range
+ * base to (base+modulo-1) inclusive
+ * @param len length of data
+ * @param base base of the data
+ * @param modulo the modulo
+ * @return the newly generated dataset
+ */
+ public static byte[] dataset(int len, int base, int modulo) {
+ byte[] dataset = new byte[len];
+ for (int i = 0; i < len; i++) {
+ dataset[i] = (byte) (base + (i % modulo));
+ }
+ return dataset;
+ }
+
+ /**
+ * Assert that a path exists -but make no assertions as to the
+ * type of that entry
+ *
+ * @param fileSystem filesystem to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws IOException IO problems
+ */
+ public static void assertPathExists(FileSystem fileSystem, String message,
+ Path path) throws IOException {
+ if (!fileSystem.exists(path)) {
+ //failure, report it
+ fail(message + ": not found " + path + " in " + path.getParent());
+ ls(fileSystem, path.getParent());
+ }
+ }
+
+ /**
+ * Assert that a path does not exist
+ *
+ * @param fileSystem filesystem to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws IOException IO problems
+ */
+ public static void assertPathDoesNotExist(FileSystem fileSystem,
+ String message,
+ Path path) throws IOException {
+ try {
+ FileStatus status = fileSystem.getFileStatus(path);
+ fail(message + ": unexpectedly found " + path + " as " + status);
+ } catch (FileNotFoundException expected) {
+ //this is expected
+
+ }
+ }
+
+
+ /**
+ * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry
+ * @param fs filesystem
+ * @param dir directory to scan
+ * @param subdir full path to look for
+ * @throws IOException IO probles
+ */
+ public static void assertListStatusFinds(FileSystem fs,
+ Path dir,
+ Path subdir) throws IOException {
+ FileStatus[] stats = fs.listStatus(dir);
+ boolean found = false;
+ StringBuilder builder = new StringBuilder();
+ for (FileStatus stat : stats) {
+ builder.append(stat.toString()).append('\n');
+ if (stat.getPath().equals(subdir)) {
+ found = true;
+ }
+ }
+ assertTrue("Path " + subdir
+ + " not found in directory " + dir + ":" + builder,
+ found);
+ }
+
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * Various utility classes for SwiftFS support
+ */
+public final class SwiftUtils {
+
+ public static final String READ = "read(buffer, offset, length)";
+
+ /**
+ * Join two (non null) paths, inserting a forward slash between them
+ * if needed
+ *
+ * @param path1 first path
+ * @param path2 second path
+ * @return the combined path
+ */
+ public static String joinPaths(String path1, String path2) {
+ StringBuilder result =
+ new StringBuilder(path1.length() + path2.length() + 1);
+ result.append(path1);
+ boolean insertSlash = true;
+ if (path1.endsWith("/")) {
+ insertSlash = false;
+ } else if (path2.startsWith("/")) {
+ insertSlash = false;
+ }
+ if (insertSlash) {
+ result.append("/");
+ }
+ result.append(path2);
+ return result.toString();
+ }
+
+ /**
+ * This test contains the is-directory logic for Swift, so if
+ * changed there is only one place for it.
+ *
+ * @param fileStatus status to examine
+ * @return true if we consider this status to be representative of a
+ * directory.
+ */
+ public static boolean isDirectory(FileStatus fileStatus) {
+ return fileStatus.isDirectory() || isFilePretendingToBeDirectory(fileStatus);
+ }
+
+ /**
+ * Test for the entry being a file that is treated as if it is a
+ * directory
+ *
+ * @param fileStatus status
+ * @return true if it meets the rules for being a directory
+ */
+ public static boolean isFilePretendingToBeDirectory(FileStatus fileStatus) {
+ return fileStatus.getLen() == 0;
+ }
+
+ /**
+ * Predicate: Is a swift object referring to the root direcory?
+ * @param swiftObject object to probe
+ * @return true iff the object refers to the root
+ */
+ public static boolean isRootDir(SwiftObjectPath swiftObject) {
+ return swiftObject.objectMatches("") || swiftObject.objectMatches("/");
+ }
+
+ /**
+ * Sprintf() to the log iff the log is at debug level. If the log
+ * is not at debug level, the printf operation is skipped, so
+ * no time is spent generating the string.
+ * @param log log to use
+ * @param text text message
+ * @param args args arguments to the print statement
+ */
+ public static void debug(Log log, String text, Object... args) {
+ if (log.isDebugEnabled()) {
+ log.debug(String.format(text, args));
+ }
+ }
+
+ /**
+ * Log an exception (in text and trace) iff the log is at debug
+ * @param log Log to use
+ * @param text text message
+ * @param ex exception
+ */
+ public static void debugEx(Log log, String text, Exception ex) {
+ if (log.isDebugEnabled()) {
+ log.debug(text + ex, ex);
+ }
+ }
+
+ /**
+ * Sprintf() to the log iff the log is at trace level. If the log
+ * is not at trace level, the printf operation is skipped, so
+ * no time is spent generating the string.
+ * @param log log to use
+ * @param text text message
+ * @param args args arguments to the print statement
+ */
+ public static void trace(Log log, String text, Object... args) {
+ if (log.isTraceEnabled()) {
+ log.trace(String.format(text, args));
+ }
+ }
+
+ /**
+ * Given a partition number, calculate the partition value.
+ * This is used in the SwiftNativeOutputStream, and is placed
+ * here for tests to be able to calculate the filename of
+ * a partition.
+ * @param partNumber part number
+ * @return a string to use as the filename
+ */
+ public static String partitionFilenameFromNumber(int partNumber) {
+ return String.format("%06d", partNumber);
+ }
+
+ /**
+ * List a a path to string
+ * @param fileSystem filesystem
+ * @param path directory
+ * @return a listing of the filestatuses of elements in the directory, one
+ * to a line, precedeed by the full path of the directory
+ * @throws IOException connectivity problems
+ */
+ public static String ls(FileSystem fileSystem, Path path) throws
+ IOException {
+ if (path == null) {
+ //surfaces when someone calls getParent() on something at the top of the path
+ return "/";
+ }
+ FileStatus[] stats;
+ String pathtext = "ls " + path;
+ try {
+ stats = fileSystem.listStatus(path);
+ } catch (FileNotFoundException e) {
+ return pathtext + " -file not found";
+ } catch (IOException e) {
+ return pathtext + " -failed: " + e;
+ }
+ return pathtext + fileStatsToString(stats, "\n");
+ }
+
+ /**
+ * Take an array of filestats and convert to a string (prefixed w/ a [01] counter
+ * @param stats array of stats
+ * @param separator separator after every entry
+ * @return a stringified set
+ */
+ public static String fileStatsToString(FileStatus[] stats, String separator) {
+ StringBuilder buf = new StringBuilder(stats.length * 128);
+ for (int i = 0; i < stats.length; i++) {
+ buf.append(String.format("[%02d] %s", i, stats[i])).append(separator);
+ }
+ return buf.toString();
+ }
+
+ /**
+ * Verify that the basic args to a read operation are valid;
+ * throws an exception if not -with meaningful text includeing
+ * @param buffer destination buffer
+ * @param off offset
+ * @param len number of bytes to read
+ * @throws NullPointerException null buffer
+ * @throws IndexOutOfBoundsException on any invalid range.
+ */
+ public static void validateReadArgs(byte[] buffer, int off, int len) {
+ if (buffer == null) {
+ throw new NullPointerException("Null byte array in"+ READ);
+ }
+ if (off < 0 ) {
+ throw new IndexOutOfBoundsException("Negative buffer offset "
+ + off
+ + " in " + READ);
+ }
+ if (len < 0 ) {
+ throw new IndexOutOfBoundsException("Negative read length "
+ + len
+ + " in " + READ);
+ }
+ if (off > buffer.length) {
+ throw new IndexOutOfBoundsException("Buffer offset of "
+ + off
+ + "beyond buffer size of "
+ + buffer.length
+ + " in " + READ);
+ }
+ }
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/site/apt/index.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/site/apt/index.apt.vm?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/site/apt/index.apt.vm (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/site/apt/index.apt.vm Fri Sep 27 11:12:42 2013
@@ -0,0 +1,686 @@
+~~ Licensed under the Apache License, Version 2.0 (the "License");
+~~ you may not use this file except in compliance with the License.
+~~ You may obtain a copy of the License at
+~~
+~~ http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing, software
+~~ distributed under the License is distributed on an "AS IS" BASIS,
+~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~~ See the License for the specific language governing permissions and
+~~ limitations under the License. See accompanying LICENSE file.
+
+ ---
+ Hadoop OpenStack Support: Swift Object Store
+ ---
+ ---
+ ${maven.build.timestamp}
+
+%{toc|section=1|fromDepth=0}
+
+Hadoop OpenStack Support: Swift Object Store
+
+* {Introduction}
+
+ {{{http://www.openstack.org/}OpenStack}} is an open source cloud infrastructure
+ which can be accessed
+ from multiple public IaaS providers, and deployed privately. It offers
+ infrastructure services such as VM hosting (Nova), authentication (Keystone)
+ and storage of binary objects (Swift).
+
+ This module enables Apache Hadoop applications -including MapReduce jobs,
+ read and write data to and from instances of the
+ {{{http://www.openstack.org/software/openstack-storage/}OpenStack Swift object store}}.
+
+* Features
+
+ * Read and write of data stored in a Swift object store
+
+ * Support of a pseudo-hierachical file system (directories, subdirectories and
+ files)
+
+ * Standard filesystem operations: <<<create>>>, <<<delete>>>, <<<mkdir>>>,
+ <<<ls>>>, <<<mv>>>, <<<stat>>>.
+
+ * Can act as a source of data in a MapReduce job, or a sink.
+
+ * Support for multiple OpenStack services, and multiple containers from a
+ single service.
+
+ * Supports in-cluster and remote access to Swift data.
+
+ * Supports OpenStack Keystone authentication with password or token.
+
+ * Released under the Apache Software License
+
+ * Tested against the Hadoop 3.x and 1.x branches, against multiple public
+ OpenStack clusters: Rackspace US, Rackspace UK, HP Cloud.
+
+ * Tested against private OpenStack clusters, including scalability tests of
+ large file uploads.
+
+* Using the Hadoop Swift Filesystem Client
+
+** Concepts: services and containers
+
+ OpenStack swift is an <Object Store>; also known as a <blobstore>. It stores
+ arbitrary binary objects by name in a <container>.
+
+ The Hadoop Swift filesystem library adds another concept, the <service>, which
+ defines which Swift blobstore hosts a container -and how to connect to it.
+
+** Containers and Objects
+
+ * Containers are created by users with accounts on the Swift filestore, and hold
+ <objects>.
+
+ * Objects can be zero bytes long, or they can contain data.
+
+ * Objects in the container can be up to 5GB; there is a special support for
+ larger files than this, which merges multiple objects in to one.
+
+ * Each object is referenced by it's <name>; there is no notion of directories.
+
+ * You can use any characters in an object name that can be 'URL-encoded'; the
+ maximum length of a name is 1034 characters -after URL encoding.
+
+ * Names can have <<</>>> characters in them, which are used to create the illusion of
+ a directory structure. For example <<<dir/dir2/name>>>. Even though this looks
+ like a directory, <it is still just a name>. There is no requirement to have
+ any entries in the container called <<<dir>>> or <<<dir/dir2>>>
+
+ * That said. if the container has zero-byte objects that look like directory
+ names above other objects, they can pretend to be directories. Continuing the
+ example, a 0-byte object called <<<dir>>> would tell clients that it is a
+ directory while <<<dir/dir2>>> or <<<dir/dir2/name>>> were present. This creates an
+ illusion of containers holding a filesystem.
+
+ Client applications talk to Swift over HTTP or HTTPS, reading, writing and
+ deleting objects using standard HTTP operations (GET, PUT and DELETE,
+ respectively). There is also a COPY operation, that creates a new object in the
+ container, with a new name, containing the old data. There is no rename
+ operation itself, objects need to be copied -then the original entry deleted.
+
+** Eventual Consistency
+
+ The Swift Filesystem is *eventually consistent*: an operation on an object may
+ not be immediately visible to that client, or other clients. This is a
+ consequence of the goal of the filesystem: to span a set of machines, across
+ multiple datacenters, in such a way that the data can still be available when
+ many of them fail. (In contrast, the Hadoop HDFS filesystem is *immediately
+ consistent*, but it does not span datacenters.)
+
+ Eventual consistency can cause surprises for client applications that expect
+ immediate consistency: after an object is deleted or overwritten, the object
+ may still be visible -or the old data still retrievable. The Swift Filesystem
+ client for Apache Hadoop attempts to handle this, in conjunction with the
+ MapReduce engine, but there may be still be occasions when eventual consistency
+ causes surprises.
+
+** Non-atomic "directory" operations.
+
+ Hadoop expects some
+ operations to be atomic, especially <<<rename()>>>, which is something
+ the MapReduce layer relies on to commit the output of a job, renaming data
+ from a temp directory to the final path. Because a rename
+ is implemented as a copy of every blob under the directory's path, followed
+ by a delete of the originals, the intermediate state of the operation
+ will be visible to other clients. If two Reducer tasks to rename their temp
+ directory to the final path, both operations may succeed, with the result that
+ output directory contains mixed data. This can happen if MapReduce jobs
+ are being run with <speculation> enabled and Swift used as the direct output
+ of the MR job (it can also happen against Amazon S3).
+
+ Other consequences of the non-atomic operations are:
+
+ 1. If a program is looking for the presence of the directory before acting
+ on the data -it may start prematurely. This can be avoided by using
+ other mechanisms to co-ordinate the programs, such as the presence of a file
+ that is written <after> any bulk directory operations.
+
+ 2. A <<<rename()>>> or <<<delete()>>> operation may include files added under
+ the source directory tree during the operation, may unintentionally delete
+ it, or delete the 0-byte swift entries that mimic directories and act
+ as parents for the files. Try to avoid doing this.
+
+ The best ways to avoid all these problems is not using Swift as
+ the filesystem between MapReduce jobs or other Hadoop workflows. It
+ can act as a source of data, and a final destination, but it doesn't meet
+ all of Hadoop's expectations of what a filesystem is -it's a <blobstore>.
+
+* Working with Swift Object Stores in Hadoop
+
+ Once installed, the Swift FileSystem client can be used by any Hadoop application
+ to read from or write to data stored in a Swift container.
+
+ Data stored in Swift can be used as the direct input to a MapReduce job
+ -simply use the <<<swift:>>> URL (see below) to declare the source of the data.
+
+ This Swift Filesystem client is designed to work with multiple
+ Swift object stores, both public and private. This allows the client to work
+ with different clusters, reading and writing data to and from either of them.
+
+ It can also work with the same object stores using multiple login details.
+
+ These features are achieved by one basic concept: using a service name in
+ the URI referring to a swift filesystem, and looking up all the connection and
+ login details for that specific service. Different service names can be defined
+ in the Hadoop XML configuration file, so defining different clusters, or
+ providing different login details for the same object store(s).
+
+
+** Swift Filesystem URIs
+
+ Hadoop uses URIs to refer to files within a filesystem. Some common examples
+ are:
+
++--
+ local://etc/hosts
+ hdfs://cluster1/users/example/data/set1
+ hdfs://cluster2.example.org:8020/users/example/data/set1
++--
+
+ The Swift Filesystem Client adds a new URL type <<<swift>>>. In a Swift Filesystem
+ URL, the hostname part of a URL identifies the container and the service to
+ work with; the path the name of the object. Here are some examples
+
++--
+ swift://container.rackspace/my-object.csv
+ swift://data.hpcloud/data/set1
+ swift://dmitry.privatecloud/out/results
++--
+
+ In the last two examples, the paths look like directories: it is not, they are
+ simply the objects named <<<data/set1>>> and <<<out/results>>> respectively.
+
+** Installing
+
+ The <<<hadoop-openstack>>> JAR must be on the classpath of the Hadoop program trying to
+ talk to the Swift service. If installed in the classpath of the Hadoop
+ MapReduce service, then all programs started by the MR engine will pick up the
+ JAR automatically. This is the easiest way to give all Hadoop jobs access to
+ Swift.
+
+ Alternatively, the JAR can be included as one of the JAR files that an
+ application uses. This lets the Hadoop jobs work with a Swift object store even
+ if the Hadoop cluster is not pre-configured for this.
+
+ The library also depends upon the Apache HttpComponents library, which
+ must also be on the classpath.
+
+** Configuring
+
+ To talk to a swift service, the user must must provide:
+
+ [[1]] The URL defining the container and the service.
+
+ [[1]] In the cluster/job configuration, the login details of that service.
+
+ Multiple service definitions can co-exist in the same configuration file: just
+ use different names for them.
+
+*** Example: Rackspace US, in-cluster access using API key
+
+ This service definition is for use in a Hadoop cluster deployed within Rackspace's
+ US infrastructure.
+
++--
+ <property>
+ <name>fs.swift.service.rackspace.auth.url</name>
+ <value>https://auth.api.rackspacecloud.com/v2.0/tokens</value>
+ <description>Rackspace US (multiregion)</description>
+ </property>
+
+ <property>
+ <name>fs.swift.service.rackspace.username</name>
+ <value>user4</value>
+ </property>
+
+ <property>
+ <name>fs.swift.service.rackspace.region</name>
+ <value>DFW</value>
+ </property>
+
+ <property>
+ <name>fs.swift.service.rackspace.apikey</name>
+ <value>fe806aa86dfffe2f6ed8</value>
+ </property>
++--
+
+ Here the API key visible in the account settings API keys page is used to log
+ in. No property for public/private access -the default is to use the private
+ endpoint for Swift operations.
+
+ This configuration also selects one of the regions, DFW, for its data.
+
+ A reference to this service would use the <<<rackspace>>> service name:
+
+---
+ swift://hadoop-container.rackspace/
+---
+
+*** Example: Rackspace UK: remote access with password authentication
+
+ This connects to Rackspace's UK ("LON") datacenter.
+
++--
+ <property>
+ <name>fs.swift.service.rackspaceuk.auth.url</name>
+ <value>https://lon.identity.api.rackspacecloud.com/v2.0/tokens</value>
+ <description>Rackspace UK</description>
+ </property>
+
+ <property>
+ <name>fs.swift.service.rackspaceuk.username</name>
+ <value>user4</value>
+ </property>
+
+ <property>
+ <name>fs.swift.service.rackspaceuk.password</name>
+ <value>insert-password-here/value>
+ </property>
+
+ <property>
+ <name>fs.swift.service.rackspace.public</name>
+ <value>true</value>
+ </property>
++--
+
+ This is a public access point connection, using a password over an API key.
+
+ A reference to this service would use the <<<rackspaceuk>>> service name:
+
++--
+ swift://hadoop-container.rackspaceuk/
++--
+
+ Because the public endpoint is used, if this service definition is used within
+ the London datacenter, all accesses will be billed at the public
+ upload/download rates, <irrespective of where the Hadoop cluster is>.
+
+*** Example: HP cloud service definition
+
+ Here is an example that connects to the HP Cloud object store.
+
++--
+ <property>
+ <name>fs.swift.service.hpcloud.auth.url</name>
+ <value>https://region-a.geo-1.identity.hpcloudsvc.com:35357/v2.0/tokens
+ </value>
+ <description>HP Cloud</description>
+ </property>
+
+ <property>
+ <name>fs.swift.service.hpcloud.tenant</name>
+ <value>FE806AA86</value>
+ </property>
+
+ <property>
+ <name>fs.swift.service.hpcloud.username</name>
+ <value>FE806AA86DFFFE2F6ED8</value>
+ </property>
+
+ <property>
+ <name>fs.swift.service.hpcloud.password</name>
+ <value>secret-password-goes-here</value>
+ </property>
+
+ <property>
+ <name>fs.swift.service.hpcloud.public</name>
+ <value>true</value>
+ </property>
++--
+
+ A reference to this service would use the <<<hpcloud>>> service name:
+
++--
+ swift://hadoop-container.hpcloud/
++--
+
+** General Swift Filesystem configuration options
+
+ Some configuration options apply to the Swift client, independent of
+ the specific Swift filesystem chosen.
+
+*** Blocksize fs.swift.blocksize
+
+ Swift does not break up files into blocks, except in the special case of files
+ over 5GB in length. Accordingly, there isn't a notion of a "block size"
+ to define where the data is kept.
+
+ Hadoop's MapReduce layer depends on files declaring their block size,
+ so that it knows how to partition work. Too small a blocksize means that
+ many mappers work on small pieces of data; too large a block size means
+ that only a few mappers get started.
+
+ The block size value reported by Swift, therefore, controls the basic workload
+ partioning of the MapReduce engine -and can be an important parameter to
+ tune for performance of the cluster.
+
+ The property has a unit of kilobytes; the default value is <<<32*1024>>>: 32 MB
+
++--
+ <property>
+ <name>fs.swift.blocksize</name>
+ <value>32768</value>
+ </property>
++--
+
+ This blocksize has no influence on how files are stored in Swift; it only controls
+ what the reported size of blocks are - a value used in Hadoop MapReduce to
+ divide work.
+
+ Note that the MapReduce engine's split logic can be tuned independently by setting
+ the <<<mapred.min.split.size>>> and <<<mapred.max.split.size>>> properties,
+ which can be done in specific job configurations.
+
++--
+ <property>
+ <name>mapred.min.split.size</name>
+ <value>524288</value>
+ </property>
+
+ <property>
+ <name>mapred.max.split.size</name>
+ <value>1048576</value>
+ </property>
++--
+
+ In an Apache Pig script, these properties would be set as:
+
+---
+ mapred.min.split.size 524288
+ mapred.max.split.size 1048576
+---
+
+*** Partition size fs.swift.partsize
+
+ The Swift filesystem client breaks very large files into partitioned files,
+ uploading each as it progresses, and writing any remaning data and an XML
+ manifest when a partitioned file is closed.
+
+ The partition size defaults to 4608 MB; 4.5GB, the maximum filesize that
+ Swift can support.
+
+ It is possible to set a smaller partition size, in the <<<fs.swift.partsize>>>
+ option. This takes a value in KB.
+
++--
+ <property>
+ <name>fs.swift.partsize</name>
+ <value>1024</value>
+ <description>upload every MB</description>
+ </property>
++--
+
+ When should this value be changed from its default?
+
+ While there is no need to ever change it for basic operation of
+ the Swift filesystem client, it can be tuned
+
+ * If a Swift filesystem is location aware, then breaking a file up into
+ smaller partitions scatters the data round the cluster. For best performance,
+ the property <<<fs.swift.blocksize>>> should be set to a smaller value than the
+ partition size of files.
+
+ * When writing to an unpartitioned file, the entire write is done in the
+ <<<close()>>> operation. When a file is partitioned, the outstanding data to
+ be written whenever the outstanding amount of data is greater than the
+ partition size. This means that data will be written more incrementally
+
+*** Request size fs.swift.requestsize
+
+ The Swift filesystem client reads files in HTTP GET operations, asking for
+ a block of data at a time.
+
+ The default value is 64KB. A larger value may be more efficient over faster
+ networks, as it reduces the overhead of setting up the HTTP operation.
+
+ However, if the file is read with many random accesses, requests for
+ data will be made from different parts of the file -discarding some of the
+ previously requested data. The benefits of larger request sizes may be wasted.
+
+ The property <<<fs.swift.requestsize>>> sets the request size in KB.
+
++--
+ <property>
+ <name>fs.swift.requestsize</name>
+ <value>128</value>
+ </property>
++--
+
+*** Connection timeout fs.swift.connect.timeout
+
+ This sets the timeout in milliseconds to connect to a Swift service.
+
++--
+ <property>
+ <name>fs.swift.connect.timeout</name>
+ <value>15000</value>
+ </property>
++--
+
+ A shorter timeout means that connection failures are raised faster -but
+ may trigger more false alarms. A longer timeout is more resilient to network
+ problems -and may be needed when talking to remote filesystems.
+
+*** Connection timeout fs.swift.socket.timeout
+
+ This sets the timeout in milliseconds to wait for data from a connected socket.
+
++--
+ <property>
+ <name>fs.swift.socket.timeout</name>
+ <value>60000</value>
+ </property>
++--
+
+ A shorter timeout means that connection failures are raised faster -but
+ may trigger more false alarms. A longer timeout is more resilient to network
+ problems -and may be needed when talking to remote filesystems.
+
+*** Connection Retry Count fs.swift.connect.retry.count
+
+ This sets the number of times to try to connect to a service whenever
+ an HTTP request is made.
+
++--
+ <property>
+ <name>fs.swift.connect.retry.count</name>
+ <value>3</value>
+ </property>
++--
+
+ The more retries, the more resilient it is to transient outages -and the
+ less rapid it is at detecting and reporting server connectivity problems.
+
+*** Connection Throttle Delay fs.swift.connect.throttle.delay
+
+ This property adds a delay between bulk file copy and delete operations,
+ to prevent requests being throttled or blocked by the remote service
+
++--
+ <property>
+ <name>fs.swift.connect.throttle.delay</name>
+ <value>0</value>
+ </property>
++--
+
+ It is measured in milliseconds; "0" means do not add any delay.
+
+ Throttling is enabled on the public endpoints of some Swift services.
+ If <<<rename()>>> or <<<delete()>>> operations fail with
+ <<<SwiftThrottledRequestException>>>
+ exceptions, try setting this property.
+
+*** HTTP Proxy
+
+ If the client can only access the Swift filesystem via a web proxy
+ server, the client configuration must specify the proxy via
+ the <<<fs.swift.connect.proxy.host>>> and <<<fs.swift.connect.proxy.port>>>
+ properties.
+
++--
+ <property>
+ <name>fs.swift.proxy.host</name>
+ <value>web-proxy</value>
+ </property>
+
+ <property>
+ <name>fs.swift.proxy.port</name>
+ <value>8088</value>
+ </property>
++--
+
+ If the host is declared, the proxy port must be set to a valid integer value.
+
+
+** Troubleshooting
+
+*** ClassNotFoundException
+
+ The <<<hadoop-openstack>>> JAR -or any dependencies- may not be on your classpath.
+
+ If it is a remote MapReduce job that is failing, make sure that the JAR is
+ installed on the servers in the cluster -or that the job submission process
+ uploads the JAR file to the distributed cache.
+
+*** Failure to Authenticate
+
+ A <<<SwiftAuthenticationFailedException>>> is thrown when the client
+ cannot authenticate with the OpenStack keystone server. This could be
+ because the URL in the service definition is wrong, or because
+ the supplied credentials are invalid.
+
+ [[1]] Check the authentication URL through <<<curl>>> or your browser
+
+ [[1]] Use a Swift client such as CyberDuck to validate your credentials
+
+ [[1]] If you have included a tenant ID, try leaving it out. Similarly,
+ try adding it if you had not included it.
+
+ [[1]] Try switching from API key authentication to password-based authentication,
+ by setting the password.
+
+ [[1]] Change your credentials. As with Amazon AWS clients, some credentials
+ don't seem to like going over the network.
+
+*** Timeout connecting to the Swift Service
+
+ This happens if the client application is running outside an OpenStack cluster,
+ where it does not have access to the private hostname/IP address for filesystem
+ operations. Set the <<<public>>> flag to true -but remember to set it to false
+ for use in-cluster.
+
+** Warnings
+
+ [[1]] Do not share your login details with anyone, which means do not log the
+ details, or check the XML configuration files into any revision control system
+ to which you do not have exclusive access.
+
+ [[1]] Similarly, do not use your real account details in any documentation *or any
+ bug reports submitted online*
+
+ [[1]] Prefer the apikey authentication over passwords as it is easier
+ to revoke a key -and some service providers allow you to set
+ an automatic expiry date on a key when issued.
+
+ [[1]] Do not use the public service endpoint from within a public OpenStack
+ cluster, as it will run up large bills.
+
+ [[1]] Remember: it's not a real filesystem or hierarchical directory structure.
+ Some operations (directory rename and delete) take time and are not atomic or
+ isolated from other operations taking place.
+
+ [[1]] Append is not supported.
+
+ [[1]] Unix-style permissions are not supported. All accounts with write access to
+ a repository have unlimited access; the same goes for those with read access.
+
+ [[1]] In the public clouds, do not make the containers public unless you are happy
+ with anyone reading your data, and are prepared to pay the costs of their
+ downloads.
+
+** Limits
+
+ * Maximum length of an object path: 1024 characters
+
+ * Maximum size of a binary object: no absolute limit. Files > 5GB are
+ partitioned into separate files in the native filesystem, and merged during
+ retrieval. <Warning:> the partitioned/large file support is the
+ most complex part of the Hadoop/Swift FS integration, and, along with
+ authentication, the most troublesome to support.
+
+** Testing the hadoop-openstack module
+
+ The <<<hadoop-openstack>>> can be remotely tested against any public
+ or private cloud infrastructure which supports the OpenStack Keystone
+ authentication mechanism. It can also be tested against private
+ OpenStack clusters. OpenStack Development teams are strongly encouraged to test
+ the Hadoop swift filesystem client against any version of Swift that they
+ are developing or deploying, to stress their cluster and to identify
+ bugs early.
+
+ The module comes with a large suite of JUnit tests -tests that are
+ only executed if the source tree includes credentials to test against a
+ specific cluster.
+
+ After checking out the Hadoop source tree, create the file:
+
++--
+ hadoop-tools/hadoop-openstack/src/test/resources/auth-keys.xml
++--
+
+ Into this file, insert the credentials needed to bond to the test filesystem,
+ as decribed above.
+
+ Next set the property <<<test.fs.swift.name>>> to the URL of a
+ swift container to test against. The tests expect exclusive access
+ to this container -do not keep any other data on it, or expect it
+ to be preserved.
+
++--
+ <property>
+ <name>test.fs.swift.name</name>
+ <value>swift://test.myswift/</value>
+ </property>
++--
+
+ In the base hadoop directory, run:
+
++--
+ mvn clean install -DskipTests
++--
+
+ This builds a set of Hadoop JARs consistent with the <<<hadoop-openstack>>>
+ module that is about to be tested.
+
+ In the <<<hadoop-tools/hadoop-openstack>>> directory run
+
++--
+ mvn test -Dtest=TestSwiftRestClient
++--
+
+ This runs some simple tests which include authenticating
+ against the remote swift service. If these tests fail, so will all
+ the rest. If it does fail: check your authentication.
+
+ Once this test succeeds, you can run the full test suite
+
++--
+ mvn test
++--
+
+ Be advised that these tests can take an hour or more, especially against a
+ remote Swift service -or one that throttles bulk operations.
+
+ Once the <<<auth-keys.xml>>> file is in place, the <<<mvn test>>> runs from
+ the Hadoop source base directory will automatically run these OpenStack tests
+ While this ensures that no regressions have occurred, it can also add significant
+ time to test runs, and may run up bills, depending on who is providing\
+ the Swift storage service. We recommend having a separate source tree
+ set up purely for the Swift tests, and running it manually or by the CI tooling
+ at a lower frequency than normal test runs.
+
+ Finally: Apache Hadoop is an open source project. Contributions of code
+ -including more tests- are very welcome.
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/site/site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/site/site.xml?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/site/site.xml (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/site/site.xml Fri Sep 27 11:12:42 2013
@@ -0,0 +1,46 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project name="Apache Hadoop ${project.version}">
+
+ <skin>
+ <groupId>org.apache.maven.skins</groupId>
+ <artifactId>maven-stylus-skin</artifactId>
+ <version>1.2</version>
+ </skin>
+
+ <body>
+ <links>
+ <item name="Apache Hadoop" href="http://hadoop.apache.org/"/>
+ </links>
+ </body>
+
+</project>
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/AcceptAllFilter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/AcceptAllFilter.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/AcceptAllFilter.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/AcceptAllFilter.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+
+/**
+ * A path filter that accepts everything
+ */
+public class AcceptAllFilter implements PathFilter {
+ @Override
+ public boolean accept(Path file) {
+ return true;
+ }
+}