You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2012/08/19 07:46:58 UTC
svn commit: r1374696 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/fs/ftp/
src/main/java/org/apache/hadoop/fs/local/
src/main/java/org/apache/hadoop/fs/viewfs/ src/...
Author: szetszwo
Date: Sun Aug 19 05:46:57 2012
New Revision: 1374696
URL: http://svn.apache.org/viewvc?rev=1374696&view=rev
Log:
HADOOP-8240. Add a new API to allow users to specify a checksum type on FileSystem.create(..). Contributed by Kihwal Lee
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsOptions.java
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374696&r1=1374695&r2=1374696&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Sun Aug 19 05:46:57 2012
@@ -222,6 +222,9 @@ Branch-2 ( Unreleased changes )
HADOOP-7754. Expose file descriptors from Hadoop-wrapped local
FileSystems (todd and ahmed via tucu)
+ HADOOP-8240. Add a new API to allow users to specify a checksum type
+ on FileSystem.create(..). (Kihwal Lee via szetszwo)
+
IMPROVEMENTS
HADOOP-8340. SNAPSHOT build versions should compare as less than their eventual
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1374696&r1=1374695&r2=1374696&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java Sun Aug 19 05:46:57 2012
@@ -39,6 +39,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -46,6 +47,7 @@ import org.apache.hadoop.fs.InvalidPathE
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Progressable;
/**
@@ -467,6 +469,7 @@ public abstract class AbstractFileSystem
short replication = -1;
long blockSize = -1;
int bytesPerChecksum = -1;
+ ChecksumOpt checksumOpt = null;
FsPermission permission = null;
Progressable progress = null;
Boolean createParent = null;
@@ -496,6 +499,12 @@ public abstract class AbstractFileSystem
"BytesPerChecksum option is set multiple times");
}
bytesPerChecksum = ((CreateOpts.BytesPerChecksum) iOpt).getValue();
+ } else if (CreateOpts.ChecksumParam.class.isInstance(iOpt)) {
+ if (checksumOpt != null) {
+ throw new HadoopIllegalArgumentException(
+ "CreateChecksumType option is set multiple times");
+ }
+ checksumOpt = ((CreateOpts.ChecksumParam) iOpt).getValue();
} else if (CreateOpts.Perms.class.isInstance(iOpt)) {
if (permission != null) {
throw new HadoopIllegalArgumentException(
@@ -533,9 +542,16 @@ public abstract class AbstractFileSystem
if (blockSize == -1) {
blockSize = ssDef.getBlockSize();
}
- if (bytesPerChecksum == -1) {
- bytesPerChecksum = ssDef.getBytesPerChecksum();
- }
+
+ // Create a checksum option honoring user input as much as possible.
+ // If bytesPerChecksum is specified, it will override the one set in
+ // checksumOpt. Any missing value will be filled in using the default.
+ ChecksumOpt defaultOpt = new ChecksumOpt(
+ ssDef.getChecksumType(),
+ ssDef.getBytesPerChecksum());
+ checksumOpt = ChecksumOpt.processChecksumOpt(defaultOpt,
+ checksumOpt, bytesPerChecksum);
+
if (bufferSize == -1) {
bufferSize = ssDef.getFileBufferSize();
}
@@ -552,7 +568,7 @@ public abstract class AbstractFileSystem
}
return this.createInternal(f, createFlag, permission, bufferSize,
- replication, blockSize, progress, bytesPerChecksum, createParent);
+ replication, blockSize, progress, checksumOpt, createParent);
}
/**
@@ -563,7 +579,7 @@ public abstract class AbstractFileSystem
public abstract FSDataOutputStream createInternal(Path f,
EnumSet<CreateFlag> flag, FsPermission absolutePermission,
int bufferSize, short replication, long blockSize, Progressable progress,
- int bytesPerChecksum, boolean createParent)
+ ChecksumOpt checksumOpt, boolean createParent)
throws AccessControlException, FileAlreadyExistsException,
FileNotFoundException, ParentNotDirectoryException,
UnsupportedFileSystemException, UnresolvedLinkException, IOException;
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java?rev=1374696&r1=1374695&r2=1374696&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java Sun Aug 19 05:46:57 2012
@@ -28,6 +28,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.PureJavaCrc32;
@@ -324,13 +325,17 @@ public abstract class ChecksumFs extends
final EnumSet<CreateFlag> createFlag,
final FsPermission absolutePermission, final int bufferSize,
final short replication, final long blockSize,
- final Progressable progress, final int bytesPerChecksum,
+ final Progressable progress, final ChecksumOpt checksumOpt,
final boolean createParent) throws IOException {
super(new PureJavaCrc32(), fs.getBytesPerSum(), 4);
+ // checksumOpt is passed down to the raw fs. Unless it implements
+ // checksum impelemts internally, checksumOpt will be ignored.
+ // If the raw fs does checksum internally, we will end up with
+ // two layers of checksumming. i.e. checksumming checksum file.
this.datas = fs.getRawFs().createInternal(file, createFlag,
absolutePermission, bufferSize, replication, blockSize, progress,
- bytesPerChecksum, createParent);
+ checksumOpt, createParent);
// Now create the chekcsumfile; adjust the buffsize
int bytesPerSum = fs.getBytesPerSum();
@@ -338,7 +343,7 @@ public abstract class ChecksumFs extends
this.sums = fs.getRawFs().createInternal(fs.getChecksumFile(file),
EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
absolutePermission, sumBufferSize, replication, blockSize, progress,
- bytesPerChecksum, createParent);
+ checksumOpt, createParent);
sums.write(CHECKSUM_VERSION, 0, CHECKSUM_VERSION.length);
sums.writeInt(bytesPerSum);
}
@@ -361,12 +366,11 @@ public abstract class ChecksumFs extends
public FSDataOutputStream createInternal(Path f,
EnumSet<CreateFlag> createFlag, FsPermission absolutePermission,
int bufferSize, short replication, long blockSize, Progressable progress,
- int bytesPerChecksum, boolean createParent) throws IOException {
-
+ ChecksumOpt checksumOpt, boolean createParent) throws IOException {
final FSDataOutputStream out = new FSDataOutputStream(
new ChecksumFSOutputSummer(this, f, createFlag, absolutePermission,
bufferSize, replication, blockSize, progress,
- bytesPerChecksum, createParent), null);
+ checksumOpt, createParent), null);
return out;
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java?rev=1374696&r1=1374695&r2=1374696&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java Sun Aug 19 05:46:57 2012
@@ -28,6 +28,7 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.token.Token;
@@ -62,7 +63,7 @@ public abstract class DelegateToFileSyst
public FSDataOutputStream createInternal (Path f,
EnumSet<CreateFlag> flag, FsPermission absolutePermission, int bufferSize,
short replication, long blockSize, Progressable progress,
- int bytesPerChecksum, boolean createParent) throws IOException {
+ ChecksumOpt checksumOpt, boolean createParent) throws IOException {
checkPath(f);
// Default impl assumes that permissions do not matter
@@ -81,8 +82,8 @@ public abstract class DelegateToFileSyst
}
// parent does exist - go ahead with create of file.
}
- return fsImpl.primitiveCreate(f, absolutePermission, flag,
- bufferSize, replication, blockSize, progress, bytesPerChecksum);
+ return fsImpl.primitiveCreate(f, absolutePermission, flag,
+ bufferSize, replication, blockSize, progress, checksumOpt);
}
@Override
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1374696&r1=1374695&r2=1374696&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java Sun Aug 19 05:46:57 2012
@@ -127,7 +127,8 @@ import org.apache.hadoop.util.ShutdownHo
* <li> replication factor
* <li> block size
* <li> buffer size
- * <li> bytesPerChecksum (if used).
+ * <li> encryptDataTransfer
+ * <li> checksum option. (checksumType and bytesPerChecksum)
* </ul>
*
* <p>
@@ -613,7 +614,8 @@ public final class FileContext {
* <li>BufferSize - buffersize used in FSDataOutputStream
* <li>Blocksize - block size for file blocks
* <li>ReplicationFactor - replication for blocks
- * <li>BytesPerChecksum - bytes per checksum
+ * <li>ChecksumParam - Checksum parameters. server default is used
+ * if not specified.
* </ul>
* </ul>
*
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1374696&r1=1374695&r2=1374696&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Sun Aug 19 05:46:57 2012
@@ -45,6 +45,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.MultipleIOException;
@@ -54,6 +55,7 @@ import org.apache.hadoop.security.Creden
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ShutdownHookManager;
@@ -656,14 +658,17 @@ public abstract class FileSystem extends
@Deprecated
public FsServerDefaults getServerDefaults() throws IOException {
Configuration conf = getConf();
+ // CRC32 is chosen as default as it is available in all
+ // releases that support checksum.
+ // The client trash configuration is ignored.
return new FsServerDefaults(getDefaultBlockSize(),
conf.getInt("io.bytes.per.checksum", 512),
64 * 1024,
getDefaultReplication(),
conf.getInt("io.file.buffer.size", 4096),
false,
- // NB: ignoring the client trash configuration
- CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT);
+ CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT,
+ DataChecksum.Type.CRC32);
}
/**
@@ -889,11 +894,40 @@ public abstract class FileSystem extends
short replication,
long blockSize,
Progressable progress) throws IOException {
- // only DFS support this
- return create(f, permission, flags.contains(CreateFlag.OVERWRITE), bufferSize, replication, blockSize, progress);
+ return create(f, permission, flags, bufferSize, replication,
+ blockSize, progress, null);
}
-
+ /**
+ * Create an FSDataOutputStream at the indicated Path with a custom
+ * checksum option
+ * @param f the file name to open
+ * @param permission
+ * @param flags {@link CreateFlag}s to use for this stream.
+ * @param bufferSize the size of the buffer to be used.
+ * @param replication required block replication for the file.
+ * @param blockSize
+ * @param progress
+ * @param checksumOpt checksum parameter. If null, the values
+ * found in conf will be used.
+ * @throws IOException
+ * @see #setPermission(Path, FsPermission)
+ */
+ public FSDataOutputStream create(Path f,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress,
+ ChecksumOpt checksumOpt) throws IOException {
+ // Checksum options are ignored by default. The file systems that
+ // implement checksum need to override this method. The full
+ // support is currently only available in DFS.
+ return create(f, permission, flags.contains(CreateFlag.OVERWRITE),
+ bufferSize, replication, blockSize, progress);
+ }
+
/*.
* This create has been added to support the FileContext that processes
* the permission
@@ -905,7 +939,7 @@ public abstract class FileSystem extends
protected FSDataOutputStream primitiveCreate(Path f,
FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize,
short replication, long blockSize, Progressable progress,
- int bytesPerChecksum) throws IOException {
+ ChecksumOpt checksumOpt) throws IOException {
boolean pathExists = exists(f);
CreateFlag.validate(f, pathExists, flag);
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=1374696&r1=1374695&r2=1374696&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java Sun Aug 19 05:46:57 2012
@@ -27,6 +27,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.util.Progressable;
/****************************************************************
@@ -410,10 +411,11 @@ public class FilterFileSystem extends Fi
@Override
protected FSDataOutputStream primitiveCreate(Path f,
FsPermission absolutePermission, EnumSet<CreateFlag> flag,
- int bufferSize, short replication, long blockSize, Progressable progress, int bytesPerChecksum)
+ int bufferSize, short replication, long blockSize,
+ Progressable progress, ChecksumOpt checksumOpt)
throws IOException {
return fs.primitiveCreate(f, absolutePermission, flag,
- bufferSize, replication, blockSize, progress, bytesPerChecksum);
+ bufferSize, replication, blockSize, progress, checksumOpt);
}
@Override
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java?rev=1374696&r1=1374695&r2=1374696&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java Sun Aug 19 05:46:57 2012
@@ -27,6 +27,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileSystem.Statistics;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Progressable;
@@ -81,11 +82,11 @@ public abstract class FilterFs extends A
public FSDataOutputStream createInternal(Path f,
EnumSet<CreateFlag> flag, FsPermission absolutePermission, int bufferSize,
short replication, long blockSize, Progressable progress,
- int bytesPerChecksum, boolean createParent)
+ ChecksumOpt checksumOpt, boolean createParent)
throws IOException, UnresolvedLinkException {
checkPath(f);
return myFs.createInternal(f, flag, absolutePermission, bufferSize,
- replication, blockSize, progress, bytesPerChecksum, createParent);
+ replication, blockSize, progress, checksumOpt, createParent);
}
@Override
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java?rev=1374696&r1=1374695&r2=1374696&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java Sun Aug 19 05:46:57 2012
@@ -26,6 +26,8 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.util.DataChecksum;
/****************************************************
* Provides server default configuration values to clients.
@@ -50,13 +52,15 @@ public class FsServerDefaults implements
private int fileBufferSize;
private boolean encryptDataTransfer;
private long trashInterval;
+ private DataChecksum.Type checksumType;
public FsServerDefaults() {
}
public FsServerDefaults(long blockSize, int bytesPerChecksum,
int writePacketSize, short replication, int fileBufferSize,
- boolean encryptDataTransfer, long trashInterval) {
+ boolean encryptDataTransfer, long trashInterval,
+ DataChecksum.Type checksumType) {
this.blockSize = blockSize;
this.bytesPerChecksum = bytesPerChecksum;
this.writePacketSize = writePacketSize;
@@ -64,6 +68,7 @@ public class FsServerDefaults implements
this.fileBufferSize = fileBufferSize;
this.encryptDataTransfer = encryptDataTransfer;
this.trashInterval = trashInterval;
+ this.checksumType = checksumType;
}
public long getBlockSize() {
@@ -94,6 +99,10 @@ public class FsServerDefaults implements
return trashInterval;
}
+ public DataChecksum.Type getChecksumType() {
+ return checksumType;
+ }
+
// /////////////////////////////////////////
// Writable
// /////////////////////////////////////////
@@ -104,6 +113,7 @@ public class FsServerDefaults implements
out.writeInt(writePacketSize);
out.writeShort(replication);
out.writeInt(fileBufferSize);
+ WritableUtils.writeEnum(out, checksumType);
}
@InterfaceAudience.Private
@@ -113,5 +123,6 @@ public class FsServerDefaults implements
writePacketSize = in.readInt();
replication = in.readShort();
fileBufferSize = in.readInt();
+ checksumType = WritableUtils.readEnum(in, DataChecksum.Type.class);
}
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java?rev=1374696&r1=1374695&r2=1374696&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java Sun Aug 19 05:46:57 2012
@@ -20,7 +20,9 @@ package org.apache.hadoop.fs;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.HadoopIllegalArgumentException;
/**
* This class contains options related to file system operations.
@@ -46,6 +48,10 @@ public final class Options {
public static BytesPerChecksum bytesPerChecksum(short crc) {
return new BytesPerChecksum(crc);
}
+ public static ChecksumParam checksumParam(
+ ChecksumOpt csumOpt) {
+ return new ChecksumParam(csumOpt);
+ }
public static Perms perms(FsPermission perm) {
return new Perms(perm);
}
@@ -91,7 +97,8 @@ public final class Options {
}
public int getValue() { return bufferSize; }
}
-
+
+ /** This is not needed if ChecksumParam is specified. **/
public static class BytesPerChecksum extends CreateOpts {
private final int bytesPerChecksum;
protected BytesPerChecksum(short bpc) {
@@ -103,6 +110,14 @@ public final class Options {
}
public int getValue() { return bytesPerChecksum; }
}
+
+ public static class ChecksumParam extends CreateOpts {
+ private final ChecksumOpt checksumOpt;
+ protected ChecksumParam(ChecksumOpt csumOpt) {
+ checksumOpt = csumOpt;
+ }
+ public ChecksumOpt getValue() { return checksumOpt; }
+ }
public static class Perms extends CreateOpts {
private final FsPermission permissions;
@@ -206,4 +221,116 @@ public final class Options {
return code;
}
}
+
+ /**
+ * This is used in FileSystem and FileContext to specify checksum options.
+ */
+ public static class ChecksumOpt {
+ private final int crcBlockSize;
+ private final DataChecksum.Type crcType;
+
+ /**
+ * Create a uninitialized one
+ */
+ public ChecksumOpt() {
+ crcBlockSize = -1;
+ crcType = DataChecksum.Type.DEFAULT;
+ }
+
+ /**
+ * Normal ctor
+ * @param type checksum type
+ * @param size bytes per checksum
+ */
+ public ChecksumOpt(DataChecksum.Type type, int size) {
+ crcBlockSize = size;
+ crcType = type;
+ }
+
+ public int getBytesPerChecksum() {
+ return crcBlockSize;
+ }
+
+ public DataChecksum.Type getChecksumType() {
+ return crcType;
+ }
+
+ /**
+ * Create a ChecksumOpts that disables checksum
+ */
+ public static ChecksumOpt createDisabled() {
+ return new ChecksumOpt(DataChecksum.Type.NULL, -1);
+ }
+
+ /**
+ * A helper method for processing user input and default value to
+ * create a combined checksum option. This is a bit complicated because
+ * bytesPerChecksum is kept for backward compatibility.
+ *
+ * @param defaultOpt Default checksum option
+ * @param userOpt User-specified checksum option. Ignored if null.
+ * @param userBytesPerChecksum User-specified bytesPerChecksum
+ * Ignored if < 0.
+ */
+ public static ChecksumOpt processChecksumOpt(ChecksumOpt defaultOpt,
+ ChecksumOpt userOpt, int userBytesPerChecksum) {
+ // The following is done to avoid unnecessary creation of new objects.
+ // tri-state variable: 0 default, 1 userBytesPerChecksum, 2 userOpt
+ short whichSize;
+ // true default, false userOpt
+ boolean useDefaultType;
+
+ // bytesPerChecksum - order of preference
+ // user specified value in bytesPerChecksum
+ // user specified value in checksumOpt
+ // default.
+ if (userBytesPerChecksum > 0) {
+ whichSize = 1; // userBytesPerChecksum
+ } else if (userOpt != null && userOpt.getBytesPerChecksum() > 0) {
+ whichSize = 2; // userOpt
+ } else {
+ whichSize = 0; // default
+ }
+
+ // checksum type - order of preference
+ // user specified value in checksumOpt
+ // default.
+ if (userOpt != null &&
+ userOpt.getChecksumType() != DataChecksum.Type.DEFAULT) {
+ useDefaultType = false;
+ } else {
+ useDefaultType = true;
+ }
+
+ // Short out the common and easy cases
+ if (whichSize == 0 && useDefaultType) {
+ return defaultOpt;
+ } else if (whichSize == 2 && !useDefaultType) {
+ return userOpt;
+ }
+
+ // Take care of the rest of combinations
+ DataChecksum.Type type = useDefaultType ? defaultOpt.getChecksumType() :
+ userOpt.getChecksumType();
+ if (whichSize == 0) {
+ return new ChecksumOpt(type, defaultOpt.getBytesPerChecksum());
+ } else if (whichSize == 1) {
+ return new ChecksumOpt(type, userBytesPerChecksum);
+ } else {
+ return new ChecksumOpt(type, userOpt.getBytesPerChecksum());
+ }
+ }
+
+ /**
+ * A helper method for processing user input and default value to
+ * create a combined checksum option.
+ *
+ * @param defaultOpt Default checksum option
+ * @param userOpt User-specified checksum option
+ */
+ public static ChecksumOpt processChecksumOpt(ChecksumOpt defaultOpt,
+ ChecksumOpt userOpt) {
+ return processChecksumOpt(defaultOpt, userOpt, -1);
+ }
+ }
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java?rev=1374696&r1=1374695&r2=1374696&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java Sun Aug 19 05:46:57 2012
@@ -23,10 +23,16 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FsServerDefaults;
+import org.apache.hadoop.util.DataChecksum;
/**
* This class contains constants for configuration keys used
* in the ftp file system.
+ *
+ * Note that the settings for unimplemented features are ignored.
+ * E.g. checksum related settings are just place holders. Even when
+ * wrapped with {@link ChecksumFileSystem}, these settings are not
+ * used.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -46,6 +52,8 @@ public class FtpConfigKeys extends Commo
public static final int CLIENT_WRITE_PACKET_SIZE_DEFAULT = 64*1024;
public static final boolean ENCRYPT_DATA_TRANSFER_DEFAULT = false;
public static final long FS_TRASH_INTERVAL_DEFAULT = 0;
+ public static final DataChecksum.Type CHECKSUM_TYPE_DEFAULT =
+ DataChecksum.Type.CRC32;
protected static FsServerDefaults getServerDefaults() throws IOException {
return new FsServerDefaults(
@@ -55,7 +63,8 @@ public class FtpConfigKeys extends Commo
REPLICATION_DEFAULT,
STREAM_BUFFER_SIZE_DEFAULT,
ENCRYPT_DATA_TRANSFER_DEFAULT,
- FS_TRASH_INTERVAL_DEFAULT);
+ FS_TRASH_INTERVAL_DEFAULT,
+ CHECKSUM_TYPE_DEFAULT);
}
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java?rev=1374696&r1=1374695&r2=1374696&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java Sun Aug 19 05:46:57 2012
@@ -24,11 +24,18 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FsServerDefaults;
+import org.apache.hadoop.util.DataChecksum;
/**
* This class contains constants for configuration keys used
* in the local file system, raw local fs and checksum fs.
+ *
+ * Note that the settings for unimplemented features are ignored.
+ * E.g. checksum related settings are just place holders. Even when
+ * wrapped with {@link ChecksumFileSystem}, these settings are not
+ * used.
*/
+
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class LocalConfigKeys extends CommonConfigurationKeys {
@@ -45,7 +52,8 @@ public class LocalConfigKeys extends Com
public static final int CLIENT_WRITE_PACKET_SIZE_DEFAULT = 64*1024;
public static final boolean ENCRYPT_DATA_TRANSFER_DEFAULT = false;
public static final long FS_TRASH_INTERVAL_DEFAULT = 0;
-
+ public static final DataChecksum.Type CHECKSUM_TYPE_DEFAULT =
+ DataChecksum.Type.CRC32;
public static FsServerDefaults getServerDefaults() throws IOException {
return new FsServerDefaults(
BLOCK_SIZE_DEFAULT,
@@ -54,7 +62,8 @@ public class LocalConfigKeys extends Com
REPLICATION_DEFAULT,
STREAM_BUFFER_SIZE_DEFAULT,
ENCRYPT_DATA_TRANSFER_DEFAULT,
- FS_TRASH_INTERVAL_DEFAULT);
+ FS_TRASH_INTERVAL_DEFAULT,
+ CHECKSUM_TYPE_DEFAULT);
}
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java?rev=1374696&r1=1374695&r2=1374696&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java Sun Aug 19 05:46:57 2012
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.FileChecksum
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.FsStatus;
+import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -159,11 +160,11 @@ class ChRootedFs extends AbstractFileSys
public FSDataOutputStream createInternal(final Path f,
final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
final int bufferSize, final short replication, final long blockSize,
- final Progressable progress, final int bytesPerChecksum,
+ final Progressable progress, final ChecksumOpt checksumOpt,
final boolean createParent) throws IOException, UnresolvedLinkException {
return myFs.createInternal(fullPath(f), flag,
absolutePermission, bufferSize,
- replication, blockSize, progress, bytesPerChecksum, createParent);
+ replication, blockSize, progress, checksumOpt, createParent);
}
@Override
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java?rev=1374696&r1=1374695&r2=1374696&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java Sun Aug 19 05:46:57 2012
@@ -42,6 +42,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.FsStatus;
+import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
@@ -265,7 +266,7 @@ public class ViewFs extends AbstractFile
public FSDataOutputStream createInternal(final Path f,
final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
final int bufferSize, final short replication, final long blockSize,
- final Progressable progress, final int bytesPerChecksum,
+ final Progressable progress, final ChecksumOpt checksumOpt,
final boolean createParent) throws AccessControlException,
FileAlreadyExistsException, FileNotFoundException,
ParentNotDirectoryException, UnsupportedFileSystemException,
@@ -283,7 +284,7 @@ public class ViewFs extends AbstractFile
assert(res.remainingPath != null);
return res.targetFileSystem.createInternal(res.remainingPath, flag,
absolutePermission, bufferSize, replication,
- blockSize, progress, bytesPerChecksum,
+ blockSize, progress, checksumOpt,
createParent);
}
@@ -632,7 +633,7 @@ public class ViewFs extends AbstractFile
public FSDataOutputStream createInternal(final Path f,
final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
final int bufferSize, final short replication, final long blockSize,
- final Progressable progress, final int bytesPerChecksum,
+ final Progressable progress, final ChecksumOpt checksumOpt,
final boolean createParent) throws AccessControlException,
FileAlreadyExistsException, FileNotFoundException,
ParentNotDirectoryException, UnsupportedFileSystemException,
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java?rev=1374696&r1=1374695&r2=1374696&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java Sun Aug 19 05:46:57 2012
@@ -43,12 +43,14 @@ public class DataChecksum implements Che
public static final int CHECKSUM_NULL = 0;
public static final int CHECKSUM_CRC32 = 1;
public static final int CHECKSUM_CRC32C = 2;
+ public static final int CHECKSUM_DEFAULT = 3;
/** The checksum types */
public static enum Type {
NULL (CHECKSUM_NULL, 0),
CRC32 (CHECKSUM_CRC32, 4),
- CRC32C(CHECKSUM_CRC32C, 4);
+ CRC32C(CHECKSUM_CRC32C, 4),
+ DEFAULT(CHECKSUM_DEFAULT, 0); // This cannot be used to create DataChecksum
public final int id;
public final int size;
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java?rev=1374696&r1=1374695&r2=1374696&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java Sun Aug 19 05:46:57 2012
@@ -24,6 +24,7 @@ import java.net.URISyntaxException;
import java.util.EnumSet;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.util.Progressable;
import org.junit.Test;
@@ -76,7 +77,7 @@ public class TestAfsCheckPath {
@Override
public FSDataOutputStream createInternal(Path f, EnumSet<CreateFlag> flag,
FsPermission absolutePermission, int bufferSize, short replication,
- long blockSize, Progressable progress, int bytesPerChecksum,
+ long blockSize, Progressable progress, ChecksumOpt checksumOpt,
boolean createParent) throws IOException {
// deliberately empty
return null;
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java?rev=1374696&r1=1374695&r2=1374696&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java Sun Aug 19 05:46:57 2012
@@ -32,6 +32,7 @@ import java.util.Iterator;
import org.apache.commons.logging.Log;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.security.Credentials;
@@ -80,6 +81,11 @@ public class TestFilterFileSystem {
Progressable progress) throws IOException {
return null;
}
+ public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
+ EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
+ Progressable progress, ChecksumOpt checksumOpt) throws IOException {
+ return null;
+ }
public boolean mkdirs(Path f) { return false; }
public FSDataInputStream open(Path f) { return null; }
public FSDataOutputStream create(Path f) { return null; }
@@ -138,6 +144,16 @@ public class TestFilterFileSystem {
Progressable progress) throws IOException {
return null;
}
+ public FSDataOutputStream create(Path f,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress,
+ ChecksumOpt checksumOpt) throws IOException {
+ return null;
+ }
public String getName() { return null; }
public boolean delete(Path f) { return false; }
public short getReplication(Path src) { return 0 ; }
Added: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsOptions.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsOptions.java?rev=1374696&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsOptions.java (added)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsOptions.java Sun Aug 19 05:46:57 2012
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Options.ChecksumOpt;
+import org.apache.hadoop.util.DataChecksum;
+
+import org.junit.Test;
+
+public class TestFsOptions {
+
+ @Test
+ public void testProcessChecksumOpt() {
+ ChecksumOpt defaultOpt = new ChecksumOpt(DataChecksum.Type.CRC32, 512);
+ ChecksumOpt finalOpt;
+
+ // Give a null
+ finalOpt = ChecksumOpt.processChecksumOpt(defaultOpt, null);
+ checkParams(defaultOpt, finalOpt);
+
+ // null with bpc
+ finalOpt = ChecksumOpt.processChecksumOpt(defaultOpt, null, 1024);
+ checkParams(DataChecksum.Type.CRC32, 1024, finalOpt);
+
+ ChecksumOpt myOpt = new ChecksumOpt();
+
+ // custom with unspecified parameters
+ finalOpt = ChecksumOpt.processChecksumOpt(defaultOpt, myOpt);
+ checkParams(defaultOpt, finalOpt);
+
+ myOpt = new ChecksumOpt(DataChecksum.Type.CRC32C, 2048);
+
+ // custom config
+ finalOpt = ChecksumOpt.processChecksumOpt(defaultOpt, myOpt);
+ checkParams(DataChecksum.Type.CRC32C, 2048, finalOpt);
+
+ // custom config + bpc
+ finalOpt = ChecksumOpt.processChecksumOpt(defaultOpt, myOpt, 4096);
+ checkParams(DataChecksum.Type.CRC32C, 4096, finalOpt);
+ }
+
+ private void checkParams(ChecksumOpt expected, ChecksumOpt obtained) {
+ assertEquals(expected.getChecksumType(), obtained.getChecksumType());
+ assertEquals(expected.getBytesPerChecksum(), obtained.getBytesPerChecksum());
+ }
+
+ private void checkParams(DataChecksum.Type type, int bpc, ChecksumOpt obtained) {
+ assertEquals(type, obtained.getChecksumType());
+ assertEquals(bpc, obtained.getBytesPerChecksum());
+ }
+}