You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/04/24 20:13:59 UTC
[44/83] [abbrv] [partial] hadoop git commit: HDFS-13405. Ozone:
Rename HDSL to HDDS. Contributed by Ajay Kumar, Elek Marton, Mukul Kumar Singh,
Shashikant Banerjee and Anu Engineer.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
new file mode 100644
index 0000000..8018d29
--- /dev/null
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -0,0 +1,1031 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Do not modify this file directly. Instead, copy entries that you -->
+<!-- wish to modify from this file into ozone-site.xml and change them -->
+<!-- there. If ozone-site.xml does not already exist, create it. -->
+
+<!--Tags supported are OZONE, CBLOCK, MANAGEMENT, SECURITY, PERFORMANCE, -->
+<!--DEBUG, CLIENT, SERVER, KSM, SCM, CRITICAL, RATIS, CONTAINER, REQUIRED, -->
+<!--REST, STORAGE, PIPELINE, STANDALONE -->
+
+<configuration>
+
+ <!--Container Settings used by Datanode-->
+ <property>
+ <name>ozone.container.cache.size</name>
+ <value>1024</value>
+ <tag>PERFORMANCE, CONTAINER, STORAGE</tag>
+ <description>The open container is cached on the data node side. We maintain
+ an LRU
+ cache for caching the recently used containers. This setting controls the
+ size of that cache.
+ </description>
+ </property>
+ <property>
+ <name>dfs.container.ipc</name>
+ <value>9859</value>
+ <tag>OZONE, CONTAINER, MANAGEMENT</tag>
+ <description>The ipc port number of container.</description>
+ </property>
+ <property>
+ <name>dfs.container.ipc.random.port</name>
+ <value>false</value>
+ <tag>OZONE, DEBUG, CONTAINER</tag>
+ <description>Allocates a random free port for ozone container. This is used
+ only while
+ running unit tests.
+ </description>
+ </property>
+ <property>
+ <name>dfs.container.ratis.datanode.storage.dir</name>
+ <value/>
+ <tag>OZONE, CONTAINER, STORAGE, MANAGEMENT, RATIS</tag>
+ <description>This directory is used for storing Ratis metadata like logs. If
+ this is
+ not set then default metadata dirs is used. A warning will be logged if
+ this not set. Ideally, this should be mapped to a fast disk like an SSD.
+ </description>
+ </property>
+ <property>
+ <name>dfs.container.ratis.enabled</name>
+ <value>false</value>
+ <tag>OZONE, MANAGEMENT, PIPELINE, RATIS</tag>
+ <description>Ozone supports different kinds of replication pipelines. Ratis
+ is one of
+ the replication pipeline supported by ozone.
+ </description>
+ </property>
+ <property>
+ <name>dfs.container.ratis.ipc</name>
+ <value>9858</value>
+ <tag>OZONE, CONTAINER, PIPELINE, RATIS, MANAGEMENT</tag>
+ <description>The ipc port number of container.</description>
+ </property>
+ <property>
+ <name>dfs.container.ratis.ipc.random.port</name>
+ <value>false</value>
+ <tag>OZONE,DEBUG</tag>
+ <description>Allocates a random free port for ozone ratis port for the
+ container. This
+ is used only while running unit tests.
+ </description>
+ </property>
+ <property>
+ <name>dfs.container.ratis.rpc.type</name>
+ <value>GRPC</value>
+ <tag>OZONE, RATIS, MANAGEMENT</tag>
+ <description>Ratis supports different kinds of transports like netty, GRPC,
+ Hadoop RPC
+ etc. This picks one of those for this cluster.
+ </description>
+ </property>
+ <property>
+ <name>dfs.container.ratis.num.write.chunk.threads</name>
+ <value>60</value>
+ <tag>OZONE, RATIS, PERFORMANCE</tag>
+ <description>Maximum number of threads in the thread pool that Ratis
+ will use for writing chunks (60 by default).
+ </description>
+ </property>
+ <property>
+ <name>dfs.container.ratis.segment.size</name>
+ <value>1073741824</value>
+ <tag>OZONE, RATIS, PERFORMANCE</tag>
+ <description>The size of the raft segment used by Apache Ratis on datanodes.
+ (1 GB by default)
+ </description>
+ </property>
+ <property>
+ <name>dfs.container.ratis.segment.preallocated.size</name>
+ <value>134217728</value>
+ <tag>OZONE, RATIS, PERFORMANCE</tag>
+ <description>The size of the buffer which is preallocated for raft segment
+ used by Apache Ratis on datanodes.(128 MB by default)
+ </description>
+ </property>
+ <property>
+ <name>ozone.container.report.interval</name>
+ <value>60000ms</value>
+ <tag>OZONE, CONTAINER, MANAGEMENT</tag>
+ <description>Time interval of the datanode to send container report. Each
+ datanode periodically send container report upon receive
+ sendContainerReport from SCM. Unit could be defined with
+ postfix (ns,ms,s,m,h,d)</description>
+ </property>
+ <!--Ozone Settings-->
+ <property>
+ <name>ozone.administrators</name>
+ <value/>
+ <tag>OZONE, SECURITY</tag>
+ <description>Ozone administrator users delimited by the comma.
+ If not set, only the user who launches an ozone service will be the admin
+ user. This property must be set if ozone services are started by different
+ users. Otherwise, the RPC layer will reject calls from other servers which
+ are started by users not in the list.
+ </description>
+ </property>
+ <property>
+ <name>ozone.block.deleting.container.limit.per.interval</name>
+ <value>10</value>
+ <tag>OZONE, PERFORMANCE, SCM</tag>
+ <description>A maximum number of containers to be scanned by block deleting
+ service per
+ time interval. The block deleting service spawns a thread to handle block
+ deletions in a container. This property is used to throttle the number of
+ threads spawned for block deletions.
+ </description>
+ </property>
+ <property>
+ <name>ozone.block.deleting.limit.per.task</name>
+ <value>1000</value>
+ <tag>OZONE, PERFORMANCE, SCM</tag>
+ <description>A maximum number of blocks to be deleted by block deleting
+ service per
+ time interval. This property is used to throttle the actual number of
+ block deletions on a data node per container.
+ </description>
+ </property>
+ <property>
+ <name>ozone.block.deleting.service.interval</name>
+ <value>1m</value>
+ <tag>OZONE, PERFORMANCE, SCM</tag>
+ <description>Time interval of the block deleting service.
+ The block deleting service runs on each datanode periodically and
+ deletes blocks queued for deletion. Unit could be defined with
+ postfix (ns,ms,s,m,h,d)
+ </description>
+ </property>
+ <property>
+ <name>ozone.block.deleting.service.timeout</name>
+ <value>300000ms</value>
+ <tag>OZONE, PERFORMANCE, SCM</tag>
+ <description>A timeout value of block deletion service. If this is set
+ greater than 0,
+ the service will stop waiting for the block deleting completion after this
+ time. If timeout happens to a large proportion of block deletion, this
+ needs to be increased with ozone.block.deleting.limit.per.task. This
+ setting supports multiple time unit suffixes as described in
+ dfs.heartbeat.interval. If no suffix is specified, then milliseconds is
+ assumed.
+ </description>
+ </property>
+ <property>
+ <name>ozone.client.connection.timeout</name>
+ <value>5000ms</value>
+ <tag>OZONE, PERFORMANCE, CLIENT</tag>
+ <description>Connection timeout for Ozone client in milliseconds.
+ </description>
+ </property>
+ <property>
+ <name>ozone.client.protocol</name>
+ <value>org.apache.hadoop.ozone.client.rpc.RpcClient</value>
+ <tag>OZONE, CLIENT, MANAGEMENT</tag>
+ <description>Protocol class to be used by the client to connect to ozone
+ cluster.
+ The build-in implementation includes:
+ org.apache.hadoop.ozone.client.rpc.RpcClient for RPC
+ org.apache.hadoop.ozone.client.rest.RestClient for REST
+ The default is the RpClient. Please do not change this unless you have a
+ very good understanding of what you are doing.
+ </description>
+ </property>
+ <property>
+ <name>ozone.client.socket.timeout</name>
+ <value>5000ms</value>
+ <tag>OZONE, CLIENT</tag>
+ <description>Socket timeout for Ozone client. Unit could be defined with
+ postfix (ns,ms,s,m,h,d)</description>
+ </property>
+ <property>
+ <name>ozone.enabled</name>
+ <value>false</value>
+ <tag>OZONE, REQUIRED</tag>
+ <description>
+ Status of the Ozone Object Storage service is enabled.
+ Set to true to enable Ozone.
+ Set to false to disable Ozone.
+ Unless this value is set to true, Ozone services will not be started in
+ the cluster.
+
+ Please note: By default ozone is disabled on a hadoop cluster.
+ </description>
+ </property>
+ <property>
+ <name>ozone.handler.type</name>
+ <value>distributed</value>
+ <tag>OZONE, REST</tag>
+ <description>
+ Tells ozone which storage handler to use. The possible values are:
+ distributed - The Ozone distributed storage handler, which speaks to
+ KSM/SCM on the backend and provides REST services to clients.
+ local - Local Storage handler strictly for testing - To be removed.
+ </description>
+ </property>
+ <property>
+ <name>ozone.key.deleting.limit.per.task</name>
+ <value>1000</value>
+ <tag>KSM, PERFORMANCE</tag>
+ <description>
+ A maximum number of keys to be scanned by key deleting service
+ per time interval in KSM. Those keys are sent to delete metadata and
+ generate transactions in SCM for next async deletion between SCM
+ and DataNode.
+ </description>
+ </property>
+ <property>
+ <name>ozone.ksm.address</name>
+ <value/>
+ <tag>KSM, REQUIRED</tag>
+ <description>
+ The address of the Ozone KSM service. This allows clients to discover
+ the KSMs address.
+ </description>
+ </property>
+ <property>
+ <name>ozone.ksm.group.rights</name>
+ <value>READ_WRITE</value>
+ <tag>KSM, SECURITY</tag>
+ <description>
+ Default group permissions in Ozone KSM.
+ </description>
+ </property>
+ <property>
+ <name>ozone.ksm.handler.count.key</name>
+ <value>20</value>
+ <tag>KSM, PERFORMANCE</tag>
+ <description>
+ The number of RPC handler threads for KSM service endpoints.
+ </description>
+ </property>
+ <property>
+ <name>ozone.ksm.http-address</name>
+ <value>0.0.0.0:9874</value>
+ <tag>KSM, MANAGEMENT</tag>
+ <description>
+ The address and the base port where the KSM web UI will listen on.
+
+ If the port is 0, then the server will start on a free port. However, it
+ is best to specify a well-known port, so it is easy to connect and see
+ the KSM management UI.
+ </description>
+ </property>
+ <property>
+ <name>ozone.ksm.http-bind-host</name>
+ <value>0.0.0.0</value>
+ <tag>KSM, MANAGEMENT</tag>
+ <description>
+ The actual address the KSM web server will bind to. If this optional
+ the address is set, it overrides only the hostname portion of
+ ozone.ksm.http-address.
+ </description>
+ </property>
+ <property>
+ <name>ozone.ksm.http.enabled</name>
+ <value>true</value>
+ <tag>KSM, MANAGEMENT</tag>
+ <description>
+ Property to enable or disable KSM web user interface.
+ </description>
+ </property>
+ <property>
+ <name>ozone.ksm.https-address</name>
+ <value>0.0.0.0:9875</value>
+ <tag>KSM, MANAGEMENT, SECURITY</tag>
+ <description>
+ The address and the base port where the KSM web UI will listen
+ on using HTTPS.
+ If the port is 0 then the server will start on a free port.
+ </description>
+ </property>
+ <property>
+ <name>ozone.ksm.https-bind-host</name>
+ <value>0.0.0.0</value>
+ <tag>KSM, MANAGEMENT, SECURITY</tag>
+ <description>
+ The actual address the KSM web server will bind to using HTTPS.
+ If this optional address is set, it overrides only the hostname portion of
+ ozone.ksm.http-address.
+ </description>
+ </property>
+ <property>
+ <name>ozone.ksm.keytab.file</name>
+ <value/>
+ <tag>KSM, SECURITY</tag>
+ <description>
+ The keytab file for Kerberos authentication in KSM.
+ </description>
+ </property>
+ <property>
+ <name>ozone.ksm.db.cache.size.mb</name>
+ <value>128</value>
+ <tag>KSM, PERFORMANCE</tag>
+ <description>
+ The size of KSM DB cache in MB that used for caching files.
+ This value is set to an abnormally low value in the default configuration.
+ That is to make unit testing easy. Generally, this value should be set to
+ something like 16GB or more, if you intend to use Ozone at scale.
+
+ A large value for this key allows a proportionally larger amount of KSM
+ metadata to be cached in memory. This makes KSM operations faster.
+ </description>
+ </property>
+ <property>
+ <name>ozone.ksm.user.max.volume</name>
+ <value>1024</value>
+ <tag>KSM, MANAGEMENT</tag>
+ <description>
+ The maximum number of volumes a user can have on a cluster.Increasing or
+ decreasing this number has no real impact on ozone cluster. This is
+ defined only for operational purposes. Only an administrator can create a
+ volume, once a volume is created there are no restrictions on the number
+ of buckets or keys inside each bucket a user can create.
+ </description>
+ </property>
+ <property>
+ <name>ozone.ksm.user.rights</name>
+ <value>READ_WRITE</value>
+ <tag>KSM, SECURITY</tag>
+ <description>
+ Default user permissions used in KSM.
+ </description>
+ </property>
+ <property>
+ <name>ozone.localstorage.root</name>
+ <value>${hadoop.tmp.dir}/ozone</value>
+ <tag>OZONE, DEBUG</tag>
+ <description>
+ This is used only for testing purposes. This value is used by the local
+ storage handler to simulate a REST backend. This is useful only when
+ debugging the REST front end independent of KSM and SCM. To be removed.
+ </description>
+ </property>
+ <property>
+ <name>ozone.metadata.dirs</name>
+ <value/>
+ <tag>OZONE, KSM, SCM, CONTAINER, REQUIRED, STORAGE</tag>
+ <description>
+ Ozone metadata is shared among KSM, which acts as the namespace
+ manager for ozone, SCM which acts as the block manager and data nodes
+ which maintain the name of the key(Key Name and BlockIDs). This
+ replicated and distributed metadata store is maintained under the
+ directory pointed by this key. Since metadata can be I/O intensive, at
+ least on KSM and SCM we recommend having SSDs. If you have the luxury
+ of mapping this path to SSDs on all machines in the cluster, that will
+ be excellent.
+
+ If Ratis metadata directories are not specified, Ratis server will emit a
+ warning and use this path for storing its metadata too.
+ </description>
+ </property>
+ <property>
+ <name>ozone.metastore.impl</name>
+ <value>RocksDB</value>
+ <tag>OZONE, KSM, SCM, CONTAINER, STORAGE</tag>
+ <description>
+ Ozone metadata store implementation. Ozone metadata are well
+ distributed to multiple services such as ksm, scm. They are stored in
+ some local key-value databases. This property determines which database
+ library to use. Supported value is either LevelDB or RocksDB.
+ </description>
+ </property>
+
+ <property>
+ <name>ozone.metastore.rocksdb.statistics</name>
+ <value>ALL</value>
+ <tag>OZONE, KSM, SCM, STORAGE, PERFORMANCE</tag>
+ <description>
+ The statistics level of the rocksdb store. If you use any value from
+ org.rocksdb.StatsLevel (eg. ALL or EXCEPT_DETAILED_TIMERS), the rocksdb
+ statistics will be exposed over JMX bean with the choosed setting. Set
+ it to OFF to not initialize rocksdb statistics at all. Please note that
+ collection of statistics could have 5-10% performance penalty.
+ Check the rocksdb documentation for more details.
+ </description>
+ </property>
+
+ <property>
+ <name>ozone.scm.block.client.address</name>
+ <value/>
+ <tag>OZONE, SCM</tag>
+ <description>The address of the Ozone SCM block client service. If not
+ defined value of ozone.scm.client.address is used.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.block.client.bind.host</name>
+ <value>0.0.0.0</value>
+ <tag>OZONE, SCM</tag>
+ <description>
+ The hostname or IP address used by the SCM block client
+ endpoint to bind.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.block.client.port</name>
+ <value>9863</value>
+ <tag>OZONE, SCM</tag>
+ <description>
+ The port number of the Ozone SCM block client service.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.block.deletion.max.retry</name>
+ <value>4096</value>
+ <tag>OZONE, SCM</tag>
+ <description>
+ SCM wraps up many blocks in a deletion transaction and sends that to data
+ node for physical deletion periodically. This property determines how many
+ times SCM is going to retry sending a deletion operation to the data node.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.block.size.in.mb</name>
+ <value>256</value>
+ <tag>OZONE, SCM</tag>
+ <description>
+ The default size of a scm block in bytes. This is maps to the default
+ Ozone block size.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.chunk.size</name>
+ <value>16777216</value>
+ <tag>OZONE, SCM, CONTAINER, PERFORMANCE</tag>
+ <description>
+ The chunk size for reading/writing chunk operations in bytes.
+
+ The chunk size defaults to 8MB. If the value configured is more than the
+ maximum size (16MB), it will be reset to the maximum size. This maps to
+ the network packet sizes and file write operations in the client to
+ datanode protocol.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.client.address</name>
+ <value/>
+ <tag>OZONE, SCM, REQUIRED</tag>
+ <description>
+ The address of the Ozone SCM client service. This is a required setting.
+
+ It is a string in the host:port format. The port number is optional
+ and defaults to 9860.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.client.bind.host</name>
+ <value>0.0.0.0</value>
+ <tag>OZONE, SCM, MANAGEMENT</tag>
+ <description>The hostname or IP address used by the SCM client endpoint to
+ bind.
+ This setting is used by the SCM only and never used by clients.
+
+ The setting can be useful in multi-homed setups to restrict the
+ availability of the SCM client service to a specific interface.
+
+ The default is appropriate for most clusters.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.client.port</name>
+ <value>9860</value>
+ <tag>OZONE, SCM, MANAGEMENT</tag>
+ <description>The port number of the Ozone SCM client service.</description>
+ </property>
+ <property>
+ <name>ozone.scm.container.deletion-choosing.policy</name>
+ <value>
+ org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy
+ </value>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ The policy used for choosing desire containers for block deletion.
+ Datanode selects some containers to process block deletion
+ in a certain interval defined by ozone.block.deleting.service.interval.
+ The number of containers to process in each interval is defined
+ by ozone.block.deleting.container.limit.per.interval. This property is
+ used to configure the policy applied while selecting containers.
+ There are two policies supporting now:
+ RandomContainerDeletionChoosingPolicy and
+ TopNOrderedContainerDeletionChoosingPolicy.
+ org.apache.hadoop.ozone.container.common.impl.RandomContainerDeletionChoosingPolicy
+ implements a simply random policy that to return a random list of
+ containers.
+ org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy
+ implements a policy that choosing top count number of containers in a
+ pending-deletion-blocks's num
+ based descending order.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.container.placement.impl</name>
+ <value>
+ org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom
+ </value>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>Placement policy class for containers.
+ Defaults to SCMContainerPlacementRandom.class
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.container.provision_batch_size</name>
+ <value>20</value>
+ <tag>OZONE, PERFORMANCE</tag>
+ <description>Pre-provision specified number of containers for block
+ allocation.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.container.report.processing.interval</name>
+ <value>60s</value>
+ <tag>OZONE, PERFORMANCE</tag>
+ <description>Time interval for scm to process container reports
+ for a node pool. Scm handles node pool reports in a cyclic clock
+ manner, it fetches pools periodically with this time interval.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.container.reports.wait.timeout</name>
+ <value>300s</value>
+ <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
+ <description>Maximum time to wait in seconds for processing all container
+ reports from
+ a node pool. It determines the timeout for a
+ node pool report.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.container.size.gb</name>
+ <value>5</value>
+ <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
+ <description>
+ Default container size used by Ozone. This value is specified
+ in GB.
+ There are two considerations while picking this number. The speed at which
+ a container can be replicated, determined by the network speed and the
+ metadata that each container generates. So selecting a large number
+ creates less SCM metadata, but recovery time will be more. 5GB is a number
+ that maps to quick replication times in gigabit networks, but still
+ balances the amount of metadata.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.datanode.address</name>
+ <value/>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ The address of the Ozone SCM service used for internal
+ communication between the DataNodes and the SCM.
+
+ It is a string in the host:port format. The port number is optional
+ and defaults to 9861.
+
+ This setting is optional. If unspecified then the hostname portion
+ is picked from the ozone.scm.client.address setting and the
+ default service port of 9861 is chosen.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.datanode.bind.host</name>
+ <value/>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ The hostname or IP address used by the SCM service endpoint to
+ bind.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.datanode.id</name>
+ <value/>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>The path that datanodes will use to store the datanode ID.
+ If this value is not set, then datanode ID is created under the
+ metadata directory.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.datanode.port</name>
+ <value>9861</value>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ The port number of the Ozone SCM service.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.db.cache.size.mb</name>
+ <value>128</value>
+ <tag>OZONE, PERFORMANCE</tag>
+ <description>SCM keeps track of the Containers in the cluster. This DB holds
+ the container metadata. This value is set to a small value to make the
+ unit
+ testing runs smooth. In production, we recommend a value of 16GB or
+ higher. This allows SCM to avoid disk I/O's while looking up the container
+ location.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.dead.node.interval</name>
+ <value>10m</value>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ The interval between heartbeats before a node is tagged as dead.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.handler.count.key</name>
+ <value>10</value>
+ <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
+ <description>
+ The number of RPC handler threads for each SCM service
+ endpoint.
+
+ The default is appropriate for small clusters (tens of nodes).
+
+ Set a value that is appropriate for the cluster size. Generally, HDFS
+ recommends RPC handler count is set to 20 * log2(Cluster Size) with an
+ upper limit of 200. However, SCM will not have the same amount of
+ traffic as Namenode, so a value much smaller than that will work well too.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.heartbeat.interval</name>
+ <value>30s</value>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ The heartbeat interval from a data node to SCM. Yes,
+ it is not three but 30, since most data nodes will heart beating via Ratis
+ heartbeats. If a client is not able to talk to a data node, it will notify
+ KSM/SCM eventually. So a 30 second HB seems to work. This assumes that
+ replication strategy used is Ratis if not, this value should be set to
+ something smaller like 3 seconds.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.heartbeat.log.warn.interval.count</name>
+ <value>10</value>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ Defines how frequently we will log the missing of a heartbeat to SCM.
+ For example in the default case, we will write a warning message for each
+ ten consecutive heartbeats that we miss to SCM. This helps in reducing
+ clutter in a data node log, but trade off is that logs will have less of
+ this statement.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.heartbeat.rpc-timeout</name>
+ <value>1000</value>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ Timeout value for the RPC from Datanode to SCM in milliseconds.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.heartbeat.thread.interval</name>
+ <value>3s</value>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ When a heartbeat from the data node arrives on SCM, It is queued for
+ processing with the time stamp of when the heartbeat arrived. There is a
+ heartbeat processing thread inside SCM that runs at a specified interval.
+ This value controls how frequently this thread is run.
+
+ There are some assumptions build into SCM such as this value should allow
+ the heartbeat processing thread to run at least three times more
+ frequently than heartbeats and at least five times more than stale node
+ detection time. If you specify a wrong value, SCM will gracefully refuse
+ to run. For more info look at the node manager tests in SCM.
+
+ In short, you don't need to change this.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.http-address</name>
+ <value>0.0.0.0:9876</value>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ The address and the base port where the SCM web ui will listen on.
+
+ If the port is 0 then the server will start on a free port.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.http-bind-host</name>
+ <value>0.0.0.0</value>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ The actual address the SCM web server will bind to. If this
+ optional address is set, it overrides only the hostname portion of
+ ozone.scm.http-address.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.http.enabled</name>
+ <value>true</value>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ Property to enable or disable SCM web ui.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.https-address</name>
+ <value>0.0.0.0:9877</value>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ The address and the base port where the SCM web UI will listen
+ on using HTTPS.
+
+ If the port is 0 then the server will start on a free port.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.https-bind-host</name>
+ <value>0.0.0.0</value>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ The actual address the SCM web server will bind to using HTTPS.
+ If this optional address is set, it overrides only the hostname portion of
+ ozone.scm.http-address.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.keytab.file</name>
+ <value/>
+ <tag>OZONE, SECURITY</tag>
+ <description>
+ The keytab file for Kerberos authentication in SCM.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.max.container.report.threads</name>
+ <value>100</value>
+ <tag>OZONE, PERFORMANCE</tag>
+ <description>
+ Maximum number of threads to process container reports in scm.
+ Each container report from a data node is processed by scm in a worker
+ thread, fetched from a thread pool. This property is used to control the
+ maximum size of the thread pool.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.max.hb.count.to.process</name>
+ <value>5000</value>
+ <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
+ <description>
+ The maximum number of heartbeat to process per loop of the
+ heartbeat process thread. Please see
+ ozone.scm.heartbeat.thread.interval
+ for more info.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.max.nodepool.processing.threads</name>
+ <value>1</value>
+ <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
+ <description>
+ Number of node pools to process in parallel.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.names</name>
+ <value/>
+ <tag>OZONE</tag>
+ <description>
+ The value of this property is a set of DNS | DNS:PORT | IP
+ Address | IP:PORT. Written as a comma separated string. e.g. scm1,
+ scm2:8020, 7.7.7.7:7777.
+ This property allows datanodes to discover where SCM is, so that
+ datanodes can send heartbeat to SCM.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.stale.node.interval</name>
+ <value>90s</value>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ The interval for stale node flagging. Please
+ see ozone.scm.heartbeat.thread.interval before changing this value.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.max.nodepool.processing.threads</name>
+ <value>1</value>
+ <tag>OZONE, SCM</tag>
+ <description>
+ Controls the number of node pools that can be processed in parallel by
+ Container Supervisor.
+ </description>
+ </property>
+ <property>
+ <name>ozone.trace.enabled</name>
+ <value>false</value>
+ <tag>OZONE, DEBUG</tag>
+ <description>
+ Setting this flag to true dumps the HTTP request/ response in
+ the logs. Very useful when debugging REST protocol.
+ </description>
+ </property>
+ <property>
+ <name>ozone.web.authentication.kerberos.principal</name>
+ <value/>
+ <tag>OZONE, SECURITY</tag>
+ <description>
+ The server principal used by the SCM and KSM for web UI SPNEGO
+ authentication when Kerberos security is enabled. This is typically set to
+ HTTP/_HOST@REALM.TLD The SPNEGO server principal begins with the prefix
+ HTTP/ by convention.
+
+ If the value is '*', the web server will attempt to login with
+ every principal specified in the keytab file.
+ </description>
+ </property>
+
+ <!--Client Settings-->
+ <property>
+ <name>scm.container.client.idle.threshold</name>
+ <value>10s</value>
+ <tag>OZONE, PERFORMANCE</tag>
+ <description>
+ In the standalone pipelines, the SCM clients use netty to
+ communicate with the container. It also uses connection pooling to
+ reduce client side overheads. This allows a connection to stay idle for
+ a while before the connection is closed.
+ </description>
+ </property>
+ <property>
+ <name>scm.container.client.max.size</name>
+ <value>256</value>
+ <tag>OZONE, PERFORMANCE</tag>
+ <description>
+ Controls the maximum number of connections that we cached via
+ clientconnection pooling. If the number of connection
+ exceed this count then the oldest idle connection is evicted.
+ </description>
+ </property>
+
+ <property>
+ <name>scm.container.client.max.outstanding.requests</name>
+ <value>100</value>
+ <tag>OZONE, PERFORMANCE</tag>
+ <description>
+ Controls the maximum number of outstanding async requests that can be
+ handled by the Standalone as well as Ratis client.
+ </description>
+ </property>
+
+ <property>
+ <name>ozone.scm.container.creation.lease.timeout</name>
+ <value>60s</value>
+ <tag>OZONE, SCM</tag>
+ <description>
+ Container creation timeout in milliseconds to be used by SCM. When
+ BEGIN_CREATE event happens the container is moved from ALLOCATED to
+ CREATING state, SCM will now wait for the configured amount of time
+ to get COMPLETE_CREATE event if it doesn't receive it will move the
+ container to DELETING.
+ </description>
+ </property>
+
+ <property>
+ <name>ozone.key.preallocation.maxsize</name>
+ <value>134217728</value>
+ <tag>OZONE, KSM, PERFORMANCE</tag>
+ <description>
+ When a new key write request is sent to KSM, if a size is requested, at most
+ 128MB of size is allocated at request time. If client needs more space for the
+ write, separate block allocation requests will be made.
+ </description>
+ </property>
+
+ <property>
+ <name>ozone.client.list.cache</name>
+ <value>1000</value>
+ <tag>OZONE, PERFORMANCE</tag>
+ <description>
+ Configuration property to configure the cache size of client list calls.
+ </description>
+ </property>
+
+ <property>
+ <name>ozone.replication</name>
+ <value>3</value>
+ <tag>OZONE, CLIENT</tag>
+ <description>
+ Default replication value. The actual number of replications can be
+ specified when writing the key. The default is used if replication
+ is not specified. Supported values: 1 and 3.
+ </description>
+ </property>
+
+ <property>
+ <name>ozone.replication.type</name>
+ <value>RATIS</value>
+ <tag>OZONE, CLIENT</tag>
+ <description>
+ Default replication type to be used while writing key into ozone. The
+ value can be specified when writing the key, default is used when
+ nothing is specified. Supported values: RATIS, STAND_ALONE and CHAINED.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.container.close.threshold</name>
+ <value>0.9f</value>
+ <tag>OZONE, SCM</tag>
+ <description>
+ This determines the threshold to be used for closing a container.
+ When the container used percentage reaches this threshold,
+ the container will be closed. Value should be a positive, non-zero
+ percentage in float notation (X.Yf), with 1.0f meaning 100%.
+ </description>
+ </property>
+ <property>
+ <name>ozone.rest.client.http.connection.max</name>
+ <value>100</value>
+ <tag>OZONE, CLIENT</tag>
+ <description>
+ This defines the overall connection limit for the connection pool used in
+ RestClient.
+ </description>
+ </property>
+ <property>
+ <name>ozone.rest.client.http.connection.per-route.max</name>
+ <value>20</value>
+ <tag>OZONE, CLIENT</tag>
+ <description>
+ This defines the connection limit per one HTTP route/host. Total max
+ connection is limited by ozone.rest.client.http.connection.max property.
+ </description>
+ </property>
+
+ <property>
+ <name>ozone.open.key.cleanup.service.interval.seconds</name>
+ <value>86400</value>
+ <tag>OZONE, KSM, PERFORMANCE</tag>
+ <description>
+ A background job periodically checks open key entries and delete the expired ones. This entry controls the
+ interval of this cleanup check.
+ </description>
+ </property>
+
+ <property>
+ <name>ozone.open.key.expire.threshold</name>
+ <value>86400</value>
+ <tag>OZONE, KSM, PERFORMANCE</tag>
+ <description>
+ Controls how long an open key operation is considered active. Specifically, if a key
+ has been open longer than the value of this config entry, that open key is considered as
+ expired (e.g. due to client crash). Default to 24 hours.
+ </description>
+ </property>
+
+
+ <property>
+ <name>hdds.rest.rest-csrf.enabled</name>
+ <value>false</value>
+ <description>
+ If true, then enables Object Store REST server protection against
+ cross-site request forgery (CSRF).
+ </description>
+ </property>
+
+ <property>
+ <name>hdds.rest.http-address</name>
+ <value>0.0.0.0:9880</value>
+ <description>The http address of Object Store REST server inside the
+ datanode.</description>
+ </property>
+
+
+ <property>
+ <name>hdds.rest.netty.high.watermark</name>
+ <value>65535</value>
+ <description>
+ High watermark configuration to Netty for Object Store REST server.
+ </description>
+ </property>
+
+ <property>
+ <name>hdds.rest.netty.low.watermark</name>
+ <value>32768</value>
+ <description>
+ Low watermark configuration to Netty for Object Store REST server.
+ </description>
+ </property>
+
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/TestArchive.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/TestArchive.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/TestArchive.java
new file mode 100644
index 0000000..f53f770
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/TestArchive.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.Random;
+import java.util.zip.Adler32;
+import java.util.zip.Checksum;
+
+/**
+ * Test archive creation and unpacking.
+ */
+public class TestArchive {
+ private static final int DIR_COUNT = 10;
+ private static final int SUB_DIR_COUNT = 3;
+ private static final int FILE_COUNT = 10;
+ private long checksumWrite = 0L;
+ private long checksumRead = 0L;
+ private long tmp = 0L;
+ private Checksum crc = new Adler32();
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
+ @Rule
+ public TemporaryFolder outputFolder = new TemporaryFolder();
+
+
+ @Before
+ public void setUp() throws Exception {
+ Random r = new Random();
+ final int megaByte = 1024 * 1024;
+
+ for (int x = 0; x < DIR_COUNT; x++) {
+ File subdir = folder.newFolder(String.format("dir%d", x));
+ for (int y = 0; y < SUB_DIR_COUNT; y++) {
+ File targetDir = new File(subdir.getPath().concat(File.separator)
+ .concat(String.format("subdir%d%d", x, y)));
+ if(!targetDir.mkdirs()) {
+ throw new IOException("Failed to create subdirectory. " +
+ targetDir.toString());
+ }
+ for (int z = 0; z < FILE_COUNT; z++) {
+ Path temp = Paths.get(targetDir.getPath().concat(File.separator)
+ .concat(String.format("File%d.txt", z)));
+ byte[] buf = RandomUtils.nextBytes(r.nextInt(megaByte));
+ Files.write(temp, buf);
+ crc.reset();
+ crc.update(buf, 0, buf.length);
+ tmp = crc.getValue();
+ checksumWrite +=tmp;
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testArchive() throws Exception {
+ File archiveFile = new File(outputFolder.getRoot() + File.separator
+ + "test.container.zip");
+ long zipCheckSum = FileUtil.zip(folder.getRoot(), archiveFile);
+ Assert.assertTrue(zipCheckSum > 0);
+ File decomp = new File(outputFolder.getRoot() + File.separator +
+ "decompress");
+ if (!decomp.exists() && !decomp.mkdirs()) {
+ throw new IOException("Unable to create the destination directory. " +
+ decomp.getPath());
+ }
+
+ FileUtil.unZip(archiveFile, decomp);
+ String[] patterns = {"txt"};
+ Iterator<File> iter = FileUtils.iterateFiles(decomp, patterns, true);
+ int count = 0;
+ while (iter.hasNext()) {
+ count++;
+ byte[] buf = Files.readAllBytes(iter.next().toPath());
+ crc.reset();
+ crc.update(buf, 0, buf.length);
+ tmp = crc.getValue();
+ checksumRead += tmp;
+ }
+ Assert.assertEquals(DIR_COUNT * SUB_DIR_COUNT * FILE_COUNT, count);
+ Assert.assertEquals(checksumWrite, checksumRead);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/package-info.java
new file mode 100644
index 0000000..7966941
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm;
+/**
+ Test cases for SCM client classes.
+ */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java
new file mode 100644
index 0000000..6b26b60
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java
@@ -0,0 +1,414 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.utils.BatchOperation;
+import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter;
+import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
+import org.apache.hadoop.utils.MetadataStore;
+import org.apache.hadoop.utils.MetadataStoreBuilder;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Test class for ozone metadata store.
+ */
+@RunWith(Parameterized.class)
+public class TestMetadataStore {
+
+ private final String storeImpl;
+
+ public TestMetadataStore(String metadataImpl) {
+ this.storeImpl = metadataImpl;
+ }
+
+ @Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB},
+ {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB}
+ });
+ }
+
+ private MetadataStore store;
+ private File testDir;
+ private final static int MAX_GETRANGE_LENGTH = 100;
+
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
+ @Before
+ public void init() throws IOException {
+ testDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
+ + "-" + storeImpl.toLowerCase());
+
+ Configuration conf = new OzoneConfiguration();
+ conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
+
+ store = MetadataStoreBuilder.newBuilder()
+ .setConf(conf)
+ .setCreateIfMissing(true)
+ .setDbFile(testDir)
+ .build();
+
+ // Add 20 entries.
+ // {a0 : a-value0} to {a9 : a-value9}
+ // {b0 : b-value0} to {b9 : b-value9}
+ for (int i=0; i<10; i++) {
+ store.put(getBytes("a" + i), getBytes("a-value" + i));
+ store.put(getBytes("b" + i), getBytes("b-value" + i));
+ }
+ }
+
+ @After
+ public void cleanup() throws IOException {
+ store.close();
+ store.destroy();
+ FileUtils.deleteDirectory(testDir);
+ }
+
+ private byte[] getBytes(String str) {
+ return str == null ? null :
+ DFSUtilClient.string2Bytes(str);
+ }
+
+ private String getString(byte[] bytes) {
+ return bytes == null ? null :
+ DFSUtilClient.bytes2String(bytes);
+ }
+
+ @Test
+ public void testGetDelete() throws IOException {
+ for (int i=0; i<10; i++) {
+ byte[] va = store.get(getBytes("a" + i));
+ Assert.assertEquals("a-value" + i, getString(va));
+
+ byte[] vb = store.get(getBytes("b" + i));
+ Assert.assertEquals("b-value" + i, getString(vb));
+ }
+
+ String keyToDel = "del-" + UUID.randomUUID().toString();
+ store.put(getBytes(keyToDel), getBytes(keyToDel));
+ Assert.assertEquals(keyToDel, getString(store.get(getBytes(keyToDel))));
+ store.delete(getBytes(keyToDel));
+ Assert.assertEquals(null, store.get(getBytes(keyToDel)));
+ }
+
+ @Test
+ public void testPeekFrom() throws IOException {
+ // Test peek from an element that has prev as well as next
+ testPeek("a3", "a2", "a4");
+
+ // Test peek from an element that only has prev
+ testPeek("b9", "b8", null);
+
+ // Test peek from an element that only has next
+ testPeek("a0", null, "a1");
+ }
+
+ private String getExpectedValue(String key) {
+ if (key == null) {
+ return null;
+ }
+ char[] arr = key.toCharArray();
+ return new StringBuffer().append(arr[0]).append("-value")
+ .append(arr[arr.length - 1]).toString();
+ }
+
+ private void testPeek(String peekKey, String prevKey, String nextKey)
+ throws IOException {
+ // Look for current
+ String k = null;
+ String v = null;
+ ImmutablePair<byte[], byte[]> current =
+ store.peekAround(0, getBytes(peekKey));
+ if (current != null) {
+ k = getString(current.getKey());
+ v = getString(current.getValue());
+ }
+ Assert.assertEquals(peekKey, k);
+ Assert.assertEquals(v, getExpectedValue(peekKey));
+
+ // Look for prev
+ k = null;
+ v = null;
+ ImmutablePair<byte[], byte[]> prev =
+ store.peekAround(-1, getBytes(peekKey));
+ if (prev != null) {
+ k = getString(prev.getKey());
+ v = getString(prev.getValue());
+ }
+ Assert.assertEquals(prevKey, k);
+ Assert.assertEquals(v, getExpectedValue(prevKey));
+
+ // Look for next
+ k = null;
+ v = null;
+ ImmutablePair<byte[], byte[]> next =
+ store.peekAround(1, getBytes(peekKey));
+ if (next != null) {
+ k = getString(next.getKey());
+ v = getString(next.getValue());
+ }
+ Assert.assertEquals(nextKey, k);
+ Assert.assertEquals(v, getExpectedValue(nextKey));
+ }
+
+ @Test
+ public void testIterateKeys() throws IOException {
+ // iterate keys from b0
+ ArrayList<String> result = Lists.newArrayList();
+ store.iterate(getBytes("b0"), (k, v) -> {
+ // b-value{i}
+ String value = getString(v);
+ char num = value.charAt(value.length() - 1);
+ // each value adds 1
+ int i = Character.getNumericValue(num) + 1;
+ value = value.substring(0, value.length() - 1) + i;
+ result.add(value);
+ return true;
+ });
+
+ Assert.assertFalse(result.isEmpty());
+ for (int i=0; i<result.size(); i++) {
+ Assert.assertEquals("b-value" + (i+1), result.get(i));
+ }
+
+ // iterate from a non exist key
+ result.clear();
+ store.iterate(getBytes("xyz"), (k, v) -> {
+ result.add(getString(v));
+ return true;
+ });
+ Assert.assertTrue(result.isEmpty());
+
+ // iterate from the beginning
+ result.clear();
+ store.iterate(null, (k, v) -> {
+ result.add(getString(v));
+ return true;
+ });
+ Assert.assertEquals(20, result.size());
+ }
+
+ @Test
+ public void testGetRangeKVs() throws IOException {
+ List<Map.Entry<byte[], byte[]>> result = null;
+
+ // Set empty startKey will return values from beginning.
+ result = store.getRangeKVs(null, 5);
+ Assert.assertEquals(5, result.size());
+ Assert.assertEquals("a-value2", getString(result.get(2).getValue()));
+
+ // Empty list if startKey doesn't exist.
+ result = store.getRangeKVs(getBytes("a12"), 5);
+ Assert.assertEquals(0, result.size());
+
+ // Returns max available entries after a valid startKey.
+ result = store.getRangeKVs(getBytes("b0"), MAX_GETRANGE_LENGTH);
+ Assert.assertEquals(10, result.size());
+ Assert.assertEquals("b0", getString(result.get(0).getKey()));
+ Assert.assertEquals("b-value0", getString(result.get(0).getValue()));
+ result = store.getRangeKVs(getBytes("b0"), 5);
+ Assert.assertEquals(5, result.size());
+
+ // Both startKey and count are honored.
+ result = store.getRangeKVs(getBytes("a9"), 2);
+ Assert.assertEquals(2, result.size());
+ Assert.assertEquals("a9", getString(result.get(0).getKey()));
+ Assert.assertEquals("a-value9", getString(result.get(0).getValue()));
+ Assert.assertEquals("b0", getString(result.get(1).getKey()));
+ Assert.assertEquals("b-value0", getString(result.get(1).getValue()));
+
+ // Filter keys by prefix.
+ // It should returns all "b*" entries.
+ MetadataKeyFilter filter1 = new KeyPrefixFilter("b");
+ result = store.getRangeKVs(null, 100, filter1);
+ Assert.assertEquals(10, result.size());
+ Assert.assertTrue(result.stream().allMatch(entry ->
+ new String(entry.getKey()).startsWith("b")
+ ));
+ Assert.assertEquals(20, filter1.getKeysScannedNum());
+ Assert.assertEquals(10, filter1.getKeysHintedNum());
+ result = store.getRangeKVs(null, 3, filter1);
+ Assert.assertEquals(3, result.size());
+ result = store.getRangeKVs(getBytes("b3"), 1, filter1);
+ Assert.assertEquals("b-value3", getString(result.get(0).getValue()));
+
+ // Define a customized filter that filters keys by suffix.
+ // Returns all "*2" entries.
+ MetadataKeyFilter filter2 = (preKey, currentKey, nextKey)
+ -> getString(currentKey).endsWith("2");
+ result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter2);
+ Assert.assertEquals(2, result.size());
+ Assert.assertEquals("a2", getString(result.get(0).getKey()));
+ Assert.assertEquals("b2", getString(result.get(1).getKey()));
+ result = store.getRangeKVs(null, 1, filter2);
+ Assert.assertEquals(1, result.size());
+ Assert.assertEquals("a2", getString(result.get(0).getKey()));
+
+ // Apply multiple filters.
+ result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter1, filter2);
+ Assert.assertEquals(1, result.size());
+ Assert.assertEquals("b2", getString(result.get(0).getKey()));
+ Assert.assertEquals("b-value2", getString(result.get(0).getValue()));
+
+ // If filter is null, no effect.
+ result = store.getRangeKVs(null, 1, null);
+ Assert.assertEquals(1, result.size());
+ Assert.assertEquals("a0", getString(result.get(0).getKey()));
+ }
+
+ @Test
+ public void testGetSequentialRangeKVs() throws IOException {
+ MetadataKeyFilter suffixFilter = (preKey, currentKey, nextKey)
+ -> DFSUtil.bytes2String(currentKey).endsWith("2");
+ // Suppose to return a2 and b2
+ List<Map.Entry<byte[], byte[]>> result =
+ store.getRangeKVs(null, MAX_GETRANGE_LENGTH, suffixFilter);
+ Assert.assertEquals(2, result.size());
+ Assert.assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey()));
+ Assert.assertEquals("b2", DFSUtil.bytes2String(result.get(1).getKey()));
+
+ // Suppose to return just a2, because when it iterates to a3,
+ // the filter no long matches and it should stop from there.
+ result = store.getSequentialRangeKVs(null,
+ MAX_GETRANGE_LENGTH, suffixFilter);
+ Assert.assertEquals(1, result.size());
+ Assert.assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey()));
+ }
+
+ @Test
+ public void testGetRangeLength() throws IOException {
+ List<Map.Entry<byte[], byte[]>> result = null;
+
+ result = store.getRangeKVs(null, 0);
+ Assert.assertEquals(0, result.size());
+
+ result = store.getRangeKVs(null, 1);
+ Assert.assertEquals(1, result.size());
+
+ // Count less than zero is invalid.
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Invalid count given");
+ store.getRangeKVs(null, -1);
+ }
+
+ @Test
+ public void testInvalidStartKey() throws IOException {
+ // If startKey is invalid, the returned list should be empty.
+ List<Map.Entry<byte[], byte[]>> kvs =
+ store.getRangeKVs(getBytes("unknownKey"), MAX_GETRANGE_LENGTH);
+ Assert.assertEquals(kvs.size(), 0);
+ }
+
+ @Test
+ public void testDestroyDB() throws IOException {
+ // create a new DB to test db destroy
+ Configuration conf = new OzoneConfiguration();
+ conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
+
+ File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
+ + "-" + storeImpl.toLowerCase() + "-toDestroy");
+ MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
+ .setConf(conf)
+ .setCreateIfMissing(true)
+ .setDbFile(dbDir)
+ .build();
+
+ dbStore.put(getBytes("key1"), getBytes("value1"));
+ dbStore.put(getBytes("key2"), getBytes("value2"));
+
+ Assert.assertFalse(dbStore.isEmpty());
+ Assert.assertTrue(dbDir.exists());
+ Assert.assertTrue(dbDir.listFiles().length > 0);
+
+ dbStore.destroy();
+
+ Assert.assertFalse(dbDir.exists());
+ }
+
+ @Test
+ public void testBatchWrite() throws IOException {
+ Configuration conf = new OzoneConfiguration();
+ conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
+
+ File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
+ + "-" + storeImpl.toLowerCase() + "-batchWrite");
+ MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
+ .setConf(conf)
+ .setCreateIfMissing(true)
+ .setDbFile(dbDir)
+ .build();
+
+ List<String> expectedResult = Lists.newArrayList();
+ for (int i = 0; i<10; i++) {
+ dbStore.put(getBytes("batch-" + i), getBytes("batch-value-" + i));
+ expectedResult.add("batch-" + i);
+ }
+
+ BatchOperation batch = new BatchOperation();
+ batch.delete(getBytes("batch-2"));
+ batch.delete(getBytes("batch-3"));
+ batch.delete(getBytes("batch-4"));
+ batch.put(getBytes("batch-new-2"), getBytes("batch-new-value-2"));
+
+ expectedResult.remove("batch-2");
+ expectedResult.remove("batch-3");
+ expectedResult.remove("batch-4");
+ expectedResult.add("batch-new-2");
+
+ dbStore.writeBatch(batch);
+
+ Iterator<String> it = expectedResult.iterator();
+ AtomicInteger count = new AtomicInteger(0);
+ dbStore.iterate(null, (key, value) -> {
+ count.incrementAndGet();
+ return it.hasNext() && it.next().equals(getString(key));
+ });
+
+ Assert.assertEquals(8, count.get());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java
new file mode 100644
index 0000000..03c45c5
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone;
+
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * This class is to test acl stoarge and retreival in ozone store.
+ */
+public class TestOzoneAcls {
+
+ @Test
+ public void testAclParse() {
+ HashMap<String, Boolean> testMatrix;
+ testMatrix = new HashMap<>();
+
+ testMatrix.put("user:bilbo:r", Boolean.TRUE);
+ testMatrix.put("user:bilbo:w", Boolean.TRUE);
+ testMatrix.put("user:bilbo:rw", Boolean.TRUE);
+ testMatrix.put("user:bilbo:wr", Boolean.TRUE);
+ testMatrix.put(" user:bilbo:wr ", Boolean.TRUE);
+
+
+ // ACLs makes no judgement on the quality of
+ // user names. it is for the userAuth interface
+ // to determine if a user name is really a name
+ testMatrix.put(" user:*:rw", Boolean.TRUE);
+ testMatrix.put(" user:~!:rw", Boolean.TRUE);
+
+
+ testMatrix.put("", Boolean.FALSE);
+ testMatrix.put(null, Boolean.FALSE);
+ testMatrix.put(" user:bilbo:", Boolean.FALSE);
+ testMatrix.put(" user:bilbo:rx", Boolean.FALSE);
+ testMatrix.put(" user:bilbo:mk", Boolean.FALSE);
+ testMatrix.put(" user::rw", Boolean.FALSE);
+ testMatrix.put("user11:bilbo:rw", Boolean.FALSE);
+ testMatrix.put(" user:::rw", Boolean.FALSE);
+
+ testMatrix.put(" group:hobbit:r", Boolean.TRUE);
+ testMatrix.put(" group:hobbit:w", Boolean.TRUE);
+ testMatrix.put(" group:hobbit:rw", Boolean.TRUE);
+ testMatrix.put(" group:hobbit:wr", Boolean.TRUE);
+ testMatrix.put(" group:*:rw", Boolean.TRUE);
+ testMatrix.put(" group:~!:rw", Boolean.TRUE);
+
+ testMatrix.put(" group:hobbit:", Boolean.FALSE);
+ testMatrix.put(" group:hobbit:rx", Boolean.FALSE);
+ testMatrix.put(" group:hobbit:mk", Boolean.FALSE);
+ testMatrix.put(" group::", Boolean.FALSE);
+ testMatrix.put(" group::rw", Boolean.FALSE);
+ testMatrix.put(" group22:hobbit:", Boolean.FALSE);
+ testMatrix.put(" group:::rw", Boolean.FALSE);
+
+ testMatrix.put("JUNK group:hobbit:r", Boolean.FALSE);
+ testMatrix.put("JUNK group:hobbit:w", Boolean.FALSE);
+ testMatrix.put("JUNK group:hobbit:rw", Boolean.FALSE);
+ testMatrix.put("JUNK group:hobbit:wr", Boolean.FALSE);
+ testMatrix.put("JUNK group:*:rw", Boolean.FALSE);
+ testMatrix.put("JUNK group:~!:rw", Boolean.FALSE);
+
+ testMatrix.put(" world::r", Boolean.TRUE);
+ testMatrix.put(" world::w", Boolean.TRUE);
+ testMatrix.put(" world::rw", Boolean.TRUE);
+ testMatrix.put(" world::wr", Boolean.TRUE);
+
+ testMatrix.put(" world:bilbo:w", Boolean.FALSE);
+ testMatrix.put(" world:bilbo:rw", Boolean.FALSE);
+
+ Set<String> keys = testMatrix.keySet();
+ for (String key : keys) {
+ if (testMatrix.get(key)) {
+ OzoneAcl.parseAcl(key);
+ } else {
+ try {
+ OzoneAcl.parseAcl(key);
+ // should never get here since parseAcl will throw
+ fail("An exception was expected but did not happen.");
+ } catch (IllegalArgumentException e) {
+ // nothing to do
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testAclValues() {
+ OzoneAcl acl = OzoneAcl.parseAcl("user:bilbo:rw");
+ assertEquals(acl.getName(), "bilbo");
+ assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
+ assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
+
+ acl = OzoneAcl.parseAcl("user:bilbo:wr");
+ assertEquals("bilbo", acl.getName());
+ assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
+ assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
+
+ acl = OzoneAcl.parseAcl("user:bilbo:r");
+ assertEquals("bilbo", acl.getName());
+ assertEquals(OzoneAcl.OzoneACLRights.READ, acl.getRights());
+ assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
+
+ acl = OzoneAcl.parseAcl("user:bilbo:w");
+ assertEquals("bilbo", acl.getName());
+ assertEquals(OzoneAcl.OzoneACLRights.WRITE, acl.getRights());
+ assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
+
+ acl = OzoneAcl.parseAcl("group:hobbit:wr");
+ assertEquals(acl.getName(), "hobbit");
+ assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
+ assertEquals(OzoneAcl.OzoneACLType.GROUP, acl.getType());
+
+ acl = OzoneAcl.parseAcl("world::wr");
+ assertEquals(acl.getName(), "");
+ assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
+ assertEquals(OzoneAcl.OzoneACLType.WORLD, acl.getType());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java
new file mode 100644
index 0000000..c1470bb
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.common;
+
+import org.apache.commons.collections.SetUtils;
+import org.apache.hadoop.ozone.common.statemachine
+ .InvalidStateTransitionException;
+import org.apache.hadoop.ozone.common.statemachine.StateMachine;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CLEANUP;
+import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CLOSED;
+import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CREATING;
+import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.FINAL;
+import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.INIT;
+import static org.apache.hadoop.ozone.common.TestStateMachine.STATES
+ .OPERATIONAL;
+
+/**
+ * This class is to test ozone common state machine.
+ */
+public class TestStateMachine {
+
+ /**
+ * STATES used by the test state machine.
+ */
+ public enum STATES {INIT, CREATING, OPERATIONAL, CLOSED, CLEANUP, FINAL};
+
+ /**
+ * EVENTS used by the test state machine.
+ */
+ public enum EVENTS {ALLOCATE, CREATE, UPDATE, CLOSE, DELETE, TIMEOUT};
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ @Test
+ public void testStateMachineStates() throws InvalidStateTransitionException {
+ Set<STATES> finals = new HashSet<>();
+ finals.add(FINAL);
+
+ StateMachine<STATES, EVENTS> stateMachine =
+ new StateMachine<>(INIT, finals);
+
+ stateMachine.addTransition(INIT, CREATING, EVENTS.ALLOCATE);
+ stateMachine.addTransition(CREATING, OPERATIONAL, EVENTS.CREATE);
+ stateMachine.addTransition(OPERATIONAL, OPERATIONAL, EVENTS.UPDATE);
+ stateMachine.addTransition(OPERATIONAL, CLEANUP, EVENTS.DELETE);
+ stateMachine.addTransition(OPERATIONAL, CLOSED, EVENTS.CLOSE);
+ stateMachine.addTransition(CREATING, CLEANUP, EVENTS.TIMEOUT);
+
+ // Initial and Final states
+ Assert.assertEquals("Initial State", INIT, stateMachine.getInitialState());
+ Assert.assertTrue("Final States", SetUtils.isEqualSet(finals,
+ stateMachine.getFinalStates()));
+
+ // Valid state transitions
+ Assert.assertEquals("STATE should be OPERATIONAL after being created",
+ OPERATIONAL, stateMachine.getNextState(CREATING, EVENTS.CREATE));
+ Assert.assertEquals("STATE should be OPERATIONAL after being updated",
+ OPERATIONAL, stateMachine.getNextState(OPERATIONAL, EVENTS.UPDATE));
+ Assert.assertEquals("STATE should be CLEANUP after being deleted",
+ CLEANUP, stateMachine.getNextState(OPERATIONAL, EVENTS.DELETE));
+ Assert.assertEquals("STATE should be CLEANUP after being timeout",
+ CLEANUP, stateMachine.getNextState(CREATING, EVENTS.TIMEOUT));
+ Assert.assertEquals("STATE should be CLOSED after being closed",
+ CLOSED, stateMachine.getNextState(OPERATIONAL, EVENTS.CLOSE));
+
+ // Negative cases: invalid transition
+ expectException();
+ stateMachine.getNextState(OPERATIONAL, EVENTS.CREATE);
+
+ expectException();
+ stateMachine.getNextState(CREATING, EVENTS.CLOSE);
+ }
+
+ /**
+ * We expect an InvalidStateTransitionException.
+ */
+ private void expectException() {
+ exception.expect(InvalidStateTransitionException.class);
+ exception.expectMessage("Invalid event");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java
new file mode 100644
index 0000000..517c1a7
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java
@@ -0,0 +1,374 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+/**
+ * A generic lease management API which can be used if a service
+ * needs any kind of lease management.
+ */
+
+package org.apache.hadoop.ozone.lease;
+
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Test class to check functionality and consistency of LeaseManager.
+ */
+public class TestLeaseManager {
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ /**
+ * Dummy resource on which leases can be acquired.
+ */
+ private final class DummyResource {
+
+ private final String name;
+
+ private DummyResource(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public int hashCode() {
+ return name.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if(obj instanceof DummyResource) {
+ return name.equals(((DummyResource) obj).name);
+ }
+ return false;
+ }
+ }
+
+ @Test
+ public void testLeaseAcquireAndRelease() throws LeaseException {
+ //It is assumed that the test case execution won't take more than 5 seconds,
+ //if it takes more time increase the defaultTimeout value of LeaseManager.
+ LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+ manager.start();
+ DummyResource resourceOne = new DummyResource("one");
+ DummyResource resourceTwo = new DummyResource("two");
+ DummyResource resourceThree = new DummyResource("three");
+ Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+ Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo);
+ Lease<DummyResource> leaseThree = manager.acquire(resourceThree);
+ Assert.assertEquals(leaseOne, manager.get(resourceOne));
+ Assert.assertEquals(leaseTwo, manager.get(resourceTwo));
+ Assert.assertEquals(leaseThree, manager.get(resourceThree));
+ Assert.assertFalse(leaseOne.hasExpired());
+ Assert.assertFalse(leaseTwo.hasExpired());
+ Assert.assertFalse(leaseThree.hasExpired());
+ //The below releases should not throw LeaseNotFoundException.
+ manager.release(resourceOne);
+ manager.release(resourceTwo);
+ manager.release(resourceThree);
+ Assert.assertTrue(leaseOne.hasExpired());
+ Assert.assertTrue(leaseTwo.hasExpired());
+ Assert.assertTrue(leaseThree.hasExpired());
+ manager.shutdown();
+ }
+
+ @Test
+ public void testLeaseAlreadyExist() throws LeaseException {
+ LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+ manager.start();
+ DummyResource resourceOne = new DummyResource("one");
+ DummyResource resourceTwo = new DummyResource("two");
+ Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+ Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo);
+ Assert.assertEquals(leaseOne, manager.get(resourceOne));
+ Assert.assertEquals(leaseTwo, manager.get(resourceTwo));
+
+ exception.expect(LeaseAlreadyExistException.class);
+ exception.expectMessage("Resource: " + resourceOne);
+ manager.acquire(resourceOne);
+
+ manager.release(resourceOne);
+ manager.release(resourceTwo);
+ manager.shutdown();
+ }
+
+ @Test
+ public void testLeaseNotFound() throws LeaseException, InterruptedException {
+ LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+ manager.start();
+ DummyResource resourceOne = new DummyResource("one");
+ DummyResource resourceTwo = new DummyResource("two");
+ DummyResource resourceThree = new DummyResource("three");
+
+ //Case 1: lease was never acquired.
+ exception.expect(LeaseNotFoundException.class);
+ exception.expectMessage("Resource: " + resourceOne);
+ manager.get(resourceOne);
+
+ //Case 2: lease is acquired and released.
+ Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo);
+ Assert.assertEquals(leaseTwo, manager.get(resourceTwo));
+ Assert.assertFalse(leaseTwo.hasExpired());
+ manager.release(resourceTwo);
+ Assert.assertTrue(leaseTwo.hasExpired());
+ exception.expect(LeaseNotFoundException.class);
+ exception.expectMessage("Resource: " + resourceTwo);
+ manager.get(resourceTwo);
+
+ //Case 3: lease acquired and timed out.
+ Lease<DummyResource> leaseThree = manager.acquire(resourceThree);
+ Assert.assertEquals(leaseThree, manager.get(resourceThree));
+ Assert.assertFalse(leaseThree.hasExpired());
+ long sleepTime = leaseThree.getRemainingTime() + 1000;
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException ex) {
+ //even in case of interrupt we have to wait till lease times out.
+ Thread.sleep(sleepTime);
+ }
+ Assert.assertTrue(leaseThree.hasExpired());
+ exception.expect(LeaseNotFoundException.class);
+ exception.expectMessage("Resource: " + resourceThree);
+ manager.get(resourceThree);
+ manager.shutdown();
+ }
+
+ @Test
+ public void testCustomLeaseTimeout() throws LeaseException {
+ LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+ manager.start();
+ DummyResource resourceOne = new DummyResource("one");
+ DummyResource resourceTwo = new DummyResource("two");
+ DummyResource resourceThree = new DummyResource("three");
+ Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+ Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo, 10000);
+ Lease<DummyResource> leaseThree = manager.acquire(resourceThree, 50000);
+ Assert.assertEquals(leaseOne, manager.get(resourceOne));
+ Assert.assertEquals(leaseTwo, manager.get(resourceTwo));
+ Assert.assertEquals(leaseThree, manager.get(resourceThree));
+ Assert.assertFalse(leaseOne.hasExpired());
+ Assert.assertFalse(leaseTwo.hasExpired());
+ Assert.assertFalse(leaseThree.hasExpired());
+ Assert.assertEquals(5000, leaseOne.getLeaseLifeTime());
+ Assert.assertEquals(10000, leaseTwo.getLeaseLifeTime());
+ Assert.assertEquals(50000, leaseThree.getLeaseLifeTime());
+ // Releasing of leases is done in shutdown, so don't have to worry about
+ // lease release
+ manager.shutdown();
+ }
+
+ @Test
+ public void testLeaseCallback() throws LeaseException, InterruptedException {
+ Map<DummyResource, String> leaseStatus = new HashMap<>();
+ LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+ manager.start();
+ DummyResource resourceOne = new DummyResource("one");
+ Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+ leaseStatus.put(resourceOne, "lease in use");
+ leaseOne.registerCallBack(() -> {
+ leaseStatus.put(resourceOne, "lease expired");
+ return null;
+ });
+ // wait for lease to expire
+ long sleepTime = leaseOne.getRemainingTime() + 1000;
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException ex) {
+ //even in case of interrupt we have to wait till lease times out.
+ Thread.sleep(sleepTime);
+ }
+ Assert.assertTrue(leaseOne.hasExpired());
+ exception.expect(LeaseNotFoundException.class);
+ exception.expectMessage("Resource: " + resourceOne);
+ manager.get(resourceOne);
+ // check if callback has been executed
+ Assert.assertEquals("lease expired", leaseStatus.get(resourceOne));
+ }
+
+ @Test
+ public void testCallbackExecutionInCaseOfLeaseRelease()
+ throws LeaseException, InterruptedException {
+ // Callbacks should not be executed in case of lease release
+ Map<DummyResource, String> leaseStatus = new HashMap<>();
+ LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+ manager.start();
+ DummyResource resourceOne = new DummyResource("one");
+ Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+ leaseStatus.put(resourceOne, "lease in use");
+ leaseOne.registerCallBack(() -> {
+ leaseStatus.put(resourceOne, "lease expired");
+ return null;
+ });
+ leaseStatus.put(resourceOne, "lease released");
+ manager.release(resourceOne);
+ Assert.assertTrue(leaseOne.hasExpired());
+ exception.expect(LeaseNotFoundException.class);
+ exception.expectMessage("Resource: " + resourceOne);
+ manager.get(resourceOne);
+ Assert.assertEquals("lease released", leaseStatus.get(resourceOne));
+ }
+
+ @Test
+ public void testLeaseCallbackWithMultipleLeases()
+ throws LeaseException, InterruptedException {
+ Map<DummyResource, String> leaseStatus = new HashMap<>();
+ LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+ manager.start();
+ DummyResource resourceOne = new DummyResource("one");
+ DummyResource resourceTwo = new DummyResource("two");
+ DummyResource resourceThree = new DummyResource("three");
+ DummyResource resourceFour = new DummyResource("four");
+ DummyResource resourceFive = new DummyResource("five");
+ Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+ Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo);
+ Lease<DummyResource> leaseThree = manager.acquire(resourceThree);
+ Lease<DummyResource> leaseFour = manager.acquire(resourceFour);
+ Lease<DummyResource> leaseFive = manager.acquire(resourceFive);
+ leaseStatus.put(resourceOne, "lease in use");
+ leaseStatus.put(resourceTwo, "lease in use");
+ leaseStatus.put(resourceThree, "lease in use");
+ leaseStatus.put(resourceFour, "lease in use");
+ leaseStatus.put(resourceFive, "lease in use");
+ leaseOne.registerCallBack(() -> {
+ leaseStatus.put(resourceOne, "lease expired");
+ return null;
+ });
+ leaseTwo.registerCallBack(() -> {
+ leaseStatus.put(resourceTwo, "lease expired");
+ return null;
+ });
+ leaseThree.registerCallBack(() -> {
+ leaseStatus.put(resourceThree, "lease expired");
+ return null;
+ });
+ leaseFour.registerCallBack(() -> {
+ leaseStatus.put(resourceFour, "lease expired");
+ return null;
+ });
+ leaseFive.registerCallBack(() -> {
+ leaseStatus.put(resourceFive, "lease expired");
+ return null;
+ });
+
+ // release lease one, two and three
+ leaseStatus.put(resourceOne, "lease released");
+ manager.release(resourceOne);
+ leaseStatus.put(resourceTwo, "lease released");
+ manager.release(resourceTwo);
+ leaseStatus.put(resourceThree, "lease released");
+ manager.release(resourceThree);
+
+ // wait for other leases to expire
+ long sleepTime = leaseFive.getRemainingTime() + 1000;
+
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException ex) {
+ //even in case of interrupt we have to wait till lease times out.
+ Thread.sleep(sleepTime);
+ }
+ Assert.assertTrue(leaseOne.hasExpired());
+ Assert.assertTrue(leaseTwo.hasExpired());
+ Assert.assertTrue(leaseThree.hasExpired());
+ Assert.assertTrue(leaseFour.hasExpired());
+ Assert.assertTrue(leaseFive.hasExpired());
+
+ Assert.assertEquals("lease released", leaseStatus.get(resourceOne));
+ Assert.assertEquals("lease released", leaseStatus.get(resourceTwo));
+ Assert.assertEquals("lease released", leaseStatus.get(resourceThree));
+ Assert.assertEquals("lease expired", leaseStatus.get(resourceFour));
+ Assert.assertEquals("lease expired", leaseStatus.get(resourceFive));
+ manager.shutdown();
+ }
+
+ @Test
+ public void testReuseReleasedLease() throws LeaseException {
+ LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+ manager.start();
+ DummyResource resourceOne = new DummyResource("one");
+ Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+ Assert.assertEquals(leaseOne, manager.get(resourceOne));
+ Assert.assertFalse(leaseOne.hasExpired());
+
+ manager.release(resourceOne);
+ Assert.assertTrue(leaseOne.hasExpired());
+
+ Lease<DummyResource> sameResourceLease = manager.acquire(resourceOne);
+ Assert.assertEquals(sameResourceLease, manager.get(resourceOne));
+ Assert.assertFalse(sameResourceLease.hasExpired());
+
+ manager.release(resourceOne);
+ Assert.assertTrue(sameResourceLease.hasExpired());
+ manager.shutdown();
+ }
+
+ @Test
+ public void testReuseTimedOutLease()
+ throws LeaseException, InterruptedException {
+ LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+ manager.start();
+ DummyResource resourceOne = new DummyResource("one");
+ Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+ Assert.assertEquals(leaseOne, manager.get(resourceOne));
+ Assert.assertFalse(leaseOne.hasExpired());
+
+ // wait for lease to expire
+ long sleepTime = leaseOne.getRemainingTime() + 1000;
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException ex) {
+ //even in case of interrupt we have to wait till lease times out.
+ Thread.sleep(sleepTime);
+ }
+ Assert.assertTrue(leaseOne.hasExpired());
+
+ Lease<DummyResource> sameResourceLease = manager.acquire(resourceOne);
+ Assert.assertEquals(sameResourceLease, manager.get(resourceOne));
+ Assert.assertFalse(sameResourceLease.hasExpired());
+
+ manager.release(resourceOne);
+ Assert.assertTrue(sameResourceLease.hasExpired());
+ manager.shutdown();
+ }
+
+ @Test
+ public void testRenewLease() throws LeaseException, InterruptedException {
+ LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+ manager.start();
+ DummyResource resourceOne = new DummyResource("one");
+ Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+ Assert.assertEquals(leaseOne, manager.get(resourceOne));
+ Assert.assertFalse(leaseOne.hasExpired());
+
+ // add 5 more seconds to the lease
+ leaseOne.renew(5000);
+
+ Thread.sleep(5000);
+
+ // lease should still be active
+ Assert.assertEquals(leaseOne, manager.get(resourceOne));
+ Assert.assertFalse(leaseOne.hasExpired());
+ manager.release(resourceOne);
+ manager.shutdown();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java
new file mode 100644
index 0000000..1071309
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.lease;
+/*
+ This package contains lease management unit test classes.
+ */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/package-info.java
new file mode 100644
index 0000000..0030d2e
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+/**
+ * Ozone related test helper classes and tests of common utils.
+ */
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org