You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2018/04/26 21:00:44 UTC
[15/68] [abbrv] [partial] hadoop git commit: HDFS-13405. Ozone:
Rename HDSL to HDDS. Contributed by Ajay Kumar, Elek Marton, Mukul Kumar Singh,
Shashikant Banerjee and Anu Engineer.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/common/src/main/resources/ozone-default.xml b/hadoop-hdsl/common/src/main/resources/ozone-default.xml
deleted file mode 100644
index 9feadcf..0000000
--- a/hadoop-hdsl/common/src/main/resources/ozone-default.xml
+++ /dev/null
@@ -1,1031 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!-- Do not modify this file directly. Instead, copy entries that you -->
-<!-- wish to modify from this file into ozone-site.xml and change them -->
-<!-- there. If ozone-site.xml does not already exist, create it. -->
-
-<!--Tags supported are OZONE, CBLOCK, MANAGEMENT, SECURITY, PERFORMANCE, -->
-<!--DEBUG, CLIENT, SERVER, KSM, SCM, CRITICAL, RATIS, CONTAINER, REQUIRED, -->
-<!--REST, STORAGE, PIPELINE, STANDALONE -->
-
-<configuration>
-
- <!--Container Settings used by Datanode-->
- <property>
- <name>ozone.container.cache.size</name>
- <value>1024</value>
- <tag>PERFORMANCE, CONTAINER, STORAGE</tag>
- <description>The open container is cached on the data node side. We maintain
- an LRU
- cache for caching the recently used containers. This setting controls the
- size of that cache.
- </description>
- </property>
- <property>
- <name>dfs.container.ipc</name>
- <value>9859</value>
- <tag>OZONE, CONTAINER, MANAGEMENT</tag>
- <description>The ipc port number of container.</description>
- </property>
- <property>
- <name>dfs.container.ipc.random.port</name>
- <value>false</value>
- <tag>OZONE, DEBUG, CONTAINER</tag>
- <description>Allocates a random free port for ozone container. This is used
- only while
- running unit tests.
- </description>
- </property>
- <property>
- <name>dfs.container.ratis.datanode.storage.dir</name>
- <value/>
- <tag>OZONE, CONTAINER, STORAGE, MANAGEMENT, RATIS</tag>
- <description>This directory is used for storing Ratis metadata like logs. If
- this is
- not set then default metadata dirs is used. A warning will be logged if
- this not set. Ideally, this should be mapped to a fast disk like an SSD.
- </description>
- </property>
- <property>
- <name>dfs.container.ratis.enabled</name>
- <value>false</value>
- <tag>OZONE, MANAGEMENT, PIPELINE, RATIS</tag>
- <description>Ozone supports different kinds of replication pipelines. Ratis
- is one of
- the replication pipeline supported by ozone.
- </description>
- </property>
- <property>
- <name>dfs.container.ratis.ipc</name>
- <value>9858</value>
- <tag>OZONE, CONTAINER, PIPELINE, RATIS, MANAGEMENT</tag>
- <description>The ipc port number of container.</description>
- </property>
- <property>
- <name>dfs.container.ratis.ipc.random.port</name>
- <value>false</value>
- <tag>OZONE,DEBUG</tag>
- <description>Allocates a random free port for ozone ratis port for the
- container. This
- is used only while running unit tests.
- </description>
- </property>
- <property>
- <name>dfs.container.ratis.rpc.type</name>
- <value>GRPC</value>
- <tag>OZONE, RATIS, MANAGEMENT</tag>
- <description>Ratis supports different kinds of transports like netty, GRPC,
- Hadoop RPC
- etc. This picks one of those for this cluster.
- </description>
- </property>
- <property>
- <name>dfs.container.ratis.num.write.chunk.threads</name>
- <value>60</value>
- <tag>OZONE, RATIS, PERFORMANCE</tag>
- <description>Maximum number of threads in the thread pool that Ratis
- will use for writing chunks (60 by default).
- </description>
- </property>
- <property>
- <name>dfs.container.ratis.segment.size</name>
- <value>1073741824</value>
- <tag>OZONE, RATIS, PERFORMANCE</tag>
- <description>The size of the raft segment used by Apache Ratis on datanodes.
- (1 GB by default)
- </description>
- </property>
- <property>
- <name>dfs.container.ratis.segment.preallocated.size</name>
- <value>134217728</value>
- <tag>OZONE, RATIS, PERFORMANCE</tag>
- <description>The size of the buffer which is preallocated for raft segment
- used by Apache Ratis on datanodes.(128 MB by default)
- </description>
- </property>
- <property>
- <name>ozone.container.report.interval</name>
- <value>60000ms</value>
- <tag>OZONE, CONTAINER, MANAGEMENT</tag>
- <description>Time interval of the datanode to send container report. Each
- datanode periodically send container report upon receive
- sendContainerReport from SCM. Unit could be defined with
- postfix (ns,ms,s,m,h,d)</description>
- </property>
- <!--Ozone Settings-->
- <property>
- <name>ozone.administrators</name>
- <value/>
- <tag>OZONE, SECURITY</tag>
- <description>Ozone administrator users delimited by the comma.
- If not set, only the user who launches an ozone service will be the admin
- user. This property must be set if ozone services are started by different
- users. Otherwise, the RPC layer will reject calls from other servers which
- are started by users not in the list.
- </description>
- </property>
- <property>
- <name>ozone.block.deleting.container.limit.per.interval</name>
- <value>10</value>
- <tag>OZONE, PERFORMANCE, SCM</tag>
- <description>A maximum number of containers to be scanned by block deleting
- service per
- time interval. The block deleting service spawns a thread to handle block
- deletions in a container. This property is used to throttle the number of
- threads spawned for block deletions.
- </description>
- </property>
- <property>
- <name>ozone.block.deleting.limit.per.task</name>
- <value>1000</value>
- <tag>OZONE, PERFORMANCE, SCM</tag>
- <description>A maximum number of blocks to be deleted by block deleting
- service per
- time interval. This property is used to throttle the actual number of
- block deletions on a data node per container.
- </description>
- </property>
- <property>
- <name>ozone.block.deleting.service.interval</name>
- <value>1m</value>
- <tag>OZONE, PERFORMANCE, SCM</tag>
- <description>Time interval of the block deleting service.
- The block deleting service runs on each datanode periodically and
- deletes blocks queued for deletion. Unit could be defined with
- postfix (ns,ms,s,m,h,d)
- </description>
- </property>
- <property>
- <name>ozone.block.deleting.service.timeout</name>
- <value>300000ms</value>
- <tag>OZONE, PERFORMANCE, SCM</tag>
- <description>A timeout value of block deletion service. If this is set
- greater than 0,
- the service will stop waiting for the block deleting completion after this
- time. If timeout happens to a large proportion of block deletion, this
- needs to be increased with ozone.block.deleting.limit.per.task. This
- setting supports multiple time unit suffixes as described in
- dfs.heartbeat.interval. If no suffix is specified, then milliseconds is
- assumed.
- </description>
- </property>
- <property>
- <name>ozone.client.connection.timeout</name>
- <value>5000ms</value>
- <tag>OZONE, PERFORMANCE, CLIENT</tag>
- <description>Connection timeout for Ozone client in milliseconds.
- </description>
- </property>
- <property>
- <name>ozone.client.protocol</name>
- <value>org.apache.hadoop.ozone.client.rpc.RpcClient</value>
- <tag>OZONE, CLIENT, MANAGEMENT</tag>
- <description>Protocol class to be used by the client to connect to ozone
- cluster.
- The build-in implementation includes:
- org.apache.hadoop.ozone.client.rpc.RpcClient for RPC
- org.apache.hadoop.ozone.client.rest.RestClient for REST
- The default is the RpClient. Please do not change this unless you have a
- very good understanding of what you are doing.
- </description>
- </property>
- <property>
- <name>ozone.client.socket.timeout</name>
- <value>5000ms</value>
- <tag>OZONE, CLIENT</tag>
- <description>Socket timeout for Ozone client. Unit could be defined with
- postfix (ns,ms,s,m,h,d)</description>
- </property>
- <property>
- <name>ozone.enabled</name>
- <value>false</value>
- <tag>OZONE, REQUIRED</tag>
- <description>
- Status of the Ozone Object Storage service is enabled.
- Set to true to enable Ozone.
- Set to false to disable Ozone.
- Unless this value is set to true, Ozone services will not be started in
- the cluster.
-
- Please note: By default ozone is disabled on a hadoop cluster.
- </description>
- </property>
- <property>
- <name>ozone.handler.type</name>
- <value>distributed</value>
- <tag>OZONE, REST</tag>
- <description>
- Tells ozone which storage handler to use. The possible values are:
- distributed - The Ozone distributed storage handler, which speaks to
- KSM/SCM on the backend and provides REST services to clients.
- local - Local Storage handler strictly for testing - To be removed.
- </description>
- </property>
- <property>
- <name>ozone.key.deleting.limit.per.task</name>
- <value>1000</value>
- <tag>KSM, PERFORMANCE</tag>
- <description>
- A maximum number of keys to be scanned by key deleting service
- per time interval in KSM. Those keys are sent to delete metadata and
- generate transactions in SCM for next async deletion between SCM
- and DataNode.
- </description>
- </property>
- <property>
- <name>ozone.ksm.address</name>
- <value/>
- <tag>KSM, REQUIRED</tag>
- <description>
- The address of the Ozone KSM service. This allows clients to discover
- the KSMs address.
- </description>
- </property>
- <property>
- <name>ozone.ksm.group.rights</name>
- <value>READ_WRITE</value>
- <tag>KSM, SECURITY</tag>
- <description>
- Default group permissions in Ozone KSM.
- </description>
- </property>
- <property>
- <name>ozone.ksm.handler.count.key</name>
- <value>20</value>
- <tag>KSM, PERFORMANCE</tag>
- <description>
- The number of RPC handler threads for KSM service endpoints.
- </description>
- </property>
- <property>
- <name>ozone.ksm.http-address</name>
- <value>0.0.0.0:9874</value>
- <tag>KSM, MANAGEMENT</tag>
- <description>
- The address and the base port where the KSM web UI will listen on.
-
- If the port is 0, then the server will start on a free port. However, it
- is best to specify a well-known port, so it is easy to connect and see
- the KSM management UI.
- </description>
- </property>
- <property>
- <name>ozone.ksm.http-bind-host</name>
- <value>0.0.0.0</value>
- <tag>KSM, MANAGEMENT</tag>
- <description>
- The actual address the KSM web server will bind to. If this optional
- the address is set, it overrides only the hostname portion of
- ozone.ksm.http-address.
- </description>
- </property>
- <property>
- <name>ozone.ksm.http.enabled</name>
- <value>true</value>
- <tag>KSM, MANAGEMENT</tag>
- <description>
- Property to enable or disable KSM web user interface.
- </description>
- </property>
- <property>
- <name>ozone.ksm.https-address</name>
- <value>0.0.0.0:9875</value>
- <tag>KSM, MANAGEMENT, SECURITY</tag>
- <description>
- The address and the base port where the KSM web UI will listen
- on using HTTPS.
- If the port is 0 then the server will start on a free port.
- </description>
- </property>
- <property>
- <name>ozone.ksm.https-bind-host</name>
- <value>0.0.0.0</value>
- <tag>KSM, MANAGEMENT, SECURITY</tag>
- <description>
- The actual address the KSM web server will bind to using HTTPS.
- If this optional address is set, it overrides only the hostname portion of
- ozone.ksm.http-address.
- </description>
- </property>
- <property>
- <name>ozone.ksm.keytab.file</name>
- <value/>
- <tag>KSM, SECURITY</tag>
- <description>
- The keytab file for Kerberos authentication in KSM.
- </description>
- </property>
- <property>
- <name>ozone.ksm.db.cache.size.mb</name>
- <value>128</value>
- <tag>KSM, PERFORMANCE</tag>
- <description>
- The size of KSM DB cache in MB that used for caching files.
- This value is set to an abnormally low value in the default configuration.
- That is to make unit testing easy. Generally, this value should be set to
- something like 16GB or more, if you intend to use Ozone at scale.
-
- A large value for this key allows a proportionally larger amount of KSM
- metadata to be cached in memory. This makes KSM operations faster.
- </description>
- </property>
- <property>
- <name>ozone.ksm.user.max.volume</name>
- <value>1024</value>
- <tag>KSM, MANAGEMENT</tag>
- <description>
- The maximum number of volumes a user can have on a cluster.Increasing or
- decreasing this number has no real impact on ozone cluster. This is
- defined only for operational purposes. Only an administrator can create a
- volume, once a volume is created there are no restrictions on the number
- of buckets or keys inside each bucket a user can create.
- </description>
- </property>
- <property>
- <name>ozone.ksm.user.rights</name>
- <value>READ_WRITE</value>
- <tag>KSM, SECURITY</tag>
- <description>
- Default user permissions used in KSM.
- </description>
- </property>
- <property>
- <name>ozone.localstorage.root</name>
- <value>${hadoop.tmp.dir}/ozone</value>
- <tag>OZONE, DEBUG</tag>
- <description>
- This is used only for testing purposes. This value is used by the local
- storage handler to simulate a REST backend. This is useful only when
- debugging the REST front end independent of KSM and SCM. To be removed.
- </description>
- </property>
- <property>
- <name>ozone.metadata.dirs</name>
- <value/>
- <tag>OZONE, KSM, SCM, CONTAINER, REQUIRED, STORAGE</tag>
- <description>
- Ozone metadata is shared among KSM, which acts as the namespace
- manager for ozone, SCM which acts as the block manager and data nodes
- which maintain the name of the key(Key Name and BlockIDs). This
- replicated and distributed metadata store is maintained under the
- directory pointed by this key. Since metadata can be I/O intensive, at
- least on KSM and SCM we recommend having SSDs. If you have the luxury
- of mapping this path to SSDs on all machines in the cluster, that will
- be excellent.
-
- If Ratis metadata directories are not specified, Ratis server will emit a
- warning and use this path for storing its metadata too.
- </description>
- </property>
- <property>
- <name>ozone.metastore.impl</name>
- <value>RocksDB</value>
- <tag>OZONE, KSM, SCM, CONTAINER, STORAGE</tag>
- <description>
- Ozone metadata store implementation. Ozone metadata are well
- distributed to multiple services such as ksm, scm. They are stored in
- some local key-value databases. This property determines which database
- library to use. Supported value is either LevelDB or RocksDB.
- </description>
- </property>
-
- <property>
- <name>ozone.metastore.rocksdb.statistics</name>
- <value>ALL</value>
- <tag>OZONE, KSM, SCM, STORAGE, PERFORMANCE</tag>
- <description>
- The statistics level of the rocksdb store. If you use any value from
- org.rocksdb.StatsLevel (eg. ALL or EXCEPT_DETAILED_TIMERS), the rocksdb
- statistics will be exposed over JMX bean with the choosed setting. Set
- it to OFF to not initialize rocksdb statistics at all. Please note that
- collection of statistics could have 5-10% performance penalty.
- Check the rocksdb documentation for more details.
- </description>
- </property>
-
- <property>
- <name>ozone.scm.block.client.address</name>
- <value/>
- <tag>OZONE, SCM</tag>
- <description>The address of the Ozone SCM block client service. If not
- defined value of ozone.scm.client.address is used.
- </description>
- </property>
- <property>
- <name>ozone.scm.block.client.bind.host</name>
- <value>0.0.0.0</value>
- <tag>OZONE, SCM</tag>
- <description>
- The hostname or IP address used by the SCM block client
- endpoint to bind.
- </description>
- </property>
- <property>
- <name>ozone.scm.block.client.port</name>
- <value>9863</value>
- <tag>OZONE, SCM</tag>
- <description>
- The port number of the Ozone SCM block client service.
- </description>
- </property>
- <property>
- <name>ozone.scm.block.deletion.max.retry</name>
- <value>4096</value>
- <tag>OZONE, SCM</tag>
- <description>
- SCM wraps up many blocks in a deletion transaction and sends that to data
- node for physical deletion periodically. This property determines how many
- times SCM is going to retry sending a deletion operation to the data node.
- </description>
- </property>
- <property>
- <name>ozone.scm.block.size.in.mb</name>
- <value>256</value>
- <tag>OZONE, SCM</tag>
- <description>
- The default size of a scm block in bytes. This is maps to the default
- Ozone block size.
- </description>
- </property>
- <property>
- <name>ozone.scm.chunk.size</name>
- <value>16777216</value>
- <tag>OZONE, SCM, CONTAINER, PERFORMANCE</tag>
- <description>
- The chunk size for reading/writing chunk operations in bytes.
-
- The chunk size defaults to 8MB. If the value configured is more than the
- maximum size (16MB), it will be reset to the maximum size. This maps to
- the network packet sizes and file write operations in the client to
- datanode protocol.
- </description>
- </property>
- <property>
- <name>ozone.scm.client.address</name>
- <value/>
- <tag>OZONE, SCM, REQUIRED</tag>
- <description>
- The address of the Ozone SCM client service. This is a required setting.
-
- It is a string in the host:port format. The port number is optional
- and defaults to 9860.
- </description>
- </property>
- <property>
- <name>ozone.scm.client.bind.host</name>
- <value>0.0.0.0</value>
- <tag>OZONE, SCM, MANAGEMENT</tag>
- <description>The hostname or IP address used by the SCM client endpoint to
- bind.
- This setting is used by the SCM only and never used by clients.
-
- The setting can be useful in multi-homed setups to restrict the
- availability of the SCM client service to a specific interface.
-
- The default is appropriate for most clusters.
- </description>
- </property>
- <property>
- <name>ozone.scm.client.port</name>
- <value>9860</value>
- <tag>OZONE, SCM, MANAGEMENT</tag>
- <description>The port number of the Ozone SCM client service.</description>
- </property>
- <property>
- <name>ozone.scm.container.deletion-choosing.policy</name>
- <value>
- org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy
- </value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The policy used for choosing desire containers for block deletion.
- Datanode selects some containers to process block deletion
- in a certain interval defined by ozone.block.deleting.service.interval.
- The number of containers to process in each interval is defined
- by ozone.block.deleting.container.limit.per.interval. This property is
- used to configure the policy applied while selecting containers.
- There are two policies supporting now:
- RandomContainerDeletionChoosingPolicy and
- TopNOrderedContainerDeletionChoosingPolicy.
- org.apache.hadoop.ozone.container.common.impl.RandomContainerDeletionChoosingPolicy
- implements a simply random policy that to return a random list of
- containers.
- org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy
- implements a policy that choosing top count number of containers in a
- pending-deletion-blocks's num
- based descending order.
- </description>
- </property>
- <property>
- <name>ozone.scm.container.placement.impl</name>
- <value>
- org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementRandom
- </value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>Placement policy class for containers.
- Defaults to SCMContainerPlacementRandom.class
- </description>
- </property>
- <property>
- <name>ozone.scm.container.provision_batch_size</name>
- <value>20</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>Pre-provision specified number of containers for block
- allocation.
- </description>
- </property>
- <property>
- <name>ozone.scm.container.report.processing.interval</name>
- <value>60s</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>Time interval for scm to process container reports
- for a node pool. Scm handles node pool reports in a cyclic clock
- manner, it fetches pools periodically with this time interval.
- </description>
- </property>
- <property>
- <name>ozone.scm.container.reports.wait.timeout</name>
- <value>300s</value>
- <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
- <description>Maximum time to wait in seconds for processing all container
- reports from
- a node pool. It determines the timeout for a
- node pool report.
- </description>
- </property>
- <property>
- <name>ozone.scm.container.size.gb</name>
- <value>5</value>
- <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
- <description>
- Default container size used by Ozone. This value is specified
- in GB.
- There are two considerations while picking this number. The speed at which
- a container can be replicated, determined by the network speed and the
- metadata that each container generates. So selecting a large number
- creates less SCM metadata, but recovery time will be more. 5GB is a number
- that maps to quick replication times in gigabit networks, but still
- balances the amount of metadata.
- </description>
- </property>
- <property>
- <name>ozone.scm.datanode.address</name>
- <value/>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The address of the Ozone SCM service used for internal
- communication between the DataNodes and the SCM.
-
- It is a string in the host:port format. The port number is optional
- and defaults to 9861.
-
- This setting is optional. If unspecified then the hostname portion
- is picked from the ozone.scm.client.address setting and the
- default service port of 9861 is chosen.
- </description>
- </property>
- <property>
- <name>ozone.scm.datanode.bind.host</name>
- <value/>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The hostname or IP address used by the SCM service endpoint to
- bind.
- </description>
- </property>
- <property>
- <name>ozone.scm.datanode.id</name>
- <value/>
- <tag>OZONE, MANAGEMENT</tag>
- <description>The path that datanodes will use to store the datanode ID.
- If this value is not set, then datanode ID is created under the
- metadata directory.
- </description>
- </property>
- <property>
- <name>ozone.scm.datanode.port</name>
- <value>9861</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The port number of the Ozone SCM service.
- </description>
- </property>
- <property>
- <name>ozone.scm.db.cache.size.mb</name>
- <value>128</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>SCM keeps track of the Containers in the cluster. This DB holds
- the container metadata. This value is set to a small value to make the
- unit
- testing runs smooth. In production, we recommend a value of 16GB or
- higher. This allows SCM to avoid disk I/O's while looking up the container
- location.
- </description>
- </property>
- <property>
- <name>ozone.scm.dead.node.interval</name>
- <value>10m</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The interval between heartbeats before a node is tagged as dead.
- </description>
- </property>
- <property>
- <name>ozone.scm.handler.count.key</name>
- <value>10</value>
- <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
- <description>
- The number of RPC handler threads for each SCM service
- endpoint.
-
- The default is appropriate for small clusters (tens of nodes).
-
- Set a value that is appropriate for the cluster size. Generally, HDFS
- recommends RPC handler count is set to 20 * log2(Cluster Size) with an
- upper limit of 200. However, SCM will not have the same amount of
- traffic as Namenode, so a value much smaller than that will work well too.
- </description>
- </property>
- <property>
- <name>ozone.scm.heartbeat.interval</name>
- <value>30s</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The heartbeat interval from a data node to SCM. Yes,
- it is not three but 30, since most data nodes will heart beating via Ratis
- heartbeats. If a client is not able to talk to a data node, it will notify
- KSM/SCM eventually. So a 30 second HB seems to work. This assumes that
- replication strategy used is Ratis if not, this value should be set to
- something smaller like 3 seconds.
- </description>
- </property>
- <property>
- <name>ozone.scm.heartbeat.log.warn.interval.count</name>
- <value>10</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- Defines how frequently we will log the missing of a heartbeat to SCM.
- For example in the default case, we will write a warning message for each
- ten consecutive heartbeats that we miss to SCM. This helps in reducing
- clutter in a data node log, but trade off is that logs will have less of
- this statement.
- </description>
- </property>
- <property>
- <name>ozone.scm.heartbeat.rpc-timeout</name>
- <value>1000</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- Timeout value for the RPC from Datanode to SCM in milliseconds.
- </description>
- </property>
- <property>
- <name>ozone.scm.heartbeat.thread.interval</name>
- <value>3s</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- When a heartbeat from the data node arrives on SCM, It is queued for
- processing with the time stamp of when the heartbeat arrived. There is a
- heartbeat processing thread inside SCM that runs at a specified interval.
- This value controls how frequently this thread is run.
-
- There are some assumptions build into SCM such as this value should allow
- the heartbeat processing thread to run at least three times more
- frequently than heartbeats and at least five times more than stale node
- detection time. If you specify a wrong value, SCM will gracefully refuse
- to run. For more info look at the node manager tests in SCM.
-
- In short, you don't need to change this.
- </description>
- </property>
- <property>
- <name>ozone.scm.http-address</name>
- <value>0.0.0.0:9876</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The address and the base port where the SCM web ui will listen on.
-
- If the port is 0 then the server will start on a free port.
- </description>
- </property>
- <property>
- <name>ozone.scm.http-bind-host</name>
- <value>0.0.0.0</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The actual address the SCM web server will bind to. If this
- optional address is set, it overrides only the hostname portion of
- ozone.scm.http-address.
- </description>
- </property>
- <property>
- <name>ozone.scm.http.enabled</name>
- <value>true</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- Property to enable or disable SCM web ui.
- </description>
- </property>
- <property>
- <name>ozone.scm.https-address</name>
- <value>0.0.0.0:9877</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The address and the base port where the SCM web UI will listen
- on using HTTPS.
-
- If the port is 0 then the server will start on a free port.
- </description>
- </property>
- <property>
- <name>ozone.scm.https-bind-host</name>
- <value>0.0.0.0</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The actual address the SCM web server will bind to using HTTPS.
- If this optional address is set, it overrides only the hostname portion of
- ozone.scm.http-address.
- </description>
- </property>
- <property>
- <name>ozone.scm.keytab.file</name>
- <value/>
- <tag>OZONE, SECURITY</tag>
- <description>
- The keytab file for Kerberos authentication in SCM.
- </description>
- </property>
- <property>
- <name>ozone.scm.max.container.report.threads</name>
- <value>100</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>
- Maximum number of threads to process container reports in scm.
- Each container report from a data node is processed by scm in a worker
- thread, fetched from a thread pool. This property is used to control the
- maximum size of the thread pool.
- </description>
- </property>
- <property>
- <name>ozone.scm.max.hb.count.to.process</name>
- <value>5000</value>
- <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
- <description>
- The maximum number of heartbeat to process per loop of the
- heartbeat process thread. Please see
- ozone.scm.heartbeat.thread.interval
- for more info.
- </description>
- </property>
- <property>
- <name>ozone.scm.max.nodepool.processing.threads</name>
- <value>1</value>
- <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
- <description>
- Number of node pools to process in parallel.
- </description>
- </property>
- <property>
- <name>ozone.scm.names</name>
- <value/>
- <tag>OZONE</tag>
- <description>
- The value of this property is a set of DNS | DNS:PORT | IP
- Address | IP:PORT. Written as a comma separated string. e.g. scm1,
- scm2:8020, 7.7.7.7:7777.
- This property allows datanodes to discover where SCM is, so that
- datanodes can send heartbeat to SCM.
- </description>
- </property>
- <property>
- <name>ozone.scm.stale.node.interval</name>
- <value>90s</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The interval for stale node flagging. Please
- see ozone.scm.heartbeat.thread.interval before changing this value.
- </description>
- </property>
- <property>
- <name>ozone.scm.max.nodepool.processing.threads</name>
- <value>1</value>
- <tag>OZONE, SCM</tag>
- <description>
- Controls the number of node pools that can be processed in parallel by
- Container Supervisor.
- </description>
- </property>
- <property>
- <name>ozone.trace.enabled</name>
- <value>false</value>
- <tag>OZONE, DEBUG</tag>
- <description>
- Setting this flag to true dumps the HTTP request/ response in
- the logs. Very useful when debugging REST protocol.
- </description>
- </property>
- <property>
- <name>ozone.web.authentication.kerberos.principal</name>
- <value/>
- <tag>OZONE, SECURITY</tag>
- <description>
- The server principal used by the SCM and KSM for web UI SPNEGO
- authentication when Kerberos security is enabled. This is typically set to
- HTTP/_HOST@REALM.TLD The SPNEGO server principal begins with the prefix
- HTTP/ by convention.
-
- If the value is '*', the web server will attempt to login with
- every principal specified in the keytab file.
- </description>
- </property>
-
- <!--Client Settings-->
- <property>
- <name>scm.container.client.idle.threshold</name>
- <value>10s</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>
- In the standalone pipelines, the SCM clients use netty to
- communicate with the container. It also uses connection pooling to
- reduce client side overheads. This allows a connection to stay idle for
- a while before the connection is closed.
- </description>
- </property>
- <property>
- <name>scm.container.client.max.size</name>
- <value>256</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>
- Controls the maximum number of connections that we cached via
- clientconnection pooling. If the number of connection
- exceed this count then the oldest idle connection is evicted.
- </description>
- </property>
-
- <property>
- <name>scm.container.client.max.outstanding.requests</name>
- <value>100</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>
- Controls the maximum number of outstanding async requests that can be
- handled by the Standalone as well as Ratis client.
- </description>
- </property>
-
- <property>
- <name>ozone.scm.container.creation.lease.timeout</name>
- <value>60s</value>
- <tag>OZONE, SCM</tag>
- <description>
- Container creation timeout in milliseconds to be used by SCM. When
- BEGIN_CREATE event happens the container is moved from ALLOCATED to
- CREATING state, SCM will now wait for the configured amount of time
- to get COMPLETE_CREATE event if it doesn't receive it will move the
- container to DELETING.
- </description>
- </property>
-
- <property>
- <name>ozone.key.preallocation.maxsize</name>
- <value>134217728</value>
- <tag>OZONE, KSM, PERFORMANCE</tag>
- <description>
- When a new key write request is sent to KSM, if a size is requested, at most
- 128MB of size is allocated at request time. If client needs more space for the
- write, separate block allocation requests will be made.
- </description>
- </property>
-
- <property>
- <name>ozone.client.list.cache</name>
- <value>1000</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>
- Configuration property to configure the cache size of client list calls.
- </description>
- </property>
-
- <property>
- <name>ozone.replication</name>
- <value>3</value>
- <tag>OZONE, CLIENT</tag>
- <description>
- Default replication value. The actual number of replications can be
- specified when writing the key. The default is used if replication
- is not specified. Supported values: 1 and 3.
- </description>
- </property>
-
- <property>
- <name>ozone.replication.type</name>
- <value>RATIS</value>
- <tag>OZONE, CLIENT</tag>
- <description>
- Default replication type to be used while writing key into ozone. The
- value can be specified when writing the key, default is used when
- nothing is specified. Supported values: RATIS, STAND_ALONE and CHAINED.
- </description>
- </property>
- <property>
- <name>ozone.scm.container.close.threshold</name>
- <value>0.9f</value>
- <tag>OZONE, SCM</tag>
- <description>
- This determines the threshold to be used for closing a container.
- When the container used percentage reaches this threshold,
- the container will be closed. Value should be a positive, non-zero
- percentage in float notation (X.Yf), with 1.0f meaning 100%.
- </description>
- </property>
- <property>
- <name>ozone.rest.client.http.connection.max</name>
- <value>100</value>
- <tag>OZONE, CLIENT</tag>
- <description>
- This defines the overall connection limit for the connection pool used in
- RestClient.
- </description>
- </property>
- <property>
- <name>ozone.rest.client.http.connection.per-route.max</name>
- <value>20</value>
- <tag>OZONE, CLIENT</tag>
- <description>
- This defines the connection limit per one HTTP route/host. Total max
- connection is limited by ozone.rest.client.http.connection.max property.
- </description>
- </property>
-
- <property>
- <name>ozone.open.key.cleanup.service.interval.seconds</name>
- <value>86400</value>
- <tag>OZONE, KSM, PERFORMANCE</tag>
- <description>
- A background job periodically checks open key entries and delete the expired ones. This entry controls the
- interval of this cleanup check.
- </description>
- </property>
-
- <property>
- <name>ozone.open.key.expire.threshold</name>
- <value>86400</value>
- <tag>OZONE, KSM, PERFORMANCE</tag>
- <description>
- Controls how long an open key operation is considered active. Specifically, if a key
- has been open longer than the value of this config entry, that open key is considered as
- expired (e.g. due to client crash). Default to 24 hours.
- </description>
- </property>
-
-
- <property>
- <name>hdsl.rest.rest-csrf.enabled</name>
- <value>false</value>
- <description>
- If true, then enables Object Store REST server protection against
- cross-site request forgery (CSRF).
- </description>
- </property>
-
- <property>
- <name>hdsl.rest.http-address</name>
- <value>0.0.0.0:9880</value>
- <description>The http address of Object Store REST server inside the
- datanode.</description>
- </property>
-
-
- <property>
- <name>hdsl.rest.netty.high.watermark</name>
- <value>65535</value>
- <description>
- High watermark configuration to Netty for Object Store REST server.
- </description>
- </property>
-
- <property>
- <name>hdsl.rest.netty.low.watermark</name>
- <value>32768</value>
- <description>
- Low watermark configuration to Netty for Object Store REST server.
- </description>
- </property>
-
-
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java b/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java
deleted file mode 100644
index cee66ca..0000000
--- a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java
+++ /dev/null
@@ -1,414 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.DFSUtilClient;
-import org.apache.hadoop.utils.BatchOperation;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter;
-import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.junit.Rule;
-import org.junit.Before;
-import org.junit.After;
-import org.junit.Test;
-import org.junit.Assert;
-import org.junit.rules.ExpectedException;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.ArrayList;
-import java.util.UUID;
-import java.util.Collection;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.junit.runners.Parameterized.*;
-
-/**
- * Test class for ozone metadata store.
- */
-@RunWith(Parameterized.class)
-public class TestMetadataStore {
-
- private final String storeImpl;
-
- public TestMetadataStore(String metadataImpl) {
- this.storeImpl = metadataImpl;
- }
-
- @Parameters
- public static Collection<Object[]> data() {
- return Arrays.asList(new Object[][] {
- {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB},
- {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB}
- });
- }
-
- private MetadataStore store;
- private File testDir;
- private final static int MAX_GETRANGE_LENGTH = 100;
-
- @Rule
- public ExpectedException expectedException = ExpectedException.none();
-
- @Before
- public void init() throws IOException {
- testDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
- + "-" + storeImpl.toLowerCase());
-
- Configuration conf = new OzoneConfiguration();
- conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
-
- store = MetadataStoreBuilder.newBuilder()
- .setConf(conf)
- .setCreateIfMissing(true)
- .setDbFile(testDir)
- .build();
-
- // Add 20 entries.
- // {a0 : a-value0} to {a9 : a-value9}
- // {b0 : b-value0} to {b9 : b-value9}
- for (int i=0; i<10; i++) {
- store.put(getBytes("a" + i), getBytes("a-value" + i));
- store.put(getBytes("b" + i), getBytes("b-value" + i));
- }
- }
-
- @After
- public void cleanup() throws IOException {
- store.close();
- store.destroy();
- FileUtils.deleteDirectory(testDir);
- }
-
- private byte[] getBytes(String str) {
- return str == null ? null :
- DFSUtilClient.string2Bytes(str);
- }
-
- private String getString(byte[] bytes) {
- return bytes == null ? null :
- DFSUtilClient.bytes2String(bytes);
- }
-
- @Test
- public void testGetDelete() throws IOException {
- for (int i=0; i<10; i++) {
- byte[] va = store.get(getBytes("a" + i));
- Assert.assertEquals("a-value" + i, getString(va));
-
- byte[] vb = store.get(getBytes("b" + i));
- Assert.assertEquals("b-value" + i, getString(vb));
- }
-
- String keyToDel = "del-" + UUID.randomUUID().toString();
- store.put(getBytes(keyToDel), getBytes(keyToDel));
- Assert.assertEquals(keyToDel, getString(store.get(getBytes(keyToDel))));
- store.delete(getBytes(keyToDel));
- Assert.assertEquals(null, store.get(getBytes(keyToDel)));
- }
-
- @Test
- public void testPeekFrom() throws IOException {
- // Test peek from an element that has prev as well as next
- testPeek("a3", "a2", "a4");
-
- // Test peek from an element that only has prev
- testPeek("b9", "b8", null);
-
- // Test peek from an element that only has next
- testPeek("a0", null, "a1");
- }
-
- private String getExpectedValue(String key) {
- if (key == null) {
- return null;
- }
- char[] arr = key.toCharArray();
- return new StringBuffer().append(arr[0]).append("-value")
- .append(arr[arr.length - 1]).toString();
- }
-
- private void testPeek(String peekKey, String prevKey, String nextKey)
- throws IOException {
- // Look for current
- String k = null;
- String v = null;
- ImmutablePair<byte[], byte[]> current =
- store.peekAround(0, getBytes(peekKey));
- if (current != null) {
- k = getString(current.getKey());
- v = getString(current.getValue());
- }
- Assert.assertEquals(peekKey, k);
- Assert.assertEquals(v, getExpectedValue(peekKey));
-
- // Look for prev
- k = null;
- v = null;
- ImmutablePair<byte[], byte[]> prev =
- store.peekAround(-1, getBytes(peekKey));
- if (prev != null) {
- k = getString(prev.getKey());
- v = getString(prev.getValue());
- }
- Assert.assertEquals(prevKey, k);
- Assert.assertEquals(v, getExpectedValue(prevKey));
-
- // Look for next
- k = null;
- v = null;
- ImmutablePair<byte[], byte[]> next =
- store.peekAround(1, getBytes(peekKey));
- if (next != null) {
- k = getString(next.getKey());
- v = getString(next.getValue());
- }
- Assert.assertEquals(nextKey, k);
- Assert.assertEquals(v, getExpectedValue(nextKey));
- }
-
- @Test
- public void testIterateKeys() throws IOException {
- // iterate keys from b0
- ArrayList<String> result = Lists.newArrayList();
- store.iterate(getBytes("b0"), (k, v) -> {
- // b-value{i}
- String value = getString(v);
- char num = value.charAt(value.length() - 1);
- // each value adds 1
- int i = Character.getNumericValue(num) + 1;
- value = value.substring(0, value.length() - 1) + i;
- result.add(value);
- return true;
- });
-
- Assert.assertFalse(result.isEmpty());
- for (int i=0; i<result.size(); i++) {
- Assert.assertEquals("b-value" + (i+1), result.get(i));
- }
-
- // iterate from a non exist key
- result.clear();
- store.iterate(getBytes("xyz"), (k, v) -> {
- result.add(getString(v));
- return true;
- });
- Assert.assertTrue(result.isEmpty());
-
- // iterate from the beginning
- result.clear();
- store.iterate(null, (k, v) -> {
- result.add(getString(v));
- return true;
- });
- Assert.assertEquals(20, result.size());
- }
-
- @Test
- public void testGetRangeKVs() throws IOException {
- List<Map.Entry<byte[], byte[]>> result = null;
-
- // Set empty startKey will return values from beginning.
- result = store.getRangeKVs(null, 5);
- Assert.assertEquals(5, result.size());
- Assert.assertEquals("a-value2", getString(result.get(2).getValue()));
-
- // Empty list if startKey doesn't exist.
- result = store.getRangeKVs(getBytes("a12"), 5);
- Assert.assertEquals(0, result.size());
-
- // Returns max available entries after a valid startKey.
- result = store.getRangeKVs(getBytes("b0"), MAX_GETRANGE_LENGTH);
- Assert.assertEquals(10, result.size());
- Assert.assertEquals("b0", getString(result.get(0).getKey()));
- Assert.assertEquals("b-value0", getString(result.get(0).getValue()));
- result = store.getRangeKVs(getBytes("b0"), 5);
- Assert.assertEquals(5, result.size());
-
- // Both startKey and count are honored.
- result = store.getRangeKVs(getBytes("a9"), 2);
- Assert.assertEquals(2, result.size());
- Assert.assertEquals("a9", getString(result.get(0).getKey()));
- Assert.assertEquals("a-value9", getString(result.get(0).getValue()));
- Assert.assertEquals("b0", getString(result.get(1).getKey()));
- Assert.assertEquals("b-value0", getString(result.get(1).getValue()));
-
- // Filter keys by prefix.
- // It should returns all "b*" entries.
- MetadataKeyFilter filter1 = new KeyPrefixFilter("b");
- result = store.getRangeKVs(null, 100, filter1);
- Assert.assertEquals(10, result.size());
- Assert.assertTrue(result.stream().allMatch(entry ->
- new String(entry.getKey()).startsWith("b")
- ));
- Assert.assertEquals(20, filter1.getKeysScannedNum());
- Assert.assertEquals(10, filter1.getKeysHintedNum());
- result = store.getRangeKVs(null, 3, filter1);
- Assert.assertEquals(3, result.size());
- result = store.getRangeKVs(getBytes("b3"), 1, filter1);
- Assert.assertEquals("b-value3", getString(result.get(0).getValue()));
-
- // Define a customized filter that filters keys by suffix.
- // Returns all "*2" entries.
- MetadataKeyFilter filter2 = (preKey, currentKey, nextKey)
- -> getString(currentKey).endsWith("2");
- result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter2);
- Assert.assertEquals(2, result.size());
- Assert.assertEquals("a2", getString(result.get(0).getKey()));
- Assert.assertEquals("b2", getString(result.get(1).getKey()));
- result = store.getRangeKVs(null, 1, filter2);
- Assert.assertEquals(1, result.size());
- Assert.assertEquals("a2", getString(result.get(0).getKey()));
-
- // Apply multiple filters.
- result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter1, filter2);
- Assert.assertEquals(1, result.size());
- Assert.assertEquals("b2", getString(result.get(0).getKey()));
- Assert.assertEquals("b-value2", getString(result.get(0).getValue()));
-
- // If filter is null, no effect.
- result = store.getRangeKVs(null, 1, null);
- Assert.assertEquals(1, result.size());
- Assert.assertEquals("a0", getString(result.get(0).getKey()));
- }
-
- @Test
- public void testGetSequentialRangeKVs() throws IOException {
- MetadataKeyFilter suffixFilter = (preKey, currentKey, nextKey)
- -> DFSUtil.bytes2String(currentKey).endsWith("2");
- // Suppose to return a2 and b2
- List<Map.Entry<byte[], byte[]>> result =
- store.getRangeKVs(null, MAX_GETRANGE_LENGTH, suffixFilter);
- Assert.assertEquals(2, result.size());
- Assert.assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey()));
- Assert.assertEquals("b2", DFSUtil.bytes2String(result.get(1).getKey()));
-
- // Suppose to return just a2, because when it iterates to a3,
- // the filter no long matches and it should stop from there.
- result = store.getSequentialRangeKVs(null,
- MAX_GETRANGE_LENGTH, suffixFilter);
- Assert.assertEquals(1, result.size());
- Assert.assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey()));
- }
-
- @Test
- public void testGetRangeLength() throws IOException {
- List<Map.Entry<byte[], byte[]>> result = null;
-
- result = store.getRangeKVs(null, 0);
- Assert.assertEquals(0, result.size());
-
- result = store.getRangeKVs(null, 1);
- Assert.assertEquals(1, result.size());
-
- // Count less than zero is invalid.
- expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage("Invalid count given");
- store.getRangeKVs(null, -1);
- }
-
- @Test
- public void testInvalidStartKey() throws IOException {
- // If startKey is invalid, the returned list should be empty.
- List<Map.Entry<byte[], byte[]>> kvs =
- store.getRangeKVs(getBytes("unknownKey"), MAX_GETRANGE_LENGTH);
- Assert.assertEquals(kvs.size(), 0);
- }
-
- @Test
- public void testDestroyDB() throws IOException {
- // create a new DB to test db destroy
- Configuration conf = new OzoneConfiguration();
- conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
-
- File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
- + "-" + storeImpl.toLowerCase() + "-toDestroy");
- MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf)
- .setCreateIfMissing(true)
- .setDbFile(dbDir)
- .build();
-
- dbStore.put(getBytes("key1"), getBytes("value1"));
- dbStore.put(getBytes("key2"), getBytes("value2"));
-
- Assert.assertFalse(dbStore.isEmpty());
- Assert.assertTrue(dbDir.exists());
- Assert.assertTrue(dbDir.listFiles().length > 0);
-
- dbStore.destroy();
-
- Assert.assertFalse(dbDir.exists());
- }
-
- @Test
- public void testBatchWrite() throws IOException {
- Configuration conf = new OzoneConfiguration();
- conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
-
- File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
- + "-" + storeImpl.toLowerCase() + "-batchWrite");
- MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf)
- .setCreateIfMissing(true)
- .setDbFile(dbDir)
- .build();
-
- List<String> expectedResult = Lists.newArrayList();
- for (int i = 0; i<10; i++) {
- dbStore.put(getBytes("batch-" + i), getBytes("batch-value-" + i));
- expectedResult.add("batch-" + i);
- }
-
- BatchOperation batch = new BatchOperation();
- batch.delete(getBytes("batch-2"));
- batch.delete(getBytes("batch-3"));
- batch.delete(getBytes("batch-4"));
- batch.put(getBytes("batch-new-2"), getBytes("batch-new-value-2"));
-
- expectedResult.remove("batch-2");
- expectedResult.remove("batch-3");
- expectedResult.remove("batch-4");
- expectedResult.add("batch-new-2");
-
- dbStore.writeBatch(batch);
-
- Iterator<String> it = expectedResult.iterator();
- AtomicInteger count = new AtomicInteger(0);
- dbStore.iterate(null, (key, value) -> {
- count.incrementAndGet();
- return it.hasNext() && it.next().equals(getString(key));
- });
-
- Assert.assertEquals(8, count.get());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java b/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java
deleted file mode 100644
index 03c45c5..0000000
--- a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone;
-
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * This class is to test acl stoarge and retreival in ozone store.
- */
-public class TestOzoneAcls {
-
- @Test
- public void testAclParse() {
- HashMap<String, Boolean> testMatrix;
- testMatrix = new HashMap<>();
-
- testMatrix.put("user:bilbo:r", Boolean.TRUE);
- testMatrix.put("user:bilbo:w", Boolean.TRUE);
- testMatrix.put("user:bilbo:rw", Boolean.TRUE);
- testMatrix.put("user:bilbo:wr", Boolean.TRUE);
- testMatrix.put(" user:bilbo:wr ", Boolean.TRUE);
-
-
- // ACLs makes no judgement on the quality of
- // user names. it is for the userAuth interface
- // to determine if a user name is really a name
- testMatrix.put(" user:*:rw", Boolean.TRUE);
- testMatrix.put(" user:~!:rw", Boolean.TRUE);
-
-
- testMatrix.put("", Boolean.FALSE);
- testMatrix.put(null, Boolean.FALSE);
- testMatrix.put(" user:bilbo:", Boolean.FALSE);
- testMatrix.put(" user:bilbo:rx", Boolean.FALSE);
- testMatrix.put(" user:bilbo:mk", Boolean.FALSE);
- testMatrix.put(" user::rw", Boolean.FALSE);
- testMatrix.put("user11:bilbo:rw", Boolean.FALSE);
- testMatrix.put(" user:::rw", Boolean.FALSE);
-
- testMatrix.put(" group:hobbit:r", Boolean.TRUE);
- testMatrix.put(" group:hobbit:w", Boolean.TRUE);
- testMatrix.put(" group:hobbit:rw", Boolean.TRUE);
- testMatrix.put(" group:hobbit:wr", Boolean.TRUE);
- testMatrix.put(" group:*:rw", Boolean.TRUE);
- testMatrix.put(" group:~!:rw", Boolean.TRUE);
-
- testMatrix.put(" group:hobbit:", Boolean.FALSE);
- testMatrix.put(" group:hobbit:rx", Boolean.FALSE);
- testMatrix.put(" group:hobbit:mk", Boolean.FALSE);
- testMatrix.put(" group::", Boolean.FALSE);
- testMatrix.put(" group::rw", Boolean.FALSE);
- testMatrix.put(" group22:hobbit:", Boolean.FALSE);
- testMatrix.put(" group:::rw", Boolean.FALSE);
-
- testMatrix.put("JUNK group:hobbit:r", Boolean.FALSE);
- testMatrix.put("JUNK group:hobbit:w", Boolean.FALSE);
- testMatrix.put("JUNK group:hobbit:rw", Boolean.FALSE);
- testMatrix.put("JUNK group:hobbit:wr", Boolean.FALSE);
- testMatrix.put("JUNK group:*:rw", Boolean.FALSE);
- testMatrix.put("JUNK group:~!:rw", Boolean.FALSE);
-
- testMatrix.put(" world::r", Boolean.TRUE);
- testMatrix.put(" world::w", Boolean.TRUE);
- testMatrix.put(" world::rw", Boolean.TRUE);
- testMatrix.put(" world::wr", Boolean.TRUE);
-
- testMatrix.put(" world:bilbo:w", Boolean.FALSE);
- testMatrix.put(" world:bilbo:rw", Boolean.FALSE);
-
- Set<String> keys = testMatrix.keySet();
- for (String key : keys) {
- if (testMatrix.get(key)) {
- OzoneAcl.parseAcl(key);
- } else {
- try {
- OzoneAcl.parseAcl(key);
- // should never get here since parseAcl will throw
- fail("An exception was expected but did not happen.");
- } catch (IllegalArgumentException e) {
- // nothing to do
- }
- }
- }
- }
-
- @Test
- public void testAclValues() {
- OzoneAcl acl = OzoneAcl.parseAcl("user:bilbo:rw");
- assertEquals(acl.getName(), "bilbo");
- assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
- assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
-
- acl = OzoneAcl.parseAcl("user:bilbo:wr");
- assertEquals("bilbo", acl.getName());
- assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
- assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
-
- acl = OzoneAcl.parseAcl("user:bilbo:r");
- assertEquals("bilbo", acl.getName());
- assertEquals(OzoneAcl.OzoneACLRights.READ, acl.getRights());
- assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
-
- acl = OzoneAcl.parseAcl("user:bilbo:w");
- assertEquals("bilbo", acl.getName());
- assertEquals(OzoneAcl.OzoneACLRights.WRITE, acl.getRights());
- assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
-
- acl = OzoneAcl.parseAcl("group:hobbit:wr");
- assertEquals(acl.getName(), "hobbit");
- assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
- assertEquals(OzoneAcl.OzoneACLType.GROUP, acl.getType());
-
- acl = OzoneAcl.parseAcl("world::wr");
- assertEquals(acl.getName(), "");
- assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
- assertEquals(OzoneAcl.OzoneACLType.WORLD, acl.getType());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java b/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java
deleted file mode 100644
index d4c626e..0000000
--- a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.common;
-
-import org.apache.commons.collections.SetUtils;
-import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException;
-import org.apache.hadoop.ozone.common.statemachine.StateMachine;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.INIT;
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CREATING;
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.OPERATIONAL;
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CLOSED;
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CLEANUP;
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.FINAL;
-
-/**
- * This class is to test ozone common state machine.
- */
-public class TestStateMachine {
-
- /**
- * STATES used by the test state machine.
- */
- public enum STATES {INIT, CREATING, OPERATIONAL, CLOSED, CLEANUP, FINAL};
-
- /**
- * EVENTS used by the test state machine.
- */
- public enum EVENTS {ALLOCATE, CREATE, UPDATE, CLOSE, DELETE, TIMEOUT};
-
- @Rule
- public ExpectedException exception = ExpectedException.none();
-
- @Test
- public void testStateMachineStates() throws InvalidStateTransitionException {
- Set<STATES> finals = new HashSet<>();
- finals.add(FINAL);
-
- StateMachine<STATES, EVENTS> stateMachine =
- new StateMachine<>(INIT, finals);
-
- stateMachine.addTransition(INIT, CREATING, EVENTS.ALLOCATE);
- stateMachine.addTransition(CREATING, OPERATIONAL, EVENTS.CREATE);
- stateMachine.addTransition(OPERATIONAL, OPERATIONAL, EVENTS.UPDATE);
- stateMachine.addTransition(OPERATIONAL, CLEANUP, EVENTS.DELETE);
- stateMachine.addTransition(OPERATIONAL, CLOSED, EVENTS.CLOSE);
- stateMachine.addTransition(CREATING, CLEANUP, EVENTS.TIMEOUT);
-
- // Initial and Final states
- Assert.assertEquals("Initial State", INIT, stateMachine.getInitialState());
- Assert.assertTrue("Final States", SetUtils.isEqualSet(finals,
- stateMachine.getFinalStates()));
-
- // Valid state transitions
- Assert.assertEquals("STATE should be OPERATIONAL after being created",
- OPERATIONAL, stateMachine.getNextState(CREATING, EVENTS.CREATE));
- Assert.assertEquals("STATE should be OPERATIONAL after being updated",
- OPERATIONAL, stateMachine.getNextState(OPERATIONAL, EVENTS.UPDATE));
- Assert.assertEquals("STATE should be CLEANUP after being deleted",
- CLEANUP, stateMachine.getNextState(OPERATIONAL, EVENTS.DELETE));
- Assert.assertEquals("STATE should be CLEANUP after being timeout",
- CLEANUP, stateMachine.getNextState(CREATING, EVENTS.TIMEOUT));
- Assert.assertEquals("STATE should be CLOSED after being closed",
- CLOSED, stateMachine.getNextState(OPERATIONAL, EVENTS.CLOSE));
-
- // Negative cases: invalid transition
- expectException();
- stateMachine.getNextState(OPERATIONAL, EVENTS.CREATE);
-
- expectException();
- stateMachine.getNextState(CREATING, EVENTS.CLOSE);
- }
-
- /**
- * We expect an InvalidStateTransitionException.
- */
- private void expectException() {
- exception.expect(InvalidStateTransitionException.class);
- exception.expectMessage("Invalid event");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java b/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java
deleted file mode 100644
index 517c1a7..0000000
--- a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java
+++ /dev/null
@@ -1,374 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-/**
- * A generic lease management API which can be used if a service
- * needs any kind of lease management.
- */
-
-package org.apache.hadoop.ozone.lease;
-
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Test class to check functionality and consistency of LeaseManager.
- */
-public class TestLeaseManager {
-
- @Rule
- public ExpectedException exception = ExpectedException.none();
-
- /**
- * Dummy resource on which leases can be acquired.
- */
- private final class DummyResource {
-
- private final String name;
-
- private DummyResource(String name) {
- this.name = name;
- }
-
- @Override
- public int hashCode() {
- return name.hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if(obj instanceof DummyResource) {
- return name.equals(((DummyResource) obj).name);
- }
- return false;
- }
- }
-
- @Test
- public void testLeaseAcquireAndRelease() throws LeaseException {
- //It is assumed that the test case execution won't take more than 5 seconds,
- //if it takes more time increase the defaultTimeout value of LeaseManager.
- LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
- manager.start();
- DummyResource resourceOne = new DummyResource("one");
- DummyResource resourceTwo = new DummyResource("two");
- DummyResource resourceThree = new DummyResource("three");
- Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
- Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo);
- Lease<DummyResource> leaseThree = manager.acquire(resourceThree);
- Assert.assertEquals(leaseOne, manager.get(resourceOne));
- Assert.assertEquals(leaseTwo, manager.get(resourceTwo));
- Assert.assertEquals(leaseThree, manager.get(resourceThree));
- Assert.assertFalse(leaseOne.hasExpired());
- Assert.assertFalse(leaseTwo.hasExpired());
- Assert.assertFalse(leaseThree.hasExpired());
- //The below releases should not throw LeaseNotFoundException.
- manager.release(resourceOne);
- manager.release(resourceTwo);
- manager.release(resourceThree);
- Assert.assertTrue(leaseOne.hasExpired());
- Assert.assertTrue(leaseTwo.hasExpired());
- Assert.assertTrue(leaseThree.hasExpired());
- manager.shutdown();
- }
-
- @Test
- public void testLeaseAlreadyExist() throws LeaseException {
- LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
- manager.start();
- DummyResource resourceOne = new DummyResource("one");
- DummyResource resourceTwo = new DummyResource("two");
- Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
- Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo);
- Assert.assertEquals(leaseOne, manager.get(resourceOne));
- Assert.assertEquals(leaseTwo, manager.get(resourceTwo));
-
- exception.expect(LeaseAlreadyExistException.class);
- exception.expectMessage("Resource: " + resourceOne);
- manager.acquire(resourceOne);
-
- manager.release(resourceOne);
- manager.release(resourceTwo);
- manager.shutdown();
- }
-
- @Test
- public void testLeaseNotFound() throws LeaseException, InterruptedException {
- LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
- manager.start();
- DummyResource resourceOne = new DummyResource("one");
- DummyResource resourceTwo = new DummyResource("two");
- DummyResource resourceThree = new DummyResource("three");
-
- //Case 1: lease was never acquired.
- exception.expect(LeaseNotFoundException.class);
- exception.expectMessage("Resource: " + resourceOne);
- manager.get(resourceOne);
-
- //Case 2: lease is acquired and released.
- Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo);
- Assert.assertEquals(leaseTwo, manager.get(resourceTwo));
- Assert.assertFalse(leaseTwo.hasExpired());
- manager.release(resourceTwo);
- Assert.assertTrue(leaseTwo.hasExpired());
- exception.expect(LeaseNotFoundException.class);
- exception.expectMessage("Resource: " + resourceTwo);
- manager.get(resourceTwo);
-
- //Case 3: lease acquired and timed out.
- Lease<DummyResource> leaseThree = manager.acquire(resourceThree);
- Assert.assertEquals(leaseThree, manager.get(resourceThree));
- Assert.assertFalse(leaseThree.hasExpired());
- long sleepTime = leaseThree.getRemainingTime() + 1000;
- try {
- Thread.sleep(sleepTime);
- } catch (InterruptedException ex) {
- //even in case of interrupt we have to wait till lease times out.
- Thread.sleep(sleepTime);
- }
- Assert.assertTrue(leaseThree.hasExpired());
- exception.expect(LeaseNotFoundException.class);
- exception.expectMessage("Resource: " + resourceThree);
- manager.get(resourceThree);
- manager.shutdown();
- }
-
- @Test
- public void testCustomLeaseTimeout() throws LeaseException {
- LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
- manager.start();
- DummyResource resourceOne = new DummyResource("one");
- DummyResource resourceTwo = new DummyResource("two");
- DummyResource resourceThree = new DummyResource("three");
- Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
- Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo, 10000);
- Lease<DummyResource> leaseThree = manager.acquire(resourceThree, 50000);
- Assert.assertEquals(leaseOne, manager.get(resourceOne));
- Assert.assertEquals(leaseTwo, manager.get(resourceTwo));
- Assert.assertEquals(leaseThree, manager.get(resourceThree));
- Assert.assertFalse(leaseOne.hasExpired());
- Assert.assertFalse(leaseTwo.hasExpired());
- Assert.assertFalse(leaseThree.hasExpired());
- Assert.assertEquals(5000, leaseOne.getLeaseLifeTime());
- Assert.assertEquals(10000, leaseTwo.getLeaseLifeTime());
- Assert.assertEquals(50000, leaseThree.getLeaseLifeTime());
- // Releasing of leases is done in shutdown, so don't have to worry about
- // lease release
- manager.shutdown();
- }
-
- @Test
- public void testLeaseCallback() throws LeaseException, InterruptedException {
- Map<DummyResource, String> leaseStatus = new HashMap<>();
- LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
- manager.start();
- DummyResource resourceOne = new DummyResource("one");
- Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
- leaseStatus.put(resourceOne, "lease in use");
- leaseOne.registerCallBack(() -> {
- leaseStatus.put(resourceOne, "lease expired");
- return null;
- });
- // wait for lease to expire
- long sleepTime = leaseOne.getRemainingTime() + 1000;
- try {
- Thread.sleep(sleepTime);
- } catch (InterruptedException ex) {
- //even in case of interrupt we have to wait till lease times out.
- Thread.sleep(sleepTime);
- }
- Assert.assertTrue(leaseOne.hasExpired());
- exception.expect(LeaseNotFoundException.class);
- exception.expectMessage("Resource: " + resourceOne);
- manager.get(resourceOne);
- // check if callback has been executed
- Assert.assertEquals("lease expired", leaseStatus.get(resourceOne));
- }
-
- @Test
- public void testCallbackExecutionInCaseOfLeaseRelease()
- throws LeaseException, InterruptedException {
- // Callbacks should not be executed in case of lease release
- Map<DummyResource, String> leaseStatus = new HashMap<>();
- LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
- manager.start();
- DummyResource resourceOne = new DummyResource("one");
- Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
- leaseStatus.put(resourceOne, "lease in use");
- leaseOne.registerCallBack(() -> {
- leaseStatus.put(resourceOne, "lease expired");
- return null;
- });
- leaseStatus.put(resourceOne, "lease released");
- manager.release(resourceOne);
- Assert.assertTrue(leaseOne.hasExpired());
- exception.expect(LeaseNotFoundException.class);
- exception.expectMessage("Resource: " + resourceOne);
- manager.get(resourceOne);
- Assert.assertEquals("lease released", leaseStatus.get(resourceOne));
- }
-
- @Test
- public void testLeaseCallbackWithMultipleLeases()
- throws LeaseException, InterruptedException {
- Map<DummyResource, String> leaseStatus = new HashMap<>();
- LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
- manager.start();
- DummyResource resourceOne = new DummyResource("one");
- DummyResource resourceTwo = new DummyResource("two");
- DummyResource resourceThree = new DummyResource("three");
- DummyResource resourceFour = new DummyResource("four");
- DummyResource resourceFive = new DummyResource("five");
- Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
- Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo);
- Lease<DummyResource> leaseThree = manager.acquire(resourceThree);
- Lease<DummyResource> leaseFour = manager.acquire(resourceFour);
- Lease<DummyResource> leaseFive = manager.acquire(resourceFive);
- leaseStatus.put(resourceOne, "lease in use");
- leaseStatus.put(resourceTwo, "lease in use");
- leaseStatus.put(resourceThree, "lease in use");
- leaseStatus.put(resourceFour, "lease in use");
- leaseStatus.put(resourceFive, "lease in use");
- leaseOne.registerCallBack(() -> {
- leaseStatus.put(resourceOne, "lease expired");
- return null;
- });
- leaseTwo.registerCallBack(() -> {
- leaseStatus.put(resourceTwo, "lease expired");
- return null;
- });
- leaseThree.registerCallBack(() -> {
- leaseStatus.put(resourceThree, "lease expired");
- return null;
- });
- leaseFour.registerCallBack(() -> {
- leaseStatus.put(resourceFour, "lease expired");
- return null;
- });
- leaseFive.registerCallBack(() -> {
- leaseStatus.put(resourceFive, "lease expired");
- return null;
- });
-
- // release lease one, two and three
- leaseStatus.put(resourceOne, "lease released");
- manager.release(resourceOne);
- leaseStatus.put(resourceTwo, "lease released");
- manager.release(resourceTwo);
- leaseStatus.put(resourceThree, "lease released");
- manager.release(resourceThree);
-
- // wait for other leases to expire
- long sleepTime = leaseFive.getRemainingTime() + 1000;
-
- try {
- Thread.sleep(sleepTime);
- } catch (InterruptedException ex) {
- //even in case of interrupt we have to wait till lease times out.
- Thread.sleep(sleepTime);
- }
- Assert.assertTrue(leaseOne.hasExpired());
- Assert.assertTrue(leaseTwo.hasExpired());
- Assert.assertTrue(leaseThree.hasExpired());
- Assert.assertTrue(leaseFour.hasExpired());
- Assert.assertTrue(leaseFive.hasExpired());
-
- Assert.assertEquals("lease released", leaseStatus.get(resourceOne));
- Assert.assertEquals("lease released", leaseStatus.get(resourceTwo));
- Assert.assertEquals("lease released", leaseStatus.get(resourceThree));
- Assert.assertEquals("lease expired", leaseStatus.get(resourceFour));
- Assert.assertEquals("lease expired", leaseStatus.get(resourceFive));
- manager.shutdown();
- }
-
- @Test
- public void testReuseReleasedLease() throws LeaseException {
- LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
- manager.start();
- DummyResource resourceOne = new DummyResource("one");
- Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
- Assert.assertEquals(leaseOne, manager.get(resourceOne));
- Assert.assertFalse(leaseOne.hasExpired());
-
- manager.release(resourceOne);
- Assert.assertTrue(leaseOne.hasExpired());
-
- Lease<DummyResource> sameResourceLease = manager.acquire(resourceOne);
- Assert.assertEquals(sameResourceLease, manager.get(resourceOne));
- Assert.assertFalse(sameResourceLease.hasExpired());
-
- manager.release(resourceOne);
- Assert.assertTrue(sameResourceLease.hasExpired());
- manager.shutdown();
- }
-
- @Test
- public void testReuseTimedOutLease()
- throws LeaseException, InterruptedException {
- LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
- manager.start();
- DummyResource resourceOne = new DummyResource("one");
- Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
- Assert.assertEquals(leaseOne, manager.get(resourceOne));
- Assert.assertFalse(leaseOne.hasExpired());
-
- // wait for lease to expire
- long sleepTime = leaseOne.getRemainingTime() + 1000;
- try {
- Thread.sleep(sleepTime);
- } catch (InterruptedException ex) {
- //even in case of interrupt we have to wait till lease times out.
- Thread.sleep(sleepTime);
- }
- Assert.assertTrue(leaseOne.hasExpired());
-
- Lease<DummyResource> sameResourceLease = manager.acquire(resourceOne);
- Assert.assertEquals(sameResourceLease, manager.get(resourceOne));
- Assert.assertFalse(sameResourceLease.hasExpired());
-
- manager.release(resourceOne);
- Assert.assertTrue(sameResourceLease.hasExpired());
- manager.shutdown();
- }
-
- @Test
- public void testRenewLease() throws LeaseException, InterruptedException {
- LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
- manager.start();
- DummyResource resourceOne = new DummyResource("one");
- Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
- Assert.assertEquals(leaseOne, manager.get(resourceOne));
- Assert.assertFalse(leaseOne.hasExpired());
-
- // add 5 more seconds to the lease
- leaseOne.renew(5000);
-
- Thread.sleep(5000);
-
- // lease should still be active
- Assert.assertEquals(leaseOne, manager.get(resourceOne));
- Assert.assertFalse(leaseOne.hasExpired());
- manager.release(resourceOne);
- manager.shutdown();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java b/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java
deleted file mode 100644
index 1071309..0000000
--- a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.lease;
-/*
- This package contains lease management unit test classes.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/package-info.java b/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/package-info.java
deleted file mode 100644
index 0030d2e..0000000
--- a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone;
-/**
- * Ozone related test helper classes and tests of common utils.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/common/src/test/java/org/apache/hadoop/scm/TestArchive.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/scm/TestArchive.java b/hadoop-hdsl/common/src/test/java/org/apache/hadoop/scm/TestArchive.java
deleted file mode 100644
index 86adddb..0000000
--- a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/scm/TestArchive.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.fs.FileUtil;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Iterator;
-import java.util.Random;
-import java.util.zip.Adler32;
-import java.util.zip.Checksum;
-
-/**
- * Test archive creation and unpacking.
- */
-public class TestArchive {
- private static final int DIR_COUNT = 10;
- private static final int SUB_DIR_COUNT = 3;
- private static final int FILE_COUNT = 10;
- private long checksumWrite = 0L;
- private long checksumRead = 0L;
- private long tmp = 0L;
- private Checksum crc = new Adler32();
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- @Rule
- public TemporaryFolder outputFolder = new TemporaryFolder();
-
-
- @Before
- public void setUp() throws Exception {
- Random r = new Random();
- final int megaByte = 1024 * 1024;
-
- for (int x = 0; x < DIR_COUNT; x++) {
- File subdir = folder.newFolder(String.format("dir%d", x));
- for (int y = 0; y < SUB_DIR_COUNT; y++) {
- File targetDir = new File(subdir.getPath().concat(File.separator)
- .concat(String.format("subdir%d%d", x, y)));
- if(!targetDir.mkdirs()) {
- throw new IOException("Failed to create subdirectory. " +
- targetDir.toString());
- }
- for (int z = 0; z < FILE_COUNT; z++) {
- Path temp = Paths.get(targetDir.getPath().concat(File.separator)
- .concat(String.format("File%d.txt", z)));
- byte[] buf = RandomUtils.nextBytes(r.nextInt(megaByte));
- Files.write(temp, buf);
- crc.reset();
- crc.update(buf, 0, buf.length);
- tmp = crc.getValue();
- checksumWrite +=tmp;
- }
- }
- }
- }
-
- @Test
- public void testArchive() throws Exception {
- File archiveFile = new File(outputFolder.getRoot() + File.separator
- + "test.container.zip");
- long zipCheckSum = FileUtil.zip(folder.getRoot(), archiveFile);
- Assert.assertTrue(zipCheckSum > 0);
- File decomp = new File(outputFolder.getRoot() + File.separator +
- "decompress");
- if (!decomp.exists() && !decomp.mkdirs()) {
- throw new IOException("Unable to create the destination directory. " +
- decomp.getPath());
- }
-
- FileUtil.unZip(archiveFile, decomp);
- String[] patterns = {"txt"};
- Iterator<File> iter = FileUtils.iterateFiles(decomp, patterns, true);
- int count = 0;
- while (iter.hasNext()) {
- count++;
- byte[] buf = Files.readAllBytes(iter.next().toPath());
- crc.reset();
- crc.update(buf, 0, buf.length);
- tmp = crc.getValue();
- checksumRead += tmp;
- }
- Assert.assertEquals(DIR_COUNT * SUB_DIR_COUNT * FILE_COUNT, count);
- Assert.assertEquals(checksumWrite, checksumRead);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/common/src/test/java/org/apache/hadoop/scm/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/scm/package-info.java b/hadoop-hdsl/common/src/test/java/org/apache/hadoop/scm/package-info.java
deleted file mode 100644
index 9c480d6..0000000
--- a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/scm/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm;
-/**
- Test cases for SCM client classes.
- */
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org