You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2018/03/15 16:32:16 UTC

[01/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7240 a5dfae69f -> ce23d9adf


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java
deleted file mode 100644
index 8cb57d6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.util;
-
-import org.apache.hadoop.cblock.meta.ContainerDescriptor;
-import org.apache.hadoop.ozone.container.ContainerTestHelper;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * NOTE : This class is only for testing purpose.
- *
- * Mock an underlying container storage layer, expose to CBlock to perform
- * IO. While in this mock implementation, a container is nothing more than
- * a in memory hashmap.
- *
- * This is to allow volume creation call and perform standalone tests.
- */
-public final class ContainerLookUpService {
-  private static ConcurrentHashMap<String, ContainerDescriptor>
-      containers = new ConcurrentHashMap<>();
-
-  /**
-   * Return an *existing* container with given Id.
-   *
-   * TODO : for testing purpose, return a new container if the given Id
-   * is not found
-   *
-   * found
-   * @param containerID
-   * @return the corresponding pipeline instance (create if not exist)
-   */
-  public static ContainerDescriptor lookUp(String containerID)
-      throws IOException {
-    if (!containers.containsKey(containerID)) {
-      Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline(
-          containerID);
-      ContainerDescriptor cd = new ContainerDescriptor(containerID);
-      cd.setPipeline(pipeline);
-      containers.put(containerID, cd);
-    }
-    return containers.get(containerID);
-  }
-
-  public static void addContainer(String containerID) throws IOException {
-    Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline(
-        containerID);
-    ContainerDescriptor cd = new ContainerDescriptor(containerID);
-    cd.setPipeline(pipeline);
-    containers.put(containerID, cd);
-  }
-
-  private ContainerLookUpService() {
-
-  }
-}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[08/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-nvd3-1.0.9.min.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-nvd3-1.0.9.min.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-nvd3-1.0.9.min.js
deleted file mode 100644
index 4aced57..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-nvd3-1.0.9.min.js
+++ /dev/null
@@ -1 +0,0 @@
-!function(window){"use strict";var nv=window.nv;"undefined"!=typeof exports&&(nv=require("nvd3")),angular.module("nvd3",[]).directive("nvd3",["nvd3Utils",function(nvd3Utils){return{restrict:"AE",scope:{data:"=",options:"=",api:"=?",events:"=?",config:"=?",onReady:"&?"},link:function(scope,element,attrs){function configure(chart,options,chartType){chart&&options&&angular.forEach(chart,function(value,key){"_"===key[0]||("dispatch"===key?(void 0!==options[key]&&null!==options[key]||scope._config.extended&&(options[key]={}),configureEvents(value,options[key])):"tooltip"===key?(void 0!==options[key]&&null!==options[key]||scope._config.extended&&(options[key]={}),configure(chart[key],options[key],chartType)):"contentGenerator"===key?options[key]&&chart[key](options[key]):-1===["axis","clearHighlights","defined","highlightPoint","nvPointerEventsClass","options","rangeBand","rangeBands","scatter","open","close","node"].indexOf(key)&&(void 0===options[key]||null===options[key]?scope._config.
 extended&&(options[key]=value()):chart[key](options[key])))})}function configureEvents(dispatch,options){dispatch&&options&&angular.forEach(dispatch,function(value,key){void 0===options[key]||null===options[key]?scope._config.extended&&(options[key]=value.on):dispatch.on(key+"._",options[key])})}function configureWrapper(name){var _=nvd3Utils.deepExtend(defaultWrapper(name),scope.options[name]||{});scope._config.extended&&(scope.options[name]=_);var wrapElement=angular.element("<div></div>").html(_.html||"").addClass(name).addClass(_.className).removeAttr("style").css(_.css);_.html||wrapElement.text(_.text),_.enable&&("title"===name?element.prepend(wrapElement):"subtitle"===name?angular.element(element[0].querySelector(".title")).after(wrapElement):"caption"===name&&element.append(wrapElement))}function configureStyles(){var _=nvd3Utils.deepExtend(defaultStyles(),scope.options.styles||{});scope._config.extended&&(scope.options.styles=_),angular.forEach(_.classes,function(value,key){
 value?element.addClass(key):element.removeClass(key)}),element.removeAttr("style").css(_.css)}function defaultWrapper(_){switch(_){case"title":return{enable:!1,text:"Write Your Title",className:"h4",css:{width:scope.options.chart.width+"px",textAlign:"center"}};case"subtitle":return{enable:!1,text:"Write Your Subtitle",css:{width:scope.options.chart.width+"px",textAlign:"center"}};case"caption":return{enable:!1,text:"Figure 1. Write Your Caption text.",css:{width:scope.options.chart.width+"px",textAlign:"center"}}}}function defaultStyles(){return{classes:{"with-3d-shadow":!0,"with-transitions":!0,gallery:!1},css:{}}}function dataWatchFn(newData,oldData){newData!==oldData&&(scope._config.disabled||(scope._config.refreshDataOnly?scope.api.update():scope.api.refresh()))}var defaultConfig={extended:!1,visible:!0,disabled:!1,refreshDataOnly:!0,deepWatchOptions:!0,deepWatchData:!0,deepWatchDataDepth:2,debounce:10,debounceImmediate:!0};scope.isReady=!1,scope._config=angular.extend(defaultC
 onfig,scope.config),scope.api={refresh:function(){scope.api.updateWithOptions(),scope.isReady=!0},refreshWithTimeout:function(t){setTimeout(function(){scope.api.refresh()},t)},update:function(){scope.chart&&scope.svg?"sunburstChart"===scope.options.chart.type?scope.svg.datum(angular.copy(scope.data)).call(scope.chart):scope.svg.datum(scope.data).call(scope.chart):scope.api.refresh()},updateWithTimeout:function(t){setTimeout(function(){scope.api.update()},t)},updateWithOptions:function(options){if(arguments.length){if(scope.options=options,scope._config.deepWatchOptions&&!scope._config.disabled)return}else options=scope.options;scope.api.clearElement(),angular.isDefined(options)!==!1&&scope._config.visible&&(scope.chart=nv.models[options.chart.type](),scope.chart.id=Math.random().toString(36).substr(2,15),angular.forEach(scope.chart,function(value,key){"_"===key[0]||["clearHighlights","highlightPoint","id","options","resizeHandler","state","open","close","tooltipContent"].indexOf(key
 )>=0||("dispatch"===key?(void 0!==options.chart[key]&&null!==options.chart[key]||scope._config.extended&&(options.chart[key]={}),configureEvents(scope.chart[key],options.chart[key])):["bars","bars1","bars2","boxplot","bullet","controls","discretebar","distX","distY","focus","interactiveLayer","legend","lines","lines1","lines2","multibar","pie","scatter","scatters1","scatters2","sparkline","stack1","stack2","sunburst","tooltip","x2Axis","xAxis","y1Axis","y2Axis","y3Axis","y4Axis","yAxis","yAxis1","yAxis2"].indexOf(key)>=0||"stacked"===key&&"stackedAreaChart"===options.chart.type?(void 0!==options.chart[key]&&null!==options.chart[key]||scope._config.extended&&(options.chart[key]={}),configure(scope.chart[key],options.chart[key],options.chart.type)):"focusHeight"===key&&"lineChart"===options.chart.type||"focusHeight"===key&&"lineWithFocusChart"===options.chart.type||("xTickFormat"!==key&&"yTickFormat"!==key||"lineWithFocusChart"!==options.chart.type)&&("tooltips"===key&&"boxPlotChart"=
 ==options.chart.type||("tooltipXContent"!==key&&"tooltipYContent"!==key||"scatterChart"!==options.chart.type)&&("x"!==key&&"y"!==key||"forceDirectedGraph"!==options.chart.type)&&(void 0===options.chart[key]||null===options.chart[key]?scope._config.extended&&("barColor"===key?options.chart[key]=value()():options.chart[key]=value()):scope.chart[key](options.chart[key]))))}),scope.api.updateWithData(),(options.title||scope._config.extended)&&configureWrapper("title"),(options.subtitle||scope._config.extended)&&configureWrapper("subtitle"),(options.caption||scope._config.extended)&&configureWrapper("caption"),(options.styles||scope._config.extended)&&configureStyles(),nv.addGraph(function(){return scope.chart?(scope.chart.resizeHandler&&scope.chart.resizeHandler.clear(),scope.chart.resizeHandler=nv.utils.windowResize(function(){scope.chart&&scope.chart.update&&scope.chart.update()}),void 0!==options.chart.zoom&&["scatterChart","lineChart","candlestickBarChart","cumulativeLineChart","his
 toricalBarChart","ohlcBarChart","stackedAreaChart"].indexOf(options.chart.type)>-1&&nvd3Utils.zoom(scope,options),scope.chart):void 0},options.chart.callback))},updateWithData:function(data){if(arguments.length){if(scope.data=data,scope._config.deepWatchData&&!scope._config.disabled)return}else data="sunburstChart"===scope.options.chart.type?angular.copy(scope.data):scope.data;if(data){d3.select(element[0]).select("svg").remove();var h,w;scope.svg=d3.select(element[0]).insert("svg",".caption"),(h=scope.options.chart.height)&&(isNaN(+h)||(h+="px"),scope.svg.attr("height",h).style({height:h})),(w=scope.options.chart.width)?(isNaN(+w)||(w+="px"),scope.svg.attr("width",w).style({width:w})):scope.svg.attr("width","100%").style({width:"100%"}),scope.svg.datum(data).call(scope.chart),scope.chart&&scope.chart.zoomRender&&scope.chart.zoomRender()}},clearElement:function(){if(element.find(".title").remove(),element.find(".subtitle").remove(),element.find(".caption").remove(),element.empty(),s
 cope.chart&&scope.chart.tooltip&&scope.chart.tooltip.id&&d3.select("#"+scope.chart.tooltip.id()).remove(),nv.graphs&&scope.chart)for(var i=nv.graphs.length-1;i>=0;i--)nv.graphs[i]&&nv.graphs[i].id===scope.chart.id&&nv.graphs.splice(i,1);nv.tooltip&&nv.tooltip.cleanup&&nv.tooltip.cleanup(),scope.chart&&scope.chart.resizeHandler&&scope.chart.resizeHandler.clear(),scope.chart=null},getScope:function(){return scope},getElement:function(){return element}},scope._config.deepWatchOptions&&scope.$watch("options",nvd3Utils.debounce(function(newOptions){scope._config.disabled||scope.api.refresh()},scope._config.debounce,scope._config.debounceImmediate),!0),scope._config.deepWatchData&&(1===scope._config.deepWatchDataDepth?scope.$watchCollection("data",dataWatchFn):scope.$watch("data",dataWatchFn,2===scope._config.deepWatchDataDepth)),scope.$watch("config",function(newConfig,oldConfig){newConfig!==oldConfig&&(scope._config=angular.extend(defaultConfig,newConfig),scope.api.refresh())},!0),scope
 ._config.deepWatchOptions||scope._config.deepWatchData||scope.api.refresh(),angular.forEach(scope.events,function(eventHandler,event){scope.$on(event,function(e,args){return eventHandler(e,scope,args)})}),element.on("$destroy",function(){scope.api.clearElement()}),scope.$watch("isReady",function(isReady){isReady&&scope.onReady&&"function"==typeof scope.onReady()&&scope.onReady()(scope,element)})}}}]).factory("nvd3Utils",function(){return{debounce:function(func,wait,immediate){var timeout;return function(){var context=this,args=arguments,later=function(){timeout=null,immediate||func.apply(context,args)},callNow=immediate&&!timeout;clearTimeout(timeout),timeout=setTimeout(later,wait),callNow&&func.apply(context,args)}},deepExtend:function(dst){var me=this;return angular.forEach(arguments,function(obj){obj!==dst&&angular.forEach(obj,function(value,key){dst[key]&&dst[key].constructor&&dst[key].constructor===Object?me.deepExtend(dst[key],value):dst[key]=value})}),dst},zoom:function(scope
 ,options){var zoom=options.chart.zoom,enabled="undefined"==typeof zoom.enabled||null===zoom.enabled?!0:zoom.enabled;if(enabled){var fixDomain,d3zoom,zoomed,unzoomed,zoomend,xScale=scope.chart.xAxis.scale(),yScale=scope.chart.yAxis.scale(),xDomain=scope.chart.xDomain||xScale.domain,yDomain=scope.chart.yDomain||yScale.domain,x_boundary=xScale.domain().slice(),y_boundary=yScale.domain().slice(),scale=zoom.scale||1,translate=zoom.translate||[0,0],scaleExtent=zoom.scaleExtent||[1,10],useFixedDomain=zoom.useFixedDomain||!1,useNiceScale=zoom.useNiceScale||!1,horizontalOff=zoom.horizontalOff||!1,verticalOff=zoom.verticalOff||!1,unzoomEventType=zoom.unzoomEventType||"dblclick.zoom";useNiceScale&&(xScale.nice(),yScale.nice()),fixDomain=function(domain,boundary){return domain[0]=Math.min(Math.max(domain[0],boundary[0]),boundary[1]-boundary[1]/scaleExtent[1]),domain[1]=Math.max(boundary[0]+boundary[1]/scaleExtent[1],Math.min(domain[1],boundary[1])),domain},zoomed=function(){if(void 0!==zoom.zoo
 med){var domains=zoom.zoomed(xScale.domain(),yScale.domain());horizontalOff||xDomain([domains.x1,domains.x2]),verticalOff||yDomain([domains.y1,domains.y2])}else horizontalOff||xDomain(useFixedDomain?fixDomain(xScale.domain(),x_boundary):xScale.domain()),verticalOff||yDomain(useFixedDomain?fixDomain(yScale.domain(),y_boundary):yScale.domain());scope.chart&&scope.chart.update()},unzoomed=function(){if(void 0!==zoom.unzoomed){var domains=zoom.unzoomed(xScale.domain(),yScale.domain());horizontalOff||xDomain([domains.x1,domains.x2]),verticalOff||yDomain([domains.y1,domains.y2])}else horizontalOff||xDomain(x_boundary),verticalOff||yDomain(y_boundary);d3zoom.scale(scale).translate(translate),scope.chart&&scope.chart.update()},zoomend=function(){void 0!==zoom.zoomend&&zoom.zoomend()},d3zoom=d3.behavior.zoom().x(xScale).y(yScale).scaleExtent(scaleExtent).on("zoom",zoomed).on("zoomend",zoomend),scope.svg&&(scope.svg.call(d3zoom),d3zoom.scale(scale).translate(translate).event(scope.svg),"none"
 !==unzoomEventType&&scope.svg.on(unzoomEventType,unzoomed)),scope.chart&&(scope.chart.zoomRender=function(){d3zoom.scale(scale).translate(translate),xScale=scope.chart.xAxis.scale(),yScale=scope.chart.yAxis.scale(),xDomain=scope.chart.xDomain||xScale.domain,yDomain=scope.chart.yDomain||yScale.domain,x_boundary=xScale.domain().slice(),y_boundary=yScale.domain().slice(),d3zoom.x(xScale).y(yScale),scope.svg.call(d3zoom),"none"!==unzoomEventType&&scope.svg.on(unzoomEventType,unzoomed)})}}}})}(window);
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-route-1.6.4.min.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-route-1.6.4.min.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-route-1.6.4.min.js
deleted file mode 100644
index 3f985d1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-route-1.6.4.min.js
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
- AngularJS v1.6.4
- (c) 2010-2017 Google, Inc. http://angularjs.org
- License: MIT
-*/
-(function(J,d){'use strict';function A(d){k&&d.get("$route")}function B(t,u,g){return{restrict:"ECA",terminal:!0,priority:400,transclude:"element",link:function(a,f,b,c,m){function v(){l&&(g.cancel(l),l=null);n&&(n.$destroy(),n=null);p&&(l=g.leave(p),l.done(function(a){!1!==a&&(l=null)}),p=null)}function E(){var b=t.current&&t.current.locals;if(d.isDefined(b&&b.$template)){var b=a.$new(),c=t.current;p=m(b,function(b){g.enter(b,null,p||f).done(function(b){!1===b||!d.isDefined(w)||w&&!a.$eval(w)||u()});
-v()});n=c.scope=b;n.$emit("$viewContentLoaded");n.$eval(k)}else v()}var n,p,l,w=b.autoscroll,k=b.onload||"";a.$on("$routeChangeSuccess",E);E()}}}function C(d,k,g){return{restrict:"ECA",priority:-400,link:function(a,f){var b=g.current,c=b.locals;f.html(c.$template);var m=d(f.contents());if(b.controller){c.$scope=a;var v=k(b.controller,c);b.controllerAs&&(a[b.controllerAs]=v);f.data("$ngControllerController",v);f.children().data("$ngControllerController",v)}a[b.resolveAs||"$resolve"]=c;m(a)}}}var x,
-y,F,G,z=d.module("ngRoute",[]).info({angularVersion:"1.6.4"}).provider("$route",function(){function t(a,f){return d.extend(Object.create(a),f)}function u(a,d){var b=d.caseInsensitiveMatch,c={originalPath:a,regexp:a},g=c.keys=[];a=a.replace(/([().])/g,"\\$1").replace(/(\/)?:(\w+)(\*\?|[?*])?/g,function(a,b,d,c){a="?"===c||"*?"===c?"?":null;c="*"===c||"*?"===c?"*":null;g.push({name:d,optional:!!a});b=b||"";return""+(a?"":b)+"(?:"+(a?b:"")+(c&&"(.+?)"||"([^/]+)")+(a||"")+")"+(a||"")}).replace(/([/$*])/g,
-"\\$1");c.regexp=new RegExp("^"+a+"$",b?"i":"");return c}x=d.isArray;y=d.isObject;F=d.isDefined;G=d.noop;var g={};this.when=function(a,f){var b;b=void 0;if(x(f)){b=b||[];for(var c=0,m=f.length;c<m;c++)b[c]=f[c]}else if(y(f))for(c in b=b||{},f)if("$"!==c.charAt(0)||"$"!==c.charAt(1))b[c]=f[c];b=b||f;d.isUndefined(b.reloadOnSearch)&&(b.reloadOnSearch=!0);d.isUndefined(b.caseInsensitiveMatch)&&(b.caseInsensitiveMatch=this.caseInsensitiveMatch);g[a]=d.extend(b,a&&u(a,b));a&&(c="/"===a[a.length-1]?a.substr(0,
-a.length-1):a+"/",g[c]=d.extend({redirectTo:a},u(c,b)));return this};this.caseInsensitiveMatch=!1;this.otherwise=function(a){"string"===typeof a&&(a={redirectTo:a});this.when(null,a);return this};k=!0;this.eagerInstantiationEnabled=function(a){return F(a)?(k=a,this):k};this.$get=["$rootScope","$location","$routeParams","$q","$injector","$templateRequest","$sce","$browser",function(a,f,b,c,m,k,u,n){function p(e){var h=q.current;(y=(s=C())&&h&&s.$$route===h.$$route&&d.equals(s.pathParams,h.pathParams)&&
-!s.reloadOnSearch&&!D)||!h&&!s||a.$broadcast("$routeChangeStart",s,h).defaultPrevented&&e&&e.preventDefault()}function l(){var e=q.current,h=s;if(y)e.params=h.params,d.copy(e.params,b),a.$broadcast("$routeUpdate",e);else if(h||e){D=!1;q.current=h;var H=c.resolve(h);n.$$incOutstandingRequestCount();H.then(w).then(z).then(function(c){return c&&H.then(A).then(function(c){h===q.current&&(h&&(h.locals=c,d.copy(h.params,b)),a.$broadcast("$routeChangeSuccess",h,e))})}).catch(function(b){h===q.current&&a.$broadcast("$routeChangeError",
-h,e,b)}).finally(function(){n.$$completeOutstandingRequest(G)})}}function w(e){var a={route:e,hasRedirection:!1};if(e)if(e.redirectTo)if(d.isString(e.redirectTo))a.path=x(e.redirectTo,e.params),a.search=e.params,a.hasRedirection=!0;else{var b=f.path(),g=f.search();e=e.redirectTo(e.pathParams,b,g);d.isDefined(e)&&(a.url=e,a.hasRedirection=!0)}else if(e.resolveRedirectTo)return c.resolve(m.invoke(e.resolveRedirectTo)).then(function(e){d.isDefined(e)&&(a.url=e,a.hasRedirection=!0);return a});return a}
-function z(a){var b=!0;if(a.route!==q.current)b=!1;else if(a.hasRedirection){var d=f.url(),c=a.url;c?f.url(c).replace():c=f.path(a.path).search(a.search).replace().url();c!==d&&(b=!1)}return b}function A(a){if(a){var b=d.extend({},a.resolve);d.forEach(b,function(a,e){b[e]=d.isString(a)?m.get(a):m.invoke(a,null,null,e)});a=B(a);d.isDefined(a)&&(b.$template=a);return c.all(b)}}function B(a){var b,c;d.isDefined(b=a.template)?d.isFunction(b)&&(b=b(a.params)):d.isDefined(c=a.templateUrl)&&(d.isFunction(c)&&
-(c=c(a.params)),d.isDefined(c)&&(a.loadedTemplateUrl=u.valueOf(c),b=k(c)));return b}function C(){var a,b;d.forEach(g,function(c,g){var r;if(r=!b){var k=f.path();r=c.keys;var m={};if(c.regexp)if(k=c.regexp.exec(k)){for(var l=1,n=k.length;l<n;++l){var p=r[l-1],q=k[l];p&&q&&(m[p.name]=q)}r=m}else r=null;else r=null;r=a=r}r&&(b=t(c,{params:d.extend({},f.search(),a),pathParams:a}),b.$$route=c)});return b||g[null]&&t(g[null],{params:{},pathParams:{}})}function x(a,b){var c=[];d.forEach((a||"").split(":"),
-function(a,d){if(0===d)c.push(a);else{var e=a.match(/(\w+)(?:[?*])?(.*)/),f=e[1];c.push(b[f]);c.push(e[2]||"");delete b[f]}});return c.join("")}var D=!1,s,y,q={routes:g,reload:function(){D=!0;var b={defaultPrevented:!1,preventDefault:function(){this.defaultPrevented=!0;D=!1}};a.$evalAsync(function(){p(b);b.defaultPrevented||l()})},updateParams:function(a){if(this.current&&this.current.$$route)a=d.extend({},this.current.params,a),f.path(x(this.current.$$route.originalPath,a)),f.search(a);else throw I("norout");
-}};a.$on("$locationChangeStart",p);a.$on("$locationChangeSuccess",l);return q}]}).run(A),I=d.$$minErr("ngRoute"),k;A.$inject=["$injector"];z.provider("$routeParams",function(){this.$get=function(){return{}}});z.directive("ngView",B);z.directive("ngView",C);B.$inject=["$route","$anchorScroll","$animate"];C.$inject=["$compile","$controller","$route"]})(window,window.angular);
-//# sourceMappingURL=angular-route.min.js.map


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[17/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
deleted file mode 100644
index 84741f4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
+++ /dev/null
@@ -1,642 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.client;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-import org.apache.hadoop.ozone.web.response.ListKeys;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.HttpDelete;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.FileEntity;
-import org.apache.http.entity.InputStreamEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Strings;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.util.LinkedList;
-import java.util.List;
-
-import static java.net.HttpURLConnection.HTTP_CREATED;
-import static java.net.HttpURLConnection.HTTP_OK;
-import static org.apache.hadoop.ozone.web.utils.OzoneUtils.ENCODING;
-import static org.apache.hadoop.ozone.web.utils.OzoneUtils.ENCODING_NAME;
-
-/**
- * A Bucket class the represents an Ozone Bucket.
- */
-public class OzoneBucket {
-  static final Logger LOG = LoggerFactory.getLogger(OzoneBucket.class);
-
-  private BucketInfo bucketInfo;
-  private OzoneVolume volume;
-
-  /**
-   * Constructor for bucket.
-   *
-   * @param info   - BucketInfo
-   * @param volume - OzoneVolume Object that contains this bucket
-   */
-  public OzoneBucket(BucketInfo info, OzoneVolume volume) {
-    this.bucketInfo = info;
-    this.volume = volume;
-  }
-
-  /**
-   * Gets bucket Info.
-   *
-   * @return BucketInfo
-   */
-  public BucketInfo getBucketInfo() {
-    return bucketInfo;
-  }
-
-  /**
-   * Sets Bucket Info.
-   *
-   * @param bucketInfo BucketInfo
-   */
-  public void setBucketInfo(BucketInfo bucketInfo) {
-    this.bucketInfo = bucketInfo;
-  }
-
-  /**
-   * Returns the parent volume class.
-   *
-   * @return - OzoneVolume
-   */
-  OzoneVolume getVolume() {
-    return volume;
-  }
-
-  /**
-   * Returns bucket name.
-   *
-   * @return Bucket Name
-   */
-  public String getBucketName() {
-    return bucketInfo.getBucketName();
-  }
-
-  /**
-   * Returns the Acls on the bucket.
-   *
-   * @return - Acls
-   */
-  public List<OzoneAcl> getAcls() {
-    return bucketInfo.getAcls();
-  }
-
-  /**
-   * Return versioning info on the bucket - Enabled or disabled.
-   *
-   * @return - Version Enum
-   */
-  public OzoneConsts.Versioning getVersioning() {
-    return bucketInfo.getVersioning();
-  }
-
-  /**
-   * Gets the Storage class for the bucket.
-   *
-   * @return Storage Class Enum
-   */
-  public StorageType getStorageType() {
-    return bucketInfo.getStorageType();
-  }
-
-  /**
-   * Gets the creation time of the bucket.
-   *
-   * @return String
-   */
-  public String getCreatedOn() {
-    return bucketInfo.getCreatedOn();
-  }
-
-  /**
-   * Puts an Object in Ozone bucket.
-   *
-   * @param keyName - Name of the key
-   * @param data    - Data that you want to put
-   * @throws OzoneException
-   */
-  public void putKey(String keyName, String data) throws OzoneException {
-    if ((keyName == null) || keyName.isEmpty()) {
-      throw new OzoneRestClientException("Invalid key Name.");
-    }
-
-    if (data == null) {
-      throw new OzoneRestClientException("Invalid data.");
-    }
-
-    HttpPut putRequest = null;
-    InputStream is = null;
-    try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
-      URIBuilder builder = new URIBuilder(volume.getClient().getEndPointURI());
-      builder.setPath("/" + getVolume().getVolumeName() + "/" + getBucketName()
-          + "/" + keyName).build();
-
-      putRequest = getVolume().getClient().getHttpPut(builder.toString());
-
-      is = new ByteArrayInputStream(data.getBytes(ENCODING));
-      putRequest.setEntity(new InputStreamEntity(is, data.length()));
-      is.mark(data.length());
-      try {
-        putRequest.setHeader(Header.CONTENT_MD5, DigestUtils.md5Hex(is));
-      } finally {
-        is.reset();
-      }
-      executePutKey(putRequest, httpClient);
-    } catch (IOException | URISyntaxException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      IOUtils.closeStream(is);
-      OzoneClientUtils.releaseConnection(putRequest);
-    }
-  }
-
-  /**
-   * Puts an Object in Ozone Bucket.
-   *
-   * @param dataFile - File from which you want the data to be put. Key Name
-   *                 will same as the file name, devoid of any path.
-   * @throws OzoneException
-   */
-  public void putKey(File dataFile) throws OzoneException {
-    if (dataFile == null) {
-      throw new OzoneRestClientException("Invalid file object.");
-    }
-    String keyName = dataFile.getName();
-    putKey(keyName, dataFile);
-  }
-
-  /**
-   * Puts a Key in Ozone Bucket.
-   *
-   * @param keyName - Name of the Key
-   * @param file    - Stream that gets read to be put into Ozone.
-   * @throws OzoneException
-   */
-  public void putKey(String keyName, File file)
-      throws OzoneException {
-
-    if ((keyName == null) || keyName.isEmpty()) {
-      throw new OzoneRestClientException("Invalid key Name");
-    }
-
-    if (file == null) {
-      throw new OzoneRestClientException("Invalid data stream");
-    }
-
-    HttpPut putRequest = null;
-    FileInputStream fis = null;
-    try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
-      URIBuilder builder = new URIBuilder(volume.getClient().getEndPointURI());
-      builder.setPath("/" + getVolume().getVolumeName() + "/" + getBucketName()
-          + "/" + keyName).build();
-
-      putRequest = getVolume().getClient().getHttpPut(builder.toString());
-
-      FileEntity fileEntity = new FileEntity(file, ContentType
-          .APPLICATION_OCTET_STREAM);
-      putRequest.setEntity(fileEntity);
-
-      fis = new FileInputStream(file);
-      putRequest.setHeader(Header.CONTENT_MD5, DigestUtils.md5Hex(fis));
-      executePutKey(putRequest, httpClient);
-
-    } catch (IOException | URISyntaxException ex) {
-      final OzoneRestClientException orce = new OzoneRestClientException(
-          "Failed to putKey: keyName=" + keyName + ", file=" + file);
-      orce.initCause(ex);
-      LOG.trace("", orce);
-      throw orce;
-    } finally {
-      IOUtils.closeStream(fis);
-      OzoneClientUtils.releaseConnection(putRequest);
-    }
-  }
-
-  /**
-   * executePutKey executes the Put request against the Ozone Server.
-   *
-   * @param putRequest - Http Put Request
-   * @param httpClient - httpClient
-   * @throws OzoneException
-   * @throws IOException
-   */
-  public static void executePutKey(HttpPut putRequest,
-      CloseableHttpClient httpClient) throws OzoneException, IOException {
-    HttpEntity entity = null;
-    try {
-      HttpResponse response = httpClient.execute(putRequest);
-      int errorCode = response.getStatusLine().getStatusCode();
-      entity = response.getEntity();
-
-      if ((errorCode == HTTP_OK) || (errorCode == HTTP_CREATED)) {
-        return;
-      }
-
-      if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
-      }
-
-      throw OzoneException.parse(EntityUtils.toString(entity));
-    } finally {
-      if (entity != null) {
-        EntityUtils.consumeQuietly(entity);
-      }
-    }
-  }
-
-  /**
-   * Gets a key from the Ozone server and writes to the file pointed by the
-   * downloadTo PAth.
-   *
-   * @param keyName    - Key Name in Ozone.
-   * @param downloadTo File Name to download the Key's Data to
-   */
-  public void getKey(String keyName, Path downloadTo) throws OzoneException {
-
-    if ((keyName == null) || keyName.isEmpty()) {
-      throw new OzoneRestClientException("Invalid key Name");
-    }
-
-    if (downloadTo == null) {
-      throw new OzoneRestClientException("Invalid download path");
-    }
-
-    FileOutputStream outPutFile = null;
-    HttpGet getRequest = null;
-    try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
-      outPutFile = new FileOutputStream(downloadTo.toFile());
-
-      URIBuilder builder = new URIBuilder(volume.getClient().getEndPointURI());
-      builder.setPath("/" + getVolume().getVolumeName() + "/" + getBucketName()
-          + "/" + keyName).build();
-
-      getRequest = getVolume().getClient().getHttpGet(builder.toString());
-      executeGetKey(getRequest, httpClient, outPutFile);
-      outPutFile.flush();
-    } catch (IOException | URISyntaxException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      IOUtils.closeStream(outPutFile);
-      OzoneClientUtils.releaseConnection(getRequest);
-    }
-  }
-
-  /**
-   * Returns the data part of the key as a string.
-   *
-   * @param keyName - KeyName to get
-   * @return String - Data
-   * @throws OzoneException
-   */
-  public String getKey(String keyName) throws OzoneException {
-
-    if ((keyName == null) || keyName.isEmpty()) {
-      throw new OzoneRestClientException("Invalid key Name");
-    }
-
-    HttpGet getRequest = null;
-    ByteArrayOutputStream outPutStream = null;
-    try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
-      outPutStream = new ByteArrayOutputStream();
-
-      URIBuilder builder = new URIBuilder(volume.getClient().getEndPointURI());
-
-      builder.setPath("/" + getVolume().getVolumeName() + "/" + getBucketName()
-          + "/" + keyName).build();
-
-      getRequest = getVolume().getClient().getHttpGet(builder.toString());
-      executeGetKey(getRequest, httpClient, outPutStream);
-      return outPutStream.toString(ENCODING_NAME);
-    } catch (IOException | URISyntaxException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      IOUtils.closeStream(outPutStream);
-      OzoneClientUtils.releaseConnection(getRequest);
-    }
-
-  }
-
-  /**
-   * Executes get key and returns the data.
-   *
-   * @param getRequest - http Get Request
-   * @param httpClient - Client
-   * @param stream     - Stream to write data to.
-   * @throws IOException
-   * @throws OzoneException
-   */
-  public static void executeGetKey(HttpGet getRequest,
-      CloseableHttpClient httpClient, OutputStream stream)
-      throws IOException, OzoneException {
-
-    HttpEntity entity = null;
-    try {
-
-      HttpResponse response = httpClient.execute(getRequest);
-      int errorCode = response.getStatusLine().getStatusCode();
-      entity = response.getEntity();
-
-      if (errorCode == HTTP_OK) {
-        entity.writeTo(stream);
-        return;
-      }
-
-      if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
-      }
-
-      throw OzoneException.parse(EntityUtils.toString(entity));
-    } finally {
-      if (entity != null) {
-        EntityUtils.consumeQuietly(entity);
-      }
-    }
-  }
-
-  /**
-   * Deletes a key in this bucket.
-   *
-   * @param keyName - Name of the Key
-   * @throws OzoneException
-   */
-  public void deleteKey(String keyName) throws OzoneException {
-
-    if ((keyName == null) || keyName.isEmpty()) {
-      throw new OzoneRestClientException("Invalid key Name");
-    }
-
-    HttpDelete deleteRequest = null;
-    try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
-      URIBuilder builder = new URIBuilder(volume.getClient().getEndPointURI());
-      builder.setPath("/" + getVolume().getVolumeName() + "/" + getBucketName()
-          + "/" + keyName).build();
-
-      deleteRequest = getVolume()
-          .getClient().getHttpDelete(builder.toString());
-      executeDeleteKey(deleteRequest, httpClient);
-    } catch (IOException | URISyntaxException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      OzoneClientUtils.releaseConnection(deleteRequest);
-    }
-  }
-
-  /**
-   * Executes deleteKey.
-   *
-   * @param deleteRequest - http Delete Request
-   * @param httpClient    - Client
-   * @throws IOException
-   * @throws OzoneException
-   */
-  private void executeDeleteKey(HttpDelete deleteRequest,
-      CloseableHttpClient httpClient)
-      throws IOException, OzoneException {
-
-    HttpEntity entity = null;
-    try {
-
-      HttpResponse response = httpClient.execute(deleteRequest);
-      int errorCode = response.getStatusLine().getStatusCode();
-      entity = response.getEntity();
-
-      if (errorCode == HTTP_OK) {
-        return;
-      }
-
-      if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
-      }
-
-      throw OzoneException.parse(EntityUtils.toString(entity));
-    } finally {
-      if (entity != null) {
-        EntityUtils.consumeQuietly(entity);
-      }
-    }
-  }
-
-  /**
-   * List all keys in a bucket.
-   *
-   * @param resultLength The max length of listing result.
-   * @param previousKey The key from where listing should start,
-   *                    this key is excluded in the result.
-   * @param prefix The prefix that return list keys start with.
-   * @return List of OzoneKeys
-   * @throws OzoneException
-   */
-  public List<OzoneKey> listKeys(String resultLength, String previousKey,
-      String prefix) throws OzoneException {
-    HttpGet getRequest = null;
-    try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
-      OzoneRestClient client = getVolume().getClient();
-      URIBuilder builder = new URIBuilder(volume.getClient().getEndPointURI());
-      builder.setPath("/" + getVolume().getVolumeName() + "/" + getBucketName())
-          .build();
-
-      if (!Strings.isNullOrEmpty(resultLength)) {
-        builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength);
-      }
-
-      if (!Strings.isNullOrEmpty(previousKey)) {
-        builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, previousKey);
-      }
-
-      if (!Strings.isNullOrEmpty(prefix)) {
-        builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
-      }
-
-      final String uri = builder.toString();
-      getRequest = client.getHttpGet(uri);
-      LOG.trace("listKeys URI={}", uri);
-      return executeListKeys(getRequest, httpClient);
-
-    } catch (IOException | URISyntaxException e) {
-      throw new OzoneRestClientException(e.getMessage(), e);
-    } finally {
-      OzoneClientUtils.releaseConnection(getRequest);
-    }
-  }
-
-  /**
-   * List keys in a bucket with the provided prefix, with paging results.
-   *
-   * @param prefix The prefix of the object keys
-   * @param maxResult max size per response
-   * @param prevKey the previous key for paging
-   */
-  public List<OzoneKey> listKeys(String prefix, int maxResult, String prevKey)
-      throws OzoneException {
-    HttpGet getRequest = null;
-    try {
-      final URI uri =  new URIBuilder(volume.getClient().getEndPointURI())
-          .setPath(OzoneConsts.KSM_KEY_PREFIX + getVolume().getVolumeName() +
-              OzoneConsts.KSM_KEY_PREFIX + getBucketName())
-          .setParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix)
-          .setParameter(Header.OZONE_LIST_QUERY_MAXKEYS,
-              String.valueOf(maxResult))
-          .setParameter(Header.OZONE_LIST_QUERY_PREVKEY, prevKey)
-          .build();
-      final OzoneRestClient client = getVolume().getClient();
-      getRequest = client.getHttpGet(uri.toString());
-      return executeListKeys(getRequest, HttpClientBuilder.create().build());
-    } catch (IOException | URISyntaxException e) {
-      throw new OzoneRestClientException(e.getMessage());
-    } finally {
-      OzoneClientUtils.releaseConnection(getRequest);
-    }
-  }
-
-  /**
-   * Execute list Key.
-   *
-   * @param getRequest - HttpGet
-   * @param httpClient - HttpClient
-   * @return List<OzoneKey>
-   * @throws IOException
-   * @throws OzoneException
-   */
-  public static List<OzoneKey> executeListKeys(HttpGet getRequest,
-      CloseableHttpClient httpClient) throws IOException, OzoneException {
-    HttpEntity entity = null;
-    List<OzoneKey> ozoneKeyList = new LinkedList<OzoneKey>();
-    try {
-      HttpResponse response = httpClient.execute(getRequest);
-      int errorCode = response.getStatusLine().getStatusCode();
-
-      entity = response.getEntity();
-
-      if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
-      }
-      if (errorCode == HTTP_OK) {
-        String temp = EntityUtils.toString(entity);
-        ListKeys keyList = ListKeys.parse(temp);
-
-        for (KeyInfo info : keyList.getKeyList()) {
-          ozoneKeyList.add(new OzoneKey(info));
-        }
-        return ozoneKeyList;
-
-      } else {
-        throw OzoneException.parse(EntityUtils.toString(entity));
-      }
-    } finally {
-      if (entity != null) {
-        EntityUtils.consumeQuietly(entity);
-      }
-    }
-  }
-
-  /**
-   * Get info of the specified key.
-   */
-  public OzoneKey getKeyInfo(String keyName) throws OzoneException {
-    if ((keyName == null) || keyName.isEmpty()) {
-      throw new OzoneRestClientException(
-          "Unable to get key info, key name is null or empty");
-    }
-
-    HttpGet getRequest = null;
-    try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
-      OzoneRestClient client = getVolume().getClient();
-      URIBuilder builder = new URIBuilder(volume.getClient().getEndPointURI());
-      builder
-          .setPath("/" + getVolume().getVolumeName() + "/" + getBucketName()
-              + "/" + keyName)
-          .setParameter(Header.OZONE_INFO_QUERY_TAG,
-              Header.OZONE_INFO_QUERY_KEY)
-          .build();
-
-      getRequest = client.getHttpGet(builder.toString());
-      return executeGetKeyInfo(getRequest, httpClient);
-    } catch (IOException | URISyntaxException e) {
-      throw new OzoneRestClientException(e.getMessage(), e);
-    } finally {
-      OzoneClientUtils.releaseConnection(getRequest);
-    }
-  }
-
-  /**
-   * Execute get Key info.
-   *
-   * @param getRequest - HttpGet
-   * @param httpClient - HttpClient
-   * @return List<OzoneKey>
-   * @throws IOException
-   * @throws OzoneException
-   */
-  private OzoneKey executeGetKeyInfo(HttpGet getRequest,
-      CloseableHttpClient httpClient) throws IOException, OzoneException {
-    HttpEntity entity = null;
-    try {
-      HttpResponse response = httpClient.execute(getRequest);
-      int errorCode = response.getStatusLine().getStatusCode();
-      entity = response.getEntity();
-      if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
-      }
-
-      if (errorCode == HTTP_OK) {
-        OzoneKey key = new OzoneKey(
-            KeyInfo.parse(EntityUtils.toString(entity)));
-        return key;
-      }
-      throw OzoneException.parse(EntityUtils.toString(entity));
-    } finally {
-      if (entity != null) {
-        EntityUtils.consumeQuietly(entity);
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneKey.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneKey.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneKey.java
deleted file mode 100644
index 5a3a0c4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneKey.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.web.client;
-
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-
-/**
- * Client side representation of an ozone Key.
- */
-public class OzoneKey {
-  private KeyInfo keyInfo;
-
-  /**
-   * Constructor for Ozone Key.
-   * @param keyInfo - Key Info
-   */
-  public OzoneKey(KeyInfo keyInfo) {
-    this.keyInfo = keyInfo;
-  }
-
-  /**
-   * Returns Key Info.
-   * @return Object Info
-   */
-  public KeyInfo getObjectInfo() {
-    return keyInfo;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
deleted file mode 100644
index 16a2e58..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
+++ /dev/null
@@ -1,801 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.web.client;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Strings;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.response.ListVolumes;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.util.Time;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.HttpDelete;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.FileEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.util.EntityUtils;
-
-import javax.ws.rs.core.HttpHeaders;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-
-import static java.net.HttpURLConnection.HTTP_CREATED;
-import static java.net.HttpURLConnection.HTTP_OK;
-
-/**
- * Ozone client that connects to an Ozone server. Please note that this class is
- * not  thread safe.
- */
-public class OzoneRestClient implements Closeable {
-  private URI endPointURI;
-  private String userAuth;
-
-  /**
-   * Constructor for OzoneRestClient.
-   */
-  public OzoneRestClient() {
-  }
-
-  /**
-   * Constructor for OzoneRestClient.
-   */
-  public OzoneRestClient(String ozoneURI)
-      throws OzoneException, URISyntaxException {
-    setEndPoint(ozoneURI);
-  }
-
-  /**
-   * Constructor for OzoneRestClient.
-   */
-  public OzoneRestClient(String ozoneURI, String userAuth)
-      throws OzoneException, URISyntaxException {
-    setEndPoint(ozoneURI);
-    setUserAuth(userAuth);
-  }
-
-  /**
-   * Returns the end Point.
-   *
-   * @return String
-   */
-  public URI getEndPointURI() {
-    return endPointURI;
-  }
-
-  /**
-   * Sets the End Point info using an URI.
-   *
-   * @param endPointURI - URI
-   * @throws OzoneException
-   */
-  public void setEndPointURI(URI endPointURI) throws OzoneException {
-    if ((endPointURI == null) || (endPointURI.toString().isEmpty())) {
-      throw new OzoneRestClientException("Invalid ozone URI");
-    }
-    this.endPointURI = endPointURI;
-  }
-
-  /**
-   * Set endPoint.
-   *
-   * @param clusterFQDN - cluster FQDN.
-   */
-  public void setEndPoint(String clusterFQDN) throws
-      OzoneException, URISyntaxException {
-    setEndPointURI(new URI(clusterFQDN));
-  }
-
-  /**
-   * Get user Auth String.
-   *
-   * @return - User Auth String
-   */
-  public String getUserAuth() {
-    return this.userAuth;
-  }
-
-  /**
-   * Set User Auth.
-   *
-   * @param userAuth - User Auth String
-   */
-  public void setUserAuth(String userAuth) {
-    this.userAuth = userAuth;
-  }
-
-  /**
-   * create volume.
-   *
-   * @param volumeName - volume name 3 - 63 chars, small letters.
-   * @param onBehalfOf - The user on behalf we are making the call for
-   * @param quota      - Quota's are specified in a specific format. it is
-   *                   integer(MB|GB|TB), for example 100TB.
-   * @throws OzoneRestClientException
-   */
-  public OzoneVolume createVolume(String volumeName, String onBehalfOf,
-                                  String quota) throws OzoneException {
-    HttpPost httpPost = null;
-    try (CloseableHttpClient httpClient = newHttpClient()) {
-      OzoneUtils.verifyResourceName(volumeName);
-
-      URIBuilder builder = new URIBuilder(endPointURI);
-      builder.setPath("/" + volumeName);
-      if (quota != null) {
-        builder.setParameter(Header.OZONE_QUOTA_QUERY_TAG, quota);
-      }
-
-      httpPost = getHttpPost(onBehalfOf, builder.build().toString());
-      executeCreateVolume(httpPost, httpClient);
-      return getVolume(volumeName);
-    } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      OzoneClientUtils.releaseConnection(httpPost);
-    }
-  }
-
-  /**
-   * Returns information about an existing Volume. if the Volume does not exist,
-   * or if the user does not have access rights OzoneException is thrown
-   *
-   * @param volumeName - volume name 3 - 63 chars, small letters.
-   * @return OzoneVolume Ozone Client Volume Class.
-   * @throws OzoneException
-   */
-  public OzoneVolume getVolume(String volumeName) throws OzoneException {
-    HttpGet httpGet = null;
-    try (CloseableHttpClient httpClient = newHttpClient()) {
-      OzoneUtils.verifyResourceName(volumeName);
-      URIBuilder builder = new URIBuilder(endPointURI);
-      builder.setPath("/" + volumeName)
-          .setParameter(Header.OZONE_INFO_QUERY_TAG,
-              Header.OZONE_INFO_QUERY_VOLUME)
-          .build();
-
-      httpGet = getHttpGet(builder.toString());
-      return executeInfoVolume(httpGet, httpClient);
-    } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      OzoneClientUtils.releaseConnection(httpGet);
-    }
-  }
-
-  /**
-   * List all the volumes owned by the user or Owned by the user specified in
-   * the behalf of string.
-   *
-   * @param onBehalfOf
-   *  User Name of the user if it is not the caller. for example,
-   *  an admin wants to list some other users volumes.
-   * @param prefix
-   *   Return only volumes that match this prefix.
-   * @param maxKeys
-   *   Maximum number of results to return, if the result set
-   *   is smaller than requested size, it means that list is
-   *   complete.
-   * @param previousVolume
-   *   The previous volume name.
-   * @return List of Volumes
-   * @throws OzoneException
-   */
-  public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix,
-      int maxKeys, String previousVolume) throws OzoneException {
-    HttpGet httpGet = null;
-    try (CloseableHttpClient httpClient = newHttpClient()) {
-      URIBuilder builder = new URIBuilder(endPointURI);
-      if (!Strings.isNullOrEmpty(prefix)) {
-        builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
-      }
-
-      if (maxKeys > 0) {
-        builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, Integer
-            .toString(maxKeys));
-      }
-
-      if (!Strings.isNullOrEmpty(previousVolume)) {
-        builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY,
-            previousVolume);
-      }
-
-      builder.setPath("/").build();
-
-      httpGet = getHttpGet(builder.toString());
-      if (onBehalfOf != null) {
-        httpGet.addHeader(Header.OZONE_USER, onBehalfOf);
-      }
-      return executeListVolume(httpGet, httpClient);
-    } catch (IOException | URISyntaxException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      OzoneClientUtils.releaseConnection(httpGet);
-    }
-  }
-
-  /**
-   * List all the volumes owned by the user or Owned by the user specified in
-   * the behalf of string.
-   *
-   * @param onBehalfOf - User Name of the user if it is not the caller. for
-   *                   example, an admin wants to list some other users
-   *                   volumes.
-   * @param prefix     - Return only volumes that match this prefix.
-   * @param maxKeys    - Maximum number of results to return, if the result set
-   *                   is smaller than requested size, it means that list is
-   *                   complete.
-   * @param prevKey    - The last key that client got, server will continue
-   *                   returning results from that point.
-   * @return List of Volumes
-   * @throws OzoneException
-   */
-  public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix,
-      int maxKeys, OzoneVolume prevKey) throws OzoneException {
-    String volumeName = null;
-
-    if (prevKey != null) {
-      volumeName = prevKey.getVolumeName();
-    }
-
-    return listVolumes(onBehalfOf, prefix, maxKeys, volumeName);
-  }
-
-  /**
-   * List volumes of the current user or if onBehalfof is not null lists volume
-   * owned by that user. You need admin privilege to read other users volume
-   * lists.
-   *
-   * @param onBehalfOf - Name of the user you want to get volume list
-   * @return - Volume list.
-   * @throws OzoneException
-   */
-  public List<OzoneVolume> listVolumes(String onBehalfOf)
-      throws OzoneException {
-    return listVolumes(onBehalfOf, null,
-        Integer.parseInt(Header.OZONE_DEFAULT_LIST_SIZE), StringUtils.EMPTY);
-  }
-
-  /**
-   * List all volumes in a cluster. This can be invoked only by an Admin.
-   *
-   * @param prefix  - Returns only volumes that match this prefix.
-   * @param maxKeys - Maximum niumber of keys to return
-   * @param prevKey - Last Ozone Volume from the last Iteration.
-   * @return List of Volumes
-   * @throws OzoneException
-   */
-  public List<OzoneVolume> listAllVolumes(String prefix, int maxKeys,
-      OzoneVolume prevKey) throws OzoneException {
-    HttpGet httpGet = null;
-    try (CloseableHttpClient httpClient = newHttpClient()) {
-      URIBuilder builder = new URIBuilder(endPointURI);
-      if (prefix != null) {
-        builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
-      }
-
-      if (maxKeys > 0) {
-        builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, Integer
-            .toString(maxKeys));
-      }
-
-      if (prevKey != null) {
-        builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY,
-            prevKey.getOwnerName()+ "/" + prevKey.getVolumeName());
-      }
-
-      builder.addParameter(Header.OZONE_LIST_QUERY_ROOTSCAN, "true");
-      builder.setPath("/").build();
-      httpGet = getHttpGet(builder.toString());
-      return executeListVolume(httpGet, httpClient);
-
-    } catch (IOException | URISyntaxException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      OzoneClientUtils.releaseConnection(httpGet);
-    }
-  }
-
-    /**
-     * delete a given volume.
-     *
-     * @param volumeName - volume to be deleted.
-     * @throws OzoneException - Ozone Exception
-     */
-  public void deleteVolume(String volumeName) throws OzoneException {
-    HttpDelete httpDelete = null;
-    try (CloseableHttpClient httpClient = newHttpClient()) {
-      OzoneUtils.verifyResourceName(volumeName);
-      URIBuilder builder = new URIBuilder(endPointURI);
-      builder.setPath("/" + volumeName).build();
-
-      httpDelete = getHttpDelete(builder.toString());
-      executeDeleteVolume(httpDelete, httpClient);
-    } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      OzoneClientUtils.releaseConnection(httpDelete);
-    }
-  }
-
-  /**
-   * Sets the Volume Owner.
-   *
-   * @param volumeName - Volume Name
-   * @param newOwner   - New Owner Name
-   * @throws OzoneException
-   */
-  public void setVolumeOwner(String volumeName, String newOwner)
-      throws OzoneException {
-    HttpPut putRequest = null;
-    if (newOwner == null || newOwner.isEmpty()) {
-      throw new OzoneRestClientException("Invalid new owner name");
-    }
-    try (CloseableHttpClient httpClient = newHttpClient()) {
-      OzoneUtils.verifyResourceName(volumeName);
-      URIBuilder builder = new URIBuilder(endPointURI);
-      builder.setPath("/" + volumeName).build();
-
-      putRequest = getHttpPut(builder.toString());
-      putRequest.addHeader(Header.OZONE_USER, newOwner);
-      executePutVolume(putRequest, httpClient);
-
-    } catch (URISyntaxException | IllegalArgumentException | IOException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      OzoneClientUtils.releaseConnection(putRequest);
-    }
-  }
-
-  /**
-   * Sets the Volume Quota. Quota's are specified in a specific format. it is
-   * <integer>|(MB|GB|TB. for example 100TB.
-   * <p>
-   * To Remove a quota you can specify Header.OZONE_QUOTA_REMOVE
-   *
-   * @param volumeName - volume name
-   * @param quota      - Quota String or  Header.OZONE_QUOTA_REMOVE
-   * @throws OzoneException
-   */
-  public void setVolumeQuota(String volumeName, String quota)
-      throws OzoneException {
-    if (quota == null || quota.isEmpty()) {
-      throw new OzoneRestClientException("Invalid quota");
-    }
-    HttpPut putRequest = null;
-    try (CloseableHttpClient httpClient = newHttpClient()) {
-      OzoneUtils.verifyResourceName(volumeName);
-      URIBuilder builder = new URIBuilder(endPointURI);
-      builder.setPath("/" + volumeName)
-          .setParameter(Header.OZONE_QUOTA_QUERY_TAG, quota)
-          .build();
-
-      putRequest = getHttpPut(builder.toString());
-      executePutVolume(putRequest, httpClient);
-
-    } catch (URISyntaxException | IllegalArgumentException | IOException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      OzoneClientUtils.releaseConnection(putRequest);
-    }
-  }
-
-  /**
-   * Sends the create Volume request to the server.
-   *
-   * @param httppost   - http post class
-   * @param httpClient - httpClient
-   * @throws IOException    -
-   * @throws OzoneException
-   */
-  private void executeCreateVolume(HttpPost httppost,
-      final CloseableHttpClient httpClient)
-      throws IOException, OzoneException {
-    HttpEntity entity = null;
-    try {
-      HttpResponse response = httpClient.execute(httppost);
-      int errorCode = response.getStatusLine().getStatusCode();
-      entity = response.getEntity();
-
-      if ((errorCode == HTTP_OK) || (errorCode == HTTP_CREATED)) {
-        return;
-      }
-
-      if (entity != null) {
-        throw OzoneException.parse(EntityUtils.toString(entity));
-      } else {
-        throw new OzoneRestClientException("Unexpected null in http payload");
-      }
-    } finally {
-      if (entity != null) {
-        EntityUtils.consume(entity);
-      }
-    }
-  }
-
-  /**
-   * Sends the create Volume request to the server.
-   *
-   * @param httpGet - httpGet
-   * @return OzoneVolume
-   * @throws IOException    -
-   * @throws OzoneException
-   */
-  private OzoneVolume executeInfoVolume(HttpGet httpGet,
-      final CloseableHttpClient httpClient)
-      throws IOException, OzoneException {
-    HttpEntity entity = null;
-    try {
-      HttpResponse response = httpClient.execute(httpGet);
-      int errorCode = response.getStatusLine().getStatusCode();
-
-      entity = response.getEntity();
-      if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
-      }
-
-      if (errorCode == HTTP_OK) {
-        OzoneVolume volume = new OzoneVolume(this);
-        volume.setVolumeInfo(EntityUtils.toString(entity));
-        return volume;
-      } else {
-        throw OzoneException.parse(EntityUtils.toString(entity));
-      }
-    } finally {
-      if (entity != null) {
-        EntityUtils.consumeQuietly(entity);
-      }
-    }
-  }
-
-  /**
-   * Sends update volume requests to the server.
-   *
-   * @param putRequest http request
-   * @throws IOException
-   * @throws OzoneException
-   */
-  private void executePutVolume(HttpPut putRequest,
-      final CloseableHttpClient httpClient)
-      throws IOException, OzoneException {
-    HttpEntity entity = null;
-    try {
-      HttpResponse response = httpClient.execute(putRequest);
-      int errorCode = response.getStatusLine().getStatusCode();
-      entity = response.getEntity();
-      if (errorCode != HTTP_OK) {
-        throw OzoneException.parse(EntityUtils.toString(entity));
-      }
-    } finally {
-      if (entity != null) {
-        EntityUtils.consume(entity);
-      }
-    }
-  }
-
-  /**
-   * List Volumes.
-   *
-   * @param httpGet - httpGet
-   * @return OzoneVolume
-   * @throws IOException    -
-   * @throws OzoneException
-   */
-  private List<OzoneVolume> executeListVolume(HttpGet httpGet,
-      final CloseableHttpClient httpClient)
-      throws IOException, OzoneException {
-    HttpEntity entity = null;
-    List<OzoneVolume> volList = new LinkedList<>();
-    try {
-      HttpResponse response = httpClient.execute(httpGet);
-      int errorCode = response.getStatusLine().getStatusCode();
-      entity = response.getEntity();
-
-      if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
-      }
-
-      String temp = EntityUtils.toString(entity);
-      if (errorCode == HTTP_OK) {
-        ListVolumes listVolumes =
-            ListVolumes.parse(temp);
-
-        for (VolumeInfo info : listVolumes.getVolumes()) {
-          volList.add(new OzoneVolume(info, this));
-        }
-        return volList;
-
-      } else {
-        throw OzoneException.parse(EntityUtils.toString(entity));
-      }
-    } finally {
-      if (entity != null) {
-        EntityUtils.consumeQuietly(entity);
-      }
-    }
-  }
-
-  /**
-   * Delete Volume.
-   *
-   * @param httpDelete - Http Delete Request
-   * @throws IOException
-   * @throws OzoneException
-   */
-  private void executeDeleteVolume(HttpDelete httpDelete,
-      final CloseableHttpClient httpClient)
-      throws IOException, OzoneException {
-    HttpEntity entity = null;
-    try {
-      HttpResponse response = httpClient.execute(httpDelete);
-      int errorCode = response.getStatusLine().getStatusCode();
-      entity = response.getEntity();
-
-      if (errorCode != HTTP_OK) {
-        throw OzoneException.parse(EntityUtils.toString(entity));
-      }
-    } finally {
-      if (entity != null) {
-        EntityUtils.consumeQuietly(entity);
-      }
-    }
-  }
-
-  /**
-   * Puts a Key in Ozone Bucket.
-   *
-   * @param volumeName - Name of the Volume
-   * @param bucketName - Name of the Bucket
-   * @param keyName - Name of the Key
-   * @param file    - Stream that gets read to be put into Ozone.
-   * @throws OzoneException
-   */
-  public void putKey(String volumeName, String bucketName, String keyName,
-      File file) throws OzoneException {
-    OzoneUtils.verifyResourceName(volumeName);
-    OzoneUtils.verifyResourceName(bucketName);
-
-    if (StringUtils.isEmpty(keyName)) {
-      throw new OzoneRestClientException("Invalid key Name");
-    }
-
-    if (file == null) {
-      throw new OzoneRestClientException("Invalid data stream");
-    }
-
-    HttpPut putRequest = null;
-    FileInputStream fis = null;
-    try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
-      URIBuilder builder = new URIBuilder(getEndPointURI());
-      builder.setPath("/" + volumeName + "/" + bucketName + "/" + keyName)
-          .build();
-
-      putRequest = getHttpPut(builder.toString());
-
-      FileEntity fileEntity = new FileEntity(file, ContentType
-          .APPLICATION_OCTET_STREAM);
-      putRequest.setEntity(fileEntity);
-
-      fis = new FileInputStream(file);
-      putRequest.setHeader(Header.CONTENT_MD5, DigestUtils.md5Hex(fis));
-      OzoneBucket.executePutKey(putRequest, httpClient);
-    } catch (IOException | URISyntaxException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      IOUtils.closeStream(fis);
-      OzoneClientUtils.releaseConnection(putRequest);
-    }
-  }
-
-  /**
-   * Gets a key from the Ozone server and writes to the file pointed by the
-   * downloadTo Path.
-   *
-   * @param volumeName - Volume Name in Ozone.
-   * @param bucketName - Bucket Name in Ozone.
-   * @param keyName - Key Name in Ozone.
-   * @param downloadTo File Name to download the Key's Data to
-   */
-  public void getKey(String volumeName, String bucketName, String keyName,
-      Path downloadTo) throws OzoneException {
-    OzoneUtils.verifyResourceName(volumeName);
-    OzoneUtils.verifyResourceName(bucketName);
-
-    if (StringUtils.isEmpty(keyName)) {
-      throw new OzoneRestClientException("Invalid key Name");
-    }
-
-    if (downloadTo == null) {
-      throw new OzoneRestClientException("Invalid download path");
-    }
-
-    FileOutputStream outPutFile = null;
-    HttpGet getRequest = null;
-    try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
-      outPutFile = new FileOutputStream(downloadTo.toFile());
-
-      URIBuilder builder = new URIBuilder(getEndPointURI());
-      builder.setPath("/" + volumeName + "/" + bucketName + "/" + keyName)
-          .build();
-
-      getRequest = getHttpGet(builder.toString());
-      OzoneBucket.executeGetKey(getRequest, httpClient, outPutFile);
-      outPutFile.flush();
-    } catch (IOException | URISyntaxException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      IOUtils.closeStream(outPutFile);
-      OzoneClientUtils.releaseConnection(getRequest);
-    }
-  }
-
-  /**
-   * List all keys in the given bucket.
-   *
-   * @param volumeName - Volume name
-   * @param bucketName - Bucket name
-   * @param resultLength The max length of listing result.
-   * @param previousKey The key from where listing should start,
-   *                    this key is excluded in the result.
-   * @param prefix The prefix that return list keys start with.
-   *
-   * @return List of OzoneKeys
-   */
-  public List<OzoneKey> listKeys(String volumeName, String bucketName,
-      String resultLength, String previousKey, String prefix)
-      throws OzoneException {
-    OzoneUtils.verifyResourceName(volumeName);
-    OzoneUtils.verifyResourceName(bucketName);
-
-    HttpGet getRequest = null;
-    try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
-      URIBuilder builder = new URIBuilder(getEndPointURI());
-      builder.setPath("/" + volumeName + "/" + bucketName).build();
-
-      if (!Strings.isNullOrEmpty(resultLength)) {
-        builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength);
-      }
-
-      if (!Strings.isNullOrEmpty(previousKey)) {
-        builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, previousKey);
-      }
-
-      if (!Strings.isNullOrEmpty(prefix)) {
-        builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
-      }
-
-      getRequest = getHttpGet(builder.toString());
-      return OzoneBucket.executeListKeys(getRequest, httpClient);
-    } catch (IOException | URISyntaxException e) {
-      throw new OzoneRestClientException(e.getMessage(), e);
-    } finally {
-      OzoneClientUtils.releaseConnection(getRequest);
-    }
-  }
-
-  /**
-   * Returns a standard HttpPost Object to use for ozone post requests.
-   *
-   * @param onBehalfOf - If the use is being made on behalf of user, that user
-   * @param uriString  - UriString
-   * @return HttpPost
-   */
-  public HttpPost getHttpPost(String onBehalfOf, String uriString) {
-    HttpPost httpPost = new HttpPost(uriString);
-    addOzoneHeaders(httpPost);
-    if (onBehalfOf != null) {
-      httpPost.addHeader(Header.OZONE_USER, onBehalfOf);
-    }
-    return httpPost;
-  }
-
-  /**
-   * Returns a standard HttpGet Object to use for ozone Get requests.
-   *
-   * @param uriString - The full Uri String
-   * @return HttpGet
-   */
-  public HttpGet getHttpGet(String uriString) {
-    HttpGet httpGet = new HttpGet(uriString);
-    addOzoneHeaders(httpGet);
-    return httpGet;
-  }
-
-  /**
-   * Returns httpDelete.
-   *
-   * @param uriString - uri
-   * @return HttpDelete
-   */
-  public HttpDelete getHttpDelete(String uriString) {
-    HttpDelete httpDel = new HttpDelete(uriString);
-    addOzoneHeaders(httpDel);
-    return httpDel;
-  }
-
-  /**
-   * returns an HttpPut Object.
-   *
-   * @param uriString - Uri
-   * @return HttpPut
-   */
-  public HttpPut getHttpPut(String uriString) {
-    HttpPut httpPut = new HttpPut(uriString);
-    addOzoneHeaders(httpPut);
-    return httpPut;
-  }
-
-  /**
-   * Add Ozone Headers.
-   *
-   * @param httpRequest - Http Request
-   */
-  private void addOzoneHeaders(HttpRequestBase httpRequest) {
-    SimpleDateFormat format =
-        new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
-
-    httpRequest.addHeader(Header.OZONE_VERSION_HEADER,
-        Header.OZONE_V1_VERSION_HEADER);
-    httpRequest.addHeader(HttpHeaders.DATE,
-        format.format(new Date(Time.monotonicNow())));
-    if (getUserAuth() != null) {
-      httpRequest.addHeader(HttpHeaders.AUTHORIZATION,
-          Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
-              getUserAuth());
-    }
-  }
-
-  /**
-   * Closes this stream and releases any system resources associated with it. If
-   * the stream is already closed then invoking this method has no effect.
-   *
-   * @throws IOException if an I/O error occurs
-   */
-  @Override
-  public void close() throws IOException {
-    // TODO : Currently we create a new HTTP client. We should switch
-    // This to a Pool and cleanup the pool here.
-  }
-
-  @VisibleForTesting
-  public CloseableHttpClient newHttpClient() {
-    return OzoneClientUtils.newHttpClient();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClientException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClientException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClientException.java
deleted file mode 100644
index dfb2357..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClientException.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.web.client;
-
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-
-/**
- * This exception is thrown by the Ozone Clients.
- */
-public class OzoneRestClientException extends OzoneException {
-  /**
-   * Constructor that allows the shortMessage.
-   *
-   * @param shortMessage Short Message
-   */
-  public OzoneRestClientException(String shortMessage) {
-    super(0, shortMessage, shortMessage);
-  }
-
-  /**
-   * Constructor that allows a shortMessage and an exception.
-   *
-   * @param shortMessage short message
-   * @param ex exception
-   */
-  public OzoneRestClientException(String shortMessage, Exception ex) {
-    super(0, shortMessage, shortMessage, ex);
-  }
-
-  /**
-   * Constructor that allows the shortMessage and a longer message.
-   *
-   * @param shortMessage Short Message
-   * @param message long error message
-   */
-  public OzoneRestClientException(String shortMessage, String message) {
-    super(0, shortMessage, message);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
deleted file mode 100644
index 4fac514..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
+++ /dev/null
@@ -1,581 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.client;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Strings;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.request.OzoneQuota;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.response.ListBuckets;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.HttpDelete;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.util.EntityUtils;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import static java.net.HttpURLConnection.HTTP_CREATED;
-import static java.net.HttpURLConnection.HTTP_OK;
-
-/**
- * Ozone Volume Class.
- */
-public class OzoneVolume {
-  private VolumeInfo volumeInfo;
-  private Map<String, String> headerMap;
-  private final OzoneRestClient client;
-
-  /**
-   * Constructor for OzoneVolume.
-   */
-  public OzoneVolume(OzoneRestClient client) {
-    this.client = client;
-    this.headerMap = new HashMap<>();
-  }
-
-  /**
-   * Constructor for OzoneVolume.
-   *
-   * @param volInfo - volume Info.
-   * @param client  Client
-   */
-  public OzoneVolume(VolumeInfo volInfo, OzoneRestClient client) {
-    this.volumeInfo = volInfo;
-    this.client = client;
-  }
-
-  /**
-   * Returns a Json String of this class.
-   * @return String
-   * @throws IOException
-   */
-  public String getJsonString() throws IOException {
-    return volumeInfo.toJsonString();
-  }
-
-  /**
-   * sets the Volume Info.
-   *
-   * @param volInfoString - Volume Info String
-   */
-  public void setVolumeInfo(String volInfoString) throws IOException {
-    this.volumeInfo = VolumeInfo.parse(volInfoString);
-  }
-
-  /**
-   * @return the volume info.
-   */
-  public VolumeInfo getVolumeInfo() {
-    return this.volumeInfo;
-  }
-
-  /**
-   * Returns volume Name.
-   *
-   * @return Volume Name.
-   */
-  public String getVolumeName() {
-    return this.volumeInfo.getVolumeName();
-  }
-
-  /**
-   * Get created by.
-   *
-   * @return String
-   */
-  public String getCreatedby() {
-    return this.volumeInfo.getCreatedBy();
-  }
-
-  /**
-   * returns the Owner name.
-   *
-   * @return String
-   */
-  public String getOwnerName() {
-    return this.volumeInfo.getOwner().getName();
-  }
-
-  /**
-   * Returns Quota Info.
-   *
-   * @return Quota
-   */
-  public OzoneQuota getQuota() {
-    return volumeInfo.getQuota();
-  }
-
-  /**
-   * Returns creation time of Volume.
-   *
-   * @return String
-   */
-  public String getCreatedOn() {
-    return volumeInfo.getCreatedOn();
-  }
-
-  /**
-   * Returns a Http header from the Last Volume related call.
-   *
-   * @param headerName - Name of the header
-   * @return - Header Value
-   */
-  public String getHeader(String headerName) {
-    return headerMap.get(headerName);
-  }
-
-  /**
-   * Gets the Client, this is used by Bucket and Key Classes.
-   *
-   * @return - Ozone Client
-   */
-  OzoneRestClient getClient() {
-    return client;
-  }
-
-  /**
-   * Create Bucket - Creates a bucket under a given volume.
-   *
-   * @param bucketName - Bucket Name
-   * @param acls - Acls - User Acls
-   * @param storageType - Storage Class
-   * @param versioning - enable versioning support on a bucket.
-   *
-   *
-   * @return - a Ozone Bucket Object
-   */
-  public OzoneBucket createBucket(String bucketName, String[] acls,
-                                  StorageType storageType,
-                                  OzoneConsts.Versioning versioning)
-      throws OzoneException {
-
-    HttpPost httpPost = null;
-    try (CloseableHttpClient httpClient = newHttpClient()) {
-      OzoneUtils.verifyResourceName(bucketName);
-      URIBuilder builder = new URIBuilder(getClient().getEndPointURI());
-      builder.setPath("/" + getVolumeName() + "/" + bucketName).build();
-
-      httpPost = client.getHttpPost(null, builder.toString());
-      if (acls != null) {
-        for (String acl : acls) {
-          httpPost
-              .addHeader(Header.OZONE_ACLS, Header.OZONE_ACL_ADD + " " + acl);
-        }
-      }
-
-      httpPost.addHeader(Header.OZONE_STORAGE_TYPE, storageType.toString());
-      httpPost.addHeader(Header.OZONE_BUCKET_VERSIONING, versioning.toString());
-      executeCreateBucket(httpPost, httpClient);
-      return getBucket(bucketName);
-    } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      OzoneClientUtils.releaseConnection(httpPost);
-    }
-  }
-
-  /**
-   * Create Bucket.
-   *
-   * @param bucketName - bucket name
-   * @param acls - acls
-   * @param storageType - storage class
-   *
-   * @throws OzoneException
-   */
-  public OzoneBucket createBucket(String bucketName, String[] acls,
-                                  StorageType storageType)
-      throws OzoneException {
-    return createBucket(bucketName, acls, storageType,
-        OzoneConsts.Versioning.DISABLED);
-  }
-
-  /**
-   * Create Bucket.
-   *
-   * @param bucketName - bucket name
-   * @param acls - acls
-   *
-   * @throws OzoneException
-   */
-  public OzoneBucket createBucket(String bucketName, String[] acls)
-      throws OzoneException {
-    return createBucket(bucketName, acls, StorageType.DEFAULT,
-        OzoneConsts.Versioning.DISABLED);
-  }
-
-
-  /**
-   * Create Bucket.
-   *
-   * @param bucketName - bucket name
-   *
-   * @throws OzoneException
-   */
-  public OzoneBucket createBucket(String bucketName) throws OzoneException {
-    return createBucket(bucketName, null,  StorageType.DEFAULT,
-        OzoneConsts.Versioning.DISABLED);
-  }
-
-
-  /**
-   * execute a Create Bucket Request against Ozone server.
-   *
-   * @param httppost - httpPost
-   *
-   * @throws IOException
-   * @throws OzoneException
-   */
-  private void executeCreateBucket(HttpPost httppost,
-      CloseableHttpClient httpClient)
-      throws IOException, OzoneException {
-    HttpEntity entity = null;
-    try {
-      HttpResponse response = httpClient.execute(httppost);
-      int errorCode = response.getStatusLine().getStatusCode();
-      entity = response.getEntity();
-      if ((errorCode == HTTP_OK) || (errorCode == HTTP_CREATED)) {
-        return;
-      }
-
-      if (entity != null) {
-        throw OzoneException.parse(EntityUtils.toString(entity));
-      } else {
-        throw new OzoneRestClientException("Unexpected null in http payload");
-      }
-    } finally {
-      if (entity != null) {
-        EntityUtils.consumeQuietly(entity);
-      }
-    }
-  }
-
-  /**
-   * Adds Acls to an existing bucket.
-   *
-   * @param bucketName - Name of the bucket
-   * @param acls - Acls
-   *
-   * @throws OzoneException
-   */
-  public void addAcls(String bucketName, String[] acls) throws OzoneException {
-    HttpPut putRequest = null;
-    try (CloseableHttpClient httpClient = newHttpClient()) {
-      OzoneUtils.verifyResourceName(bucketName);
-      URIBuilder builder = new URIBuilder(getClient().getEndPointURI());
-      builder.setPath("/" + getVolumeName() + "/" + bucketName).build();
-      putRequest = client.getHttpPut(builder.toString());
-
-      for (String acl : acls) {
-        putRequest
-            .addHeader(Header.OZONE_ACLS, Header.OZONE_ACL_ADD + " " + acl);
-      }
-      executePutBucket(putRequest, httpClient);
-    } catch (URISyntaxException | IOException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      OzoneClientUtils.releaseConnection(putRequest);
-    }
-  }
-
-  /**
-   * Removes ACLs from a bucket.
-   *
-   * @param bucketName - Bucket Name
-   * @param acls - Acls to be removed
-   *
-   * @throws OzoneException
-   */
-  public void removeAcls(String bucketName, String[] acls)
-      throws OzoneException {
-    HttpPut putRequest = null;
-    try (CloseableHttpClient httpClient = newHttpClient()) {
-      OzoneUtils.verifyResourceName(bucketName);
-      URIBuilder builder = new URIBuilder(getClient().getEndPointURI());
-      builder.setPath("/" + getVolumeName() + "/" + bucketName).build();
-      putRequest = client.getHttpPut(builder.toString());
-
-      for (String acl : acls) {
-        putRequest
-            .addHeader(Header.OZONE_ACLS, Header.OZONE_ACL_REMOVE + " " + acl);
-      }
-      executePutBucket(putRequest, httpClient);
-    } catch (URISyntaxException | IOException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      OzoneClientUtils.releaseConnection(putRequest);
-    }
-  }
-
-  /**
-   * Returns information about an existing bucket.
-   *
-   * @param bucketName - BucketName
-   *
-   * @return OZoneBucket
-   */
-  public OzoneBucket getBucket(String bucketName) throws OzoneException {
-    HttpGet getRequest = null;
-    try (CloseableHttpClient httpClient = newHttpClient()) {
-      OzoneUtils.verifyResourceName(bucketName);
-      URIBuilder builder = new URIBuilder(getClient().getEndPointURI());
-      builder.setPath("/" + getVolumeName() + "/" + bucketName)
-        .setParameter(Header.OZONE_INFO_QUERY_TAG,
-            Header.OZONE_INFO_QUERY_BUCKET).build();
-      getRequest = client.getHttpGet(builder.toString());
-      return executeInfoBucket(getRequest, httpClient);
-
-    } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      OzoneClientUtils.releaseConnection(getRequest);
-    }
-  }
-
-
-  /**
-   * Execute the info bucket call.
-   *
-   * @param getRequest - httpGet Request
-   * @param httpClient - Http Client
-   *
-   * @return OzoneBucket
-   *
-   * @throws IOException
-   * @throws OzoneException
-   */
-  private OzoneBucket executeInfoBucket(HttpGet getRequest,
-      CloseableHttpClient httpClient)
-      throws IOException, OzoneException {
-    HttpEntity entity = null;
-    try {
-      HttpResponse response = httpClient.execute(getRequest);
-      int errorCode = response.getStatusLine().getStatusCode();
-      entity = response.getEntity();
-      if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
-      }
-      if ((errorCode == HTTP_OK) || (errorCode == HTTP_CREATED)) {
-        OzoneBucket bucket =
-            new OzoneBucket(BucketInfo.parse(EntityUtils.toString(entity)),
-                this);
-        return bucket;
-      }
-      throw OzoneException.parse(EntityUtils.toString(entity));
-    } finally {
-      if (entity != null) {
-        EntityUtils.consumeQuietly(entity);
-      }
-    }
-  }
-
-  /**
-   * Execute the put bucket call.
-   *
-   * @param putRequest - http put request
-   * @param httpClient - Http Client
-   *
-   * @return OzoneBucket
-   *
-   * @throws IOException
-   * @throws OzoneException
-   */
-  private void executePutBucket(HttpPut putRequest,
-      CloseableHttpClient httpClient)
-      throws IOException, OzoneException {
-    HttpEntity entity = null;
-    try {
-      HttpResponse response = httpClient.execute(putRequest);
-      int errorCode = response.getStatusLine().getStatusCode();
-      entity = response.getEntity();
-
-      if (errorCode == HTTP_OK) {
-        return;
-      }
-
-      if (entity != null) {
-        throw OzoneException.parse(EntityUtils.toString(entity));
-      }
-
-      throw new OzoneRestClientException("Unexpected null in http result");
-    } finally {
-      if (entity != null) {
-        EntityUtils.consumeQuietly(entity);
-      }
-    }
-  }
-
-  /**
-   * Gets a list of buckets on this volume.
-   *
-   * @return - List of buckets
-   *
-   * @throws OzoneException
-   */
-  public List<OzoneBucket> listBuckets(String resultLength,
-      String previousBucket, String prefix) throws OzoneException {
-    HttpGet getRequest = null;
-    try (CloseableHttpClient httpClient = newHttpClient()) {
-      URIBuilder builder = new URIBuilder(getClient().getEndPointURI());
-      builder.setPath("/" + getVolumeName()).build();
-      if (!Strings.isNullOrEmpty(resultLength)) {
-        builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength);
-      }
-      if (!Strings.isNullOrEmpty(previousBucket)) {
-        builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, previousBucket);
-      }
-      if (!Strings.isNullOrEmpty(prefix)) {
-        builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
-      }
-
-      getRequest = client.getHttpGet(builder.toString());
-      return executeListBuckets(getRequest, httpClient);
-
-    } catch (IOException | URISyntaxException e) {
-      throw new OzoneRestClientException(e.getMessage(), e);
-    } finally {
-      OzoneClientUtils.releaseConnection(getRequest);
-    }
-  }
-
-  /**
-   * executes the List Bucket Call.
-   *
-   * @param getRequest - http Request
-   * @param httpClient - http Client
-   *
-   * @return List of OzoneBuckets
-   *
-   * @throws IOException
-   * @throws OzoneException
-   */
-  private List<OzoneBucket> executeListBuckets(HttpGet getRequest,
-      CloseableHttpClient httpClient)
-      throws IOException, OzoneException {
-    HttpEntity entity = null;
-    List<OzoneBucket> ozoneBucketList = new LinkedList<OzoneBucket>();
-    try {
-      HttpResponse response = httpClient.execute(getRequest);
-      int errorCode = response.getStatusLine().getStatusCode();
-
-      entity = response.getEntity();
-
-      if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
-      }
-      if (errorCode == HTTP_OK) {
-        ListBuckets bucketList =
-            ListBuckets.parse(EntityUtils.toString(entity));
-
-        for (BucketInfo info : bucketList.getBuckets()) {
-          ozoneBucketList.add(new OzoneBucket(info, this));
-        }
-        return ozoneBucketList;
-
-      } else {
-        throw OzoneException.parse(EntityUtils.toString(entity));
-      }
-    } finally {
-      if (entity != null) {
-        EntityUtils.consumeQuietly(entity);
-      }
-    }
-  }
-
-  /**
-   * Delete an empty bucket.
-   *
-   * @param bucketName - Name of the bucket to delete
-   *
-   * @throws OzoneException
-   */
-  public void deleteBucket(String bucketName) throws OzoneException {
-    HttpDelete delRequest = null;
-    try (CloseableHttpClient httpClient = newHttpClient()) {
-      OzoneUtils.verifyResourceName(bucketName);
-      URIBuilder builder = new URIBuilder(getClient().getEndPointURI());
-      builder.setPath("/" + getVolumeName() + "/" + bucketName).build();
-
-      delRequest = client.getHttpDelete(builder.toString());
-      executeDeleteBucket(delRequest, httpClient);
-
-    } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
-    } finally {
-      OzoneClientUtils.releaseConnection(delRequest);
-    }
-  }
-
-  /**
-   * Executes delete bucket call.
-   *
-   * @param delRequest - Delete Request
-   * @param httpClient - Http Client
-7   *
-   * @throws IOException
-   * @throws OzoneException
-   */
-  private void executeDeleteBucket(HttpDelete delRequest,
-      CloseableHttpClient httpClient)
-      throws IOException, OzoneException {
-    HttpEntity entity = null;
-    try {
-      HttpResponse response = httpClient.execute(delRequest);
-      int errorCode = response.getStatusLine().getStatusCode();
-      entity = response.getEntity();
-
-      if (errorCode == HTTP_OK) {
-        return;
-      }
-
-      if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload.");
-      }
-
-      throw OzoneException.parse(EntityUtils.toString(entity));
-
-    } finally {
-      if (entity != null) {
-        EntityUtils.consumeQuietly(entity);
-      }
-    }
-  }
-
-  @VisibleForTesting
-  public CloseableHttpClient newHttpClient() {
-    return OzoneClientUtils.newHttpClient();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/package-info.java
deleted file mode 100644
index 046568b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/package-info.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-
-/**
- * Ozone client library is a java client for the Ozone
- * Object Store.
- */
-package org.apache.hadoop.ozone.web.client;
-
-/**
- This library is  a simple Ozone REST Library.
-
- This library is a very *minimal* client written for tests and
- command line utils that work against Ozone. It does not have
- things like thread-pools and support for extended security models yet.
-
- OzoneClients return OzoneVolumes and OzoneVolumes return OzoneBuckets.
- **/

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
deleted file mode 100644
index 16892e7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.exceptions;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-
-import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
-import static java.net.HttpURLConnection.HTTP_CONFLICT;
-import static java.net.HttpURLConnection.HTTP_FORBIDDEN;
-import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
-import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
-import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED;
-
-/**
- * Error Table represents the Errors from Ozone Rest API layer.
- *
- * Please note : The errors in this table are sorted by the HTTP_ERROR codes
- * if you add new error codes to this table please follow the same convention.
- */
-@InterfaceAudience.Private
-public final class ErrorTable {
-
-  /* Error 400 */
-  public static final OzoneException MISSING_VERSION =
-      new OzoneException(HTTP_BAD_REQUEST, "missingVersion",
-                         "x-ozone-version header is required.");
-
-  public static final OzoneException MISSING_DATE =
-      new OzoneException(HTTP_BAD_REQUEST, "missingDate",
-                         "Date header is required.");
-
-  public static final OzoneException BAD_DATE =
-      new OzoneException(HTTP_BAD_REQUEST, "badDate",
-                         "Unable to parse date format.");
-
-  public static final OzoneException MALFORMED_QUOTA =
-      new OzoneException(HTTP_BAD_REQUEST, "malformedQuota",
-                         "Invalid quota specified.");
-
-  public static final OzoneException MALFORMED_ACL =
-      new OzoneException(HTTP_BAD_REQUEST, "malformedACL",
-                         "Invalid ACL specified.");
-
-
-  public static final OzoneException INVALID_VOLUME_NAME =
-      new OzoneException(HTTP_BAD_REQUEST, "invalidVolumeName",
-                         "Invalid volume name.");
-
-  public static final OzoneException INVALID_QUERY_PARAM =
-      new OzoneException(HTTP_BAD_REQUEST, "invalidQueryParam",
-                         "Invalid query parameter.");
-
-  public static final OzoneException INVALID_RESOURCE_NAME =
-      new OzoneException(HTTP_BAD_REQUEST, "invalidResourceName",
-                         "Invalid volume, bucket or key name.");
-
-  public static final OzoneException INVALID_BUCKET_NAME =
-      new OzoneException(HTTP_BAD_REQUEST, "invalidBucketName",
-                         "Invalid bucket name.");
-
-  public static final OzoneException INVALID_KEY =
-      new OzoneException(HTTP_BAD_REQUEST, "invalidKey", "Invalid key.");
-
-  public static final OzoneException INVALID_REQUEST =
-      new OzoneException(HTTP_BAD_REQUEST, "invalidRequest",
-                         "Error in request.");
-
-  public static final OzoneException MALFORMED_BUCKET_VERSION =
-      new OzoneException(HTTP_BAD_REQUEST, "malformedBucketVersion",
-                         "Malformed bucket version or version not unique.");
-
-  public static final OzoneException MALFORMED_STORAGE_TYPE =
-      new OzoneException(HTTP_BAD_REQUEST, "malformedStorageType",
-                         "Invalid storage Type specified.");
-
-  public static final OzoneException MALFORMED_STORAGE_CLASS =
-      new OzoneException(HTTP_BAD_REQUEST, "malformedStorageClass",
-                         "Invalid storage class specified.");
-
-  public static final OzoneException BAD_DIGEST =
-      new OzoneException(HTTP_BAD_REQUEST, "badDigest",
-                         "Content MD5 does not match.");
-
-  public static final OzoneException INCOMPLETE_BODY =
-      new OzoneException(HTTP_BAD_REQUEST, "incompleteBody",
-                         "Content length does not match stream size.");
-
-  public static final OzoneException BAD_AUTHORIZATION =
-      new OzoneException(HTTP_BAD_REQUEST, "badAuthorization",
-                         "Missing authorization or authorization has to be " +
-                             "unique.");
-
-  public static final OzoneException BAD_PROPERTY =
-      new OzoneException(HTTP_BAD_REQUEST, "unknownProperty",
-          "This property is not supported by this server.");
-
-  /* Error 401 */
-  public static final OzoneException UNAUTHORIZED =
-      new OzoneException(HTTP_UNAUTHORIZED, "Unauthorized",
-                         "Access token is missing or invalid token.");
-
-  /* Error 403 */
-  public static final OzoneException ACCESS_DENIED =
-      new OzoneException(HTTP_FORBIDDEN, "accessDenied", "Access denied.");
-
-  /* Error 404 */
-  public static final OzoneException USER_NOT_FOUND =
-      new OzoneException(HTTP_NOT_FOUND, "userNotFound", "Invalid user name.");
-
-  public static final OzoneException VOLUME_NOT_FOUND =
-      new OzoneException(HTTP_NOT_FOUND, "volumeNotFound", "No such volume.");
-
-  /* Error 409 */
-  public static final OzoneException VOLUME_ALREADY_EXISTS =
-      new OzoneException(HTTP_CONFLICT, "volumeAlreadyExists",
-                         "Duplicate volume name.");
-
-  public static final OzoneException BUCKET_ALREADY_EXISTS =
-      new OzoneException(HTTP_CONFLICT, "bucketAlreadyExists",
-                         "Duplicate bucket name.");
-
-  public static final OzoneException VOLUME_NOT_EMPTY =
-      new OzoneException(HTTP_CONFLICT, "volumeNotEmpty",
-                         "Volume must not have any buckets.");
-
-  public static final OzoneException BUCKET_NOT_EMPTY =
-      new OzoneException(HTTP_CONFLICT, "bucketNotEmpty",
-                         "Bucket must not have any keys.");
-
-  public static final OzoneException KEY_OPERATION_CONFLICT =
-      new OzoneException(HTTP_CONFLICT, "keyOperationConflict",
-                         "Conflicting operation on the specified key is going" +
-                             " on.");
-
-  /* Error 500 */
-  public static final OzoneException SERVER_ERROR =
-      new OzoneException(HTTP_INTERNAL_ERROR, "internalServerError",
-                         "Internal server error.");
-
-  /**
-   * Create a new instance of Error.
-   *
-   * @param e Error Template
-   * @param requestID Request ID
-   * @param resource Resource Name
-   * @param hostID hostID
-   *
-   * @return creates a new instance of error based on the template
-   */
-  public static OzoneException newError(OzoneException e, String requestID,
-                                        String resource, String hostID) {
-    OzoneException err =
-        new OzoneException(e.getHttpCode(), e.getShortMessage(),
-                           e.getMessage());
-    err.setRequestId(requestID);
-    err.setResource(resource);
-    err.setHostID(hostID);
-    return err;
-  }
-
-  /**
-   * Create new instance of Error.
-   *
-   * @param e - Error Template
-   * @param args - Args
-   *
-   * @return Ozone Exception
-   */
-  public static OzoneException newError(OzoneException e, UserArgs args) {
-    OzoneException err =
-        new OzoneException(e.getHttpCode(), e.getShortMessage(),
-                           e.getMessage());
-    if (args != null) {
-      err.setRequestId(args.getRequestID());
-      err.setResource(args.getResourceName());
-      err.setHostID(args.getHostName());
-    }
-    return err;
-  }
-
-  /**
-   * Create new instance of Error.
-   *
-   * @param e - Error Template
-   * @param args - Args
-   * @param ex Exception
-   *
-   * @return Ozone Exception
-   */
-  public static OzoneException newError(OzoneException e, UserArgs args,
-                                        Exception ex) {
-    OzoneException err =
-        new OzoneException(e.getHttpCode(), e.getShortMessage(), ex);
-
-    if(args != null) {
-      err.setRequestId(args.getRequestID());
-      err.setResource(args.getResourceName());
-      err.setHostID(args.getHostName());
-    }
-    err.setMessage(ex.getMessage());
-    return err;
-  }
-
-  private ErrorTable() {
-    // Never constructed.
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/package-info.java
deleted file mode 100644
index 59cf724..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.exceptions;
-
-/**
- This package contains ozone client side libraries.
- */


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[16/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
deleted file mode 100644
index 0d5248d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.OzoneConsts;
-
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * BucketArgs packages all bucket related arguments to
- * file system calls.
- */
-public class BucketArgs extends VolumeArgs {
-  private final String bucketName;
-  private List<OzoneAcl> addAcls;
-  private List<OzoneAcl> removeAcls;
-  private OzoneConsts.Versioning versioning;
-  private StorageType storageType;
-
-  /**
-   * Constructor for BucketArgs.
-   *
-   * @param volumeName - volumeName
-   * @param bucketName - bucket Name
-   * @param userArgs - userArgs
-   */
-  public BucketArgs(String volumeName, String bucketName, UserArgs userArgs) {
-    super(volumeName, userArgs);
-    this.bucketName = bucketName;
-    this.versioning = OzoneConsts.Versioning.NOT_DEFINED;
-    this.storageType = null;
-  }
-
-
-  /**
-   * Constructor for BucketArgs.
-   *
-   * @param bucketName - bucket Name
-   * @param volumeArgs - volume Args
-   */
-  public BucketArgs(String bucketName, VolumeArgs volumeArgs) {
-    super(volumeArgs);
-    this.bucketName = bucketName;
-    this.versioning = OzoneConsts.Versioning.NOT_DEFINED;
-    this.storageType = null;
-  }
-
-  /**
-   * Constructor for BucketArgs.
-   *
-   * @param args - Bucket Args
-   */
-  public BucketArgs(BucketArgs args) {
-    this(args.getBucketName(), args);
-    this.setAddAcls(args.getAddAcls());
-    this.setRemoveAcls(args.getRemoveAcls());
-  }
-
-  /**
-   * Returns the Bucket Name.
-   *
-   * @return Bucket Name
-   */
-  public String getBucketName() {
-    return bucketName;
-  }
-
-  /**
-   * Returns Additive ACLs for the Bucket if specified.
-   *
-   * @return acls
-   */
-  public List<OzoneAcl> getAddAcls() {
-    return addAcls;
-  }
-
-  /**
-   * Set Additive ACLs.
-   *
-   * @param acl - ACL
-   */
-  public void setAddAcls(List<OzoneAcl> acl) {
-    this.addAcls = acl;
-  }
-
-  /**
-   * Returns remove ACLs for the Bucket if specified.
-   *
-   * @return acls
-   */
-  public List<OzoneAcl> getRemoveAcls() {
-    return removeAcls;
-  }
-
-  /**
-   * Takes an ACL and sets the ACL object to ACL represented by the String.
-   *
-   * @param aclString - aclString
-   */
-  public void addAcls(List<String> aclString) throws IllegalArgumentException {
-    if (aclString == null) {
-      throw new IllegalArgumentException("ACLs cannot be null");
-    }
-    if (this.addAcls == null) {
-      this.addAcls = new LinkedList<>();
-    }
-    for (String s : aclString) {
-      this.addAcls.add(OzoneAcl.parseAcl(s));
-    }
-  }
-
-  /**
-   * Takes an ACL and sets the ACL object to ACL represented by the String.
-   *
-   * @param aclString - aclString
-   */
-  public void removeAcls(List<String> aclString)
-      throws IllegalArgumentException {
-    if (aclString == null) {
-      throw new IllegalArgumentException("ACLs cannot be null");
-    }
-    if (this.removeAcls == null) {
-      this.removeAcls = new LinkedList<>();
-    }
-    for (String s : aclString) {
-      this.removeAcls.add(OzoneAcl.parseAcl(s));
-    }
-  }
-
-  /**
-   * Set remove ACLs.
-   *
-   * @param acl - ACL
-   */
-  public void setRemoveAcls(List<OzoneAcl> acl) {
-    this.removeAcls = acl;
-  }
-
-
-  /**
-   * Returns Versioning Info.
-   *
-   * @return versioning
-   */
-  public OzoneConsts.Versioning getVersioning() {
-    return versioning;
-  }
-
-
-  /**
-   * SetVersioning Info.
-   *
-   * @param versioning - Enum value
-   */
-  public void setVersioning(OzoneConsts.Versioning versioning) {
-    this.versioning = versioning;
-  }
-
-  /**
-   * returns the current Storage Class.
-   *
-   * @return Storage Class
-   */
-  public StorageType getStorageType() {
-    return storageType;
-  }
-
-  /**
-   * Sets the Storage Class.
-   *
-   * @param storageType Set Storage Class
-   */
-  public void setStorageType(StorageType storageType) {
-    this.storageType = storageType;
-  }
-
-  /**
-   * returns - Volume/bucketName.
-   *
-   * @return String
-   */
-  @Override
-  public String getResourceName() {
-    return getVolumeName() + "/" + getBucketName();
-  }
-
-  /**
-   * Returns User/Volume name which is the parent of this
-   * bucket.
-   *
-   * @return String
-   */
-  public String getParentName() {
-    return getUserName() + "/" + getVolumeName();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
deleted file mode 100644
index d090499..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- *
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.interfaces.Bucket;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.slf4j.MDC;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-
-import static java.net.HttpURLConnection.HTTP_CREATED;
-import static java.net.HttpURLConnection.HTTP_OK;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_FUNCTION;
-
-
-/**
- * Bucket Class handles all ozone Bucket related actions.
- */
-public class BucketHandler implements Bucket {
-  /**
-   * createBucket call handles the POST request for Creating a Bucket.
-   *
-   * @param volume - Volume name
-   * @param bucket - Bucket Name
-   * @param req - Http request
-   * @param info - Uri Info
-   * @param headers - Http headers
-   *
-   * @return Response
-   *
-   * @throws OzoneException
-   */
-  @Override
-  public Response createBucket(String volume, String bucket, Request req,
-                               UriInfo info, HttpHeaders headers)
-      throws OzoneException {
-    MDC.put(OZONE_FUNCTION, "createBucket");
-    return new BucketProcessTemplate() {
-      @Override
-      public Response doProcess(BucketArgs args)
-          throws OzoneException, IOException {
-        StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
-        getAclsFromHeaders(args, false);
-        args.setVersioning(getVersioning(args));
-        args.setStorageType(getStorageType(args));
-        fs.createBucket(args);
-        return OzoneUtils.getResponse(args, HTTP_CREATED, "");
-      }
-    }.handleCall(volume, bucket, req, info, headers);
-  }
-
-  /**
-   * updateBucket call handles the PUT request for updating a Bucket.
-   *
-   * There are only three possible actions currently with updateBucket.
-   * They are add/remove on ACLS, Bucket Versioning and  StorageType.
-   *  if you make a call with any other action, update just returns 200 OK.
-   *
-   * @param volume - Storage volume name
-   * @param bucket - Bucket name
-   * @param req - Http request
-   * @param info - Uri Info
-   * @param headers - Http headers
-   *
-   * @return Response
-   *
-   * @throws OzoneException
-   */
-  @Override
-  public Response updateBucket(String volume, String bucket, Request req,
-                               UriInfo info, HttpHeaders headers)
-      throws OzoneException {
-    MDC.put(OZONE_FUNCTION, "updateBucket");
-    return new BucketProcessTemplate() {
-      @Override
-      public Response doProcess(BucketArgs args)
-          throws OzoneException, IOException {
-        StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
-        getAclsFromHeaders(args, true);
-        args.setVersioning(getVersioning(args));
-        args.setStorageType(getStorageType(args));
-
-        if ((args.getAddAcls() != null) || (args.getRemoveAcls() != null)) {
-          fs.setBucketAcls(args);
-        }
-
-        if (args.getVersioning() != OzoneConsts.Versioning.NOT_DEFINED) {
-          fs.setBucketVersioning(args);
-        }
-
-        if (args.getStorageType() != null) {
-          fs.setBucketStorageClass(args);
-        }
-        return OzoneUtils.getResponse(args, HTTP_OK, "");
-      }
-    }.handleCall(volume, bucket, req, info, headers);
-  }
-
-  /**
-   * Deletes an empty bucket.
-   *
-   * @param volume Volume name
-   * @param bucket Bucket Name
-   * @param req - Http request
-   * @param info - Uri Info
-   * @param headers - Http headers
-   *
-   * @return Response
-   *
-   * @throws OzoneException
-   */
-  @Override
-  public Response deleteBucket(String volume, String bucket, Request req,
-                               UriInfo info, HttpHeaders headers)
-      throws OzoneException {
-    MDC.put(OZONE_FUNCTION, "deleteBucket");
-    return new BucketProcessTemplate() {
-      @Override
-      public Response doProcess(BucketArgs args)
-          throws OzoneException, IOException {
-        StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
-        fs.deleteBucket(args);
-        return OzoneUtils.getResponse(args, HTTP_OK, "");
-      }
-    }.handleCall(volume, bucket, req, info, headers);
-  }
-
-  /**
-   * List Buckets allows the user to list the bucket.
-   *
-   * @param volume - Storage Volume Name
-   * @param bucket - Bucket Name
-   * @param info - Uri Info
-   * @param prefix - Prefix for the keys to be fetched
-   * @param maxKeys - MaxNumber of Keys to Return
-   * @param startPage - Continuation Token
-   * @param req - Http request
-   * @param headers - Http headers
-   *
-   * @return - Json Body
-   *
-   * @throws OzoneException
-   */
-  @Override
-  public Response listBucket(String volume, String bucket, final String info,
-                             final String prefix, final int maxKeys,
-                             final String startPage, Request req,
-                             UriInfo uriInfo, HttpHeaders headers)
-      throws OzoneException {
-    MDC.put(OZONE_FUNCTION, "listBucket");
-    return new BucketProcessTemplate() {
-      @Override
-      public Response doProcess(BucketArgs args)
-          throws OzoneException, IOException {
-        switch (info) {
-        case Header.OZONE_INFO_QUERY_KEY:
-          ListArgs listArgs = new ListArgs(args, prefix, maxKeys, startPage);
-          return getBucketKeysList(listArgs);
-        case Header.OZONE_INFO_QUERY_BUCKET:
-          return getBucketInfoResponse(args);
-        default:
-          OzoneException ozException =
-              ErrorTable.newError(ErrorTable.INVALID_QUERY_PARAM, args);
-          ozException.setMessage("Unrecognized query param : " + info);
-          throw ozException;
-        }
-      }
-    }.handleCall(volume, bucket, req, uriInfo, headers);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketProcessTemplate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketProcessTemplate.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketProcessTemplate.java
deleted file mode 100644
index 4736746..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketProcessTemplate.java
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.interfaces.UserAuth;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.response.ListKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-import java.nio.file.DirectoryNotEmptyException;
-import java.nio.file.FileAlreadyExistsException;
-import java.nio.file.NoSuchFileException;
-import java.util.LinkedList;
-import java.util.List;
-
-import static java.net.HttpURLConnection.HTTP_OK;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_COMPONENT;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RESOURCE;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_REQUEST;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_USER;
-
-/**
- * This class abstracts way the repetitive tasks in
- * Bucket handling code.
- */
-public abstract class BucketProcessTemplate {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(BucketProcessTemplate.class);
-
-  /**
-   * This function serves as the common error handling function
-   * for all bucket related operations.
-   *
-   * @param volume - Volume Name
-   * @param bucket - Bucket Name
-   * @param request - Http Request
-   * @param uriInfo - Http Uri
-   * @param headers - Http Headers
-   *
-   * @return Response
-   *
-   * @throws OzoneException
-   */
-  public Response handleCall(String volume, String bucket, Request request,
-                             UriInfo uriInfo, HttpHeaders headers)
-      throws OzoneException {
-    // TODO : Add logging
-    String reqID = OzoneUtils.getRequestID();
-    String hostName = OzoneUtils.getHostName();
-    MDC.put(OZONE_COMPONENT, "ozone");
-    MDC.put(OZONE_REQUEST, reqID);
-    UserArgs userArgs = null;
-    try {
-      userArgs = new UserArgs(reqID, hostName, request, uriInfo, headers);
-
-      OzoneUtils.validate(request, headers, reqID, bucket, hostName);
-      OzoneUtils.verifyResourceName(bucket);
-
-      UserAuth auth = UserHandlerBuilder.getAuthHandler();
-      userArgs.setUserName(auth.getUser(userArgs));
-      MDC.put(OZONE_USER, userArgs.getUserName());
-
-      BucketArgs args = new BucketArgs(volume, bucket, userArgs);
-      MDC.put(OZONE_RESOURCE, args.getResourceName());
-      Response response =  doProcess(args);
-      LOG.debug("Success");
-      MDC.clear();
-      return response;
-
-    } catch (IllegalArgumentException argEx) {
-      LOG.error("Invalid bucket.", argEx);
-      throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, userArgs,
-          argEx);
-    } catch (IOException fsExp) {
-      handleIOException(bucket, reqID, hostName, fsExp);
-    }
-    return null;
-  }
-
-  /**
-   * Reads ACLs from headers and throws appropriate exception if needed.
-   *
-   * @param args - bucketArgs
-   *
-   * @throws OzoneException
-   */
-  void getAclsFromHeaders(BucketArgs args, boolean parseRemoveACL)
-      throws OzoneException {
-    try {
-      List<String> acls = getAcls(args, Header.OZONE_ACL_REMOVE);
-      if (acls != null && !acls.isEmpty()) {
-        args.removeAcls(acls);
-      }
-      if ((!parseRemoveACL) && args.getRemoveAcls() != null) {
-        OzoneException ex = ErrorTable.newError(ErrorTable.MALFORMED_ACL, args);
-        ex.setMessage("Invalid Remove ACLs");
-        throw ex;
-      }
-
-      acls = getAcls(args, Header.OZONE_ACL_ADD);
-      if (acls != null && !acls.isEmpty()) {
-        args.addAcls(acls);
-      }
-    } catch (IllegalArgumentException ex) {
-      throw ErrorTable.newError(ErrorTable.MALFORMED_ACL, args, ex);
-    }
-  }
-
-  /**
-   * Converts FileSystem IO exceptions to OZONE exceptions.
-   *
-   * @param bucket Name of the bucket
-   * @param reqID Request ID
-   * @param hostName Machine Name
-   * @param fsExp Exception
-   *
-   * @throws OzoneException
-   */
-  void handleIOException(String bucket, String reqID, String hostName,
-                         IOException fsExp) throws OzoneException {
-    LOG.error("IOException:", fsExp);
-
-    OzoneException exp = null;
-    if (fsExp instanceof FileAlreadyExistsException) {
-      exp = ErrorTable
-          .newError(ErrorTable.BUCKET_ALREADY_EXISTS, reqID, bucket, hostName);
-    }
-
-    if (fsExp instanceof DirectoryNotEmptyException) {
-      exp = ErrorTable
-          .newError(ErrorTable.BUCKET_NOT_EMPTY, reqID, bucket, hostName);
-    }
-
-    if (fsExp instanceof NoSuchFileException) {
-      exp = ErrorTable
-          .newError(ErrorTable.INVALID_BUCKET_NAME, reqID, bucket, hostName);
-    }
-
-    // Default we don't handle this exception yet,
-    // report a Server Internal Error.
-    if (exp == null) {
-      exp =
-          ErrorTable.newError(ErrorTable.SERVER_ERROR, reqID, bucket, hostName);
-      if (fsExp != null) {
-        exp.setMessage(fsExp.getMessage());
-      }
-    }
-    throw exp;
-  }
-
-  /**
-   * Abstract function that gets implemented in the BucketHandler functions.
-   * This function will just deal with the core file system related logic
-   * and will rely on handleCall function for repetitive error checks
-   *
-   * @param args - parsed bucket args, name, userName, ACLs etc
-   *
-   * @return Response
-   *
-   * @throws OzoneException
-   * @throws IOException
-   */
-  public abstract Response doProcess(BucketArgs args)
-      throws OzoneException, IOException;
-
-
-  /**
-   * Returns the ACL String if available.
-   * This function ignores all ACLs that are not prefixed with either
-   * ADD or Remove
-   *
-   * @param args - BucketArgs
-   * @param tag - Tag for different type of acls
-   *
-   * @return List of ACLs
-   *
-   */
-  List<String> getAcls(BucketArgs args, String tag)  {
-    List<String> aclStrings =
-        args.getHeaders().getRequestHeader(Header.OZONE_ACLS);
-    List<String> filteredSet = null;
-    if (aclStrings != null) {
-      filteredSet = new LinkedList<>();
-      for (String s : aclStrings) {
-        if (s.startsWith(tag)) {
-          filteredSet.add(s.replaceFirst(tag, ""));
-        }
-      }
-    }
-    return filteredSet;
-  }
-
-  /**
-   * Returns bucket versioning Info.
-   *
-   * @param args - BucketArgs
-   *
-   * @return - String
-   *
-   * @throws OzoneException
-   */
-  OzoneConsts.Versioning getVersioning(BucketArgs args) throws OzoneException {
-
-    List<String> versionStrings =
-        args.getHeaders().getRequestHeader(Header.OZONE_BUCKET_VERSIONING);
-    if (versionStrings == null) {
-      return null;
-    }
-
-    if (versionStrings.size() > 1) {
-      OzoneException ex =
-          ErrorTable.newError(ErrorTable.MALFORMED_BUCKET_VERSION, args);
-      ex.setMessage("Exactly one bucket version header required");
-      throw ex;
-    }
-
-    String version = versionStrings.get(0);
-    try {
-      return OzoneConsts.Versioning.valueOf(version);
-    } catch (IllegalArgumentException ex) {
-      LOG.debug("Malformed Version. version: {}", version);
-      throw ErrorTable.newError(ErrorTable.MALFORMED_BUCKET_VERSION, args, ex);
-    }
-  }
-
-
-  /**
-   * Returns Storage Class if Available or returns Default.
-   *
-   * @param args - bucketArgs
-   *
-   * @return StorageType
-   *
-   * @throws OzoneException
-   */
-  StorageType getStorageType(BucketArgs args) throws OzoneException {
-    List<String> storageClassString = null;
-    try {
-      storageClassString =
-          args.getHeaders().getRequestHeader(Header.OZONE_STORAGE_TYPE);
-
-      if (storageClassString == null) {
-        return null;
-      }
-      if (storageClassString.size() > 1) {
-        OzoneException ex =
-            ErrorTable.newError(ErrorTable.MALFORMED_STORAGE_TYPE, args);
-        ex.setMessage("Exactly one storage class header required");
-        throw ex;
-      }
-      return StorageType.valueOf(storageClassString.get(0).toUpperCase());
-    } catch (IllegalArgumentException ex) {
-      if(storageClassString != null) {
-        LOG.debug("Malformed storage type. Type: {}",
-            storageClassString.get(0).toUpperCase());
-      }
-      throw ErrorTable.newError(ErrorTable.MALFORMED_STORAGE_TYPE, args, ex);
-    }
-  }
-
-  /**
-   * Returns BucketInfo response.
-   *
-   * @param args - BucketArgs
-   *
-   * @return BucketInfo
-   *
-   * @throws IOException
-   * @throws OzoneException
-   */
-  Response getBucketInfoResponse(BucketArgs args)
-      throws IOException, OzoneException {
-    StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
-    BucketInfo info = fs.getBucketInfo(args);
-    return OzoneUtils.getResponse(args, HTTP_OK, info.toJsonString());
-  }
-
-  /**
-   * Returns list of objects in a bucket.
-   * @param args - ListArgs
-   * @return Response
-   * @throws IOException
-   * @throws OzoneException
-   */
-  Response getBucketKeysList(ListArgs args) throws IOException, OzoneException {
-    StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
-    ListKeys objects = fs.listKeys(args);
-    return OzoneUtils.getResponse(args.getArgs(), HTTP_OK,
-        objects.toJsonString());
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyArgs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyArgs.java
deleted file mode 100644
index 48a4cb4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyArgs.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-/**
- * Class that packages all key Arguments.
- */
-public class KeyArgs extends BucketArgs {
-  private String key;
-  private String hash;
-  private long size;
-
-  /**
-   * Constructor for Key Args.
-   *
-   * @param volumeName - Volume Name
-   * @param bucketName - Bucket Name
-   * @param objectName - Key
-   */
-  public KeyArgs(String volumeName, String bucketName,
-                 String objectName, UserArgs args) {
-    super(volumeName, bucketName, args);
-    this.key = objectName;
-  }
-
-  /**
-   * Constructor for Key Args.
-   *
-   * @param objectName - Key
-   * @param args - Bucket Args
-   */
-  public KeyArgs(String objectName, BucketArgs args) {
-    super(args);
-    this.key = objectName;
-  }
-
-  /**
-   * Get Key Name.
-   *
-   * @return String
-   */
-  public String getKeyName() {
-    return this.key;
-  }
-
-  /**
-   * Computed File hash.
-   *
-   * @return String
-   */
-  public String getHash() {
-    return hash;
-  }
-
-  /**
-   * Sets the hash String.
-   *
-   * @param hash String
-   */
-  public void setHash(String hash) {
-    this.hash = hash;
-  }
-
-  /**
-   * Returns the file size.
-   *
-   * @return long - file size
-   */
-  public long getSize() {
-    return size;
-  }
-
-  /**
-   * Set Size.
-   *
-   * @param size Size of the file
-   */
-  public void setSize(long size) {
-    this.size = size;
-  }
-
-  /**
-   * Returns the name of the resource.
-   *
-   * @return String
-   */
-  @Override
-  public String getResourceName() {
-    return super.getResourceName() + "/" + getKeyName();
-  }
-
-  /**
-   * Parent name of this resource.
-   *
-   * @return String.
-   */
-  @Override
-  public String getParentName() {
-    return super.getResourceName();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java
deleted file mode 100644
index eff2759..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.commons.codec.binary.Hex;
-import org.apache.hadoop.ozone.client.io.LengthInputStream;
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.interfaces.Keys;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-
-import static java.net.HttpURLConnection.HTTP_CREATED;
-import static java.net.HttpURLConnection.HTTP_OK;
-
-/**
- * KeyHandler deals with basic Key Operations.
- */
-public class KeyHandler implements Keys {
-
-  /**
-   * Gets the Key/key information if it exists.
-   *
-   * @param volume  Storage Volume
-   * @param bucket  Name of the bucket
-   * @param key Name of the key
-   * @param info Tag info
-   * @param req Request
-   * @param uriInfo Uri Info
-   * @param headers Http Header
-   * @return Response
-   * @throws OzoneException
-   */
-  @Override
-  public Response getKey(String volume, String bucket, String key, String info,
-      Request req, UriInfo uriInfo, HttpHeaders headers)
-      throws OzoneException {
-    return new KeyProcessTemplate() {
-      /**
-       * Abstract function that gets implemented in the KeyHandler functions.
-       * This function will just deal with the core file system related logic
-       * and will rely on handleCall function for repetitive error checks
-       *
-       * @param args - parsed bucket args, name, userName, ACLs etc
-       * @param input - The body as an Input Stream
-       * @param request - Http request
-       * @param headers - Parsed http Headers.
-       * @param uriInfo - UriInfo
-       *
-       * @return Response
-       *
-       * @throws IOException - From the file system operations
-       */
-      @Override
-      public Response doProcess(KeyArgs args, InputStream input,
-                                Request request, HttpHeaders headers,
-                                UriInfo uriInfo)
-          throws IOException, OzoneException, NoSuchAlgorithmException {
-        if (info == null) {
-          return getKey(args);
-        } else if (info.equals(Header.OZONE_INFO_QUERY_KEY)) {
-          return getKeyInfo(args);
-        }
-
-        OzoneException ozException = ErrorTable
-            .newError(ErrorTable.INVALID_QUERY_PARAM, args);
-        ozException.setMessage("Unrecognized query param : " + info);
-        throw ozException;
-      }
-    }.handleCall(volume, bucket, key, req, headers, uriInfo, null);
-  }
-
-  /**
-   * Gets the Key if it exists.
-   */
-  private Response getKey(KeyArgs args)
-      throws IOException, OzoneException {
-    StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
-    LengthInputStream stream = fs.newKeyReader(args);
-    return OzoneUtils.getResponse(args, HTTP_OK, stream);
-  }
-
-  /**
-   * Gets the Key information if it exists.
-   */
-  private Response getKeyInfo(KeyArgs args)
-      throws IOException, OzoneException {
-    StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
-    KeyInfo keyInfo = fs.getKeyInfo(args);
-    return OzoneUtils.getResponse(args, HTTP_OK, keyInfo.toJsonString());
-  }
-
-  /**
-   * Adds a key to an existing bucket. If the object already exists this call
-   * will overwrite or add with new version number if the bucket versioning is
-   * turned on.
-   *
-   * @param volume  Storage Volume Name
-   * @param bucket  Name of the bucket
-   * @param keys    Name of the Object
-   * @param is      InputStream or File Data
-   * @param req     Request
-   * @param info    - UriInfo
-   * @param headers http headers
-   * @return Response
-   * @throws OzoneException
-   */
-  @Override
-  public Response putKey(String volume, String bucket, String keys,
-                         InputStream is, Request req, UriInfo info,
-                         HttpHeaders headers) throws OzoneException {
-
-    return new KeyProcessTemplate() {
-      /**
-       * Abstract function that gets implemented in the KeyHandler functions.
-       * This function will just deal with the core file system related logic
-       * and will rely on handleCall function for repetitive error checks
-       *
-       * @param args - parsed bucket args, name, userName, ACLs etc
-       * @param input - The body as an Input Stream
-       * @param request - Http request
-       * @param headers - Parsed http Headers.
-       * @param info - UriInfo
-       *
-       * @return Response
-       *
-       * @throws IOException - From the file system operations
-       */
-      @Override
-      public Response doProcess(KeyArgs args, InputStream input,
-                                Request request, HttpHeaders headers,
-                                UriInfo info)
-          throws IOException, OzoneException, NoSuchAlgorithmException {
-        final int eof = -1;
-        StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
-
-        byte[] buffer = new byte[4 * 1024];
-        String contentLenString = getContentLength(headers, args);
-        String newLen = contentLenString.replaceAll("\"", "");
-        int contentLen = Integer.parseInt(newLen);
-        args.setSize(contentLen);
-
-        MessageDigest md5 = MessageDigest.getInstance("MD5");
-        int bytesRead = 0;
-        int len = 0;
-        OutputStream stream = fs.newKeyWriter(args);
-        while ((bytesRead < contentLen) && (len != eof)) {
-          int readSize =
-              (contentLen - bytesRead > buffer.length) ? buffer.length :
-                  contentLen - bytesRead;
-          len = input.read(buffer, 0, readSize);
-          if (len != eof) {
-            stream.write(buffer, 0, len);
-            md5.update(buffer, 0, len);
-            bytesRead += len;
-
-          }
-        }
-
-        checkFileLengthMatch(args, fs, contentLen, bytesRead);
-
-        String hashString = Hex.encodeHexString(md5.digest());
-// TODO : Enable hash value checking.
-//          String contentHash = getContentMD5(headers, args);
-//          checkFileHashMatch(args, hashString, fs, contentHash);
-        args.setHash(hashString);
-        args.setSize(bytesRead);
-        fs.commitKey(args, stream);
-        return OzoneUtils.getResponse(args, HTTP_CREATED, "");
-      }
-    }.handleCall(volume, bucket, keys, req, headers, info, is);
-  }
-
-  /**
-   * Deletes an existing key.
-   *
-   * @param volume  Storage Volume Name
-   * @param bucket  Name of the bucket
-   * @param keys    Name of the Object
-   * @param req     http Request
-   * @param info    - UriInfo
-   * @param headers HttpHeaders
-   * @return Response
-   * @throws OzoneException
-   */
-  @Override
-  public Response deleteKey(String volume, String bucket, String keys,
-                            Request req, UriInfo info, HttpHeaders headers)
-      throws OzoneException {
-    return new KeyProcessTemplate() {
-      /**
-       * Abstract function that gets implemented in the KeyHandler functions.
-       * This function will just deal with the core file system related logic
-       * and will rely on handleCall function for repetitive error checks
-       *
-       * @param args - parsed bucket args, name, userName, ACLs etc
-       * @param input - The body as an Input Stream
-       * @param request - Http request
-       * @param headers - Parsed http Headers.
-       * @param info - UriInfo
-       *
-       * @return Response
-       *
-       * @throws IOException - From the file system operations
-       */
-      @Override
-      public Response doProcess(KeyArgs args, InputStream input,
-                                Request request, HttpHeaders headers,
-                                UriInfo info)
-          throws IOException, OzoneException, NoSuchAlgorithmException {
-        StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
-        fs.deleteKey(args);
-        return OzoneUtils.getResponse(args, HTTP_OK, "");
-      }
-    }.handleCall(volume, bucket, keys, req, headers, info, null);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyProcessTemplate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyProcessTemplate.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyProcessTemplate.java
deleted file mode 100644
index dcf0fc5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyProcessTemplate.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos;
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.interfaces.UserAuth;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-import java.io.InputStream;
-import java.security.NoSuchAlgorithmException;
-import java.util.List;
-
-import static org.apache.hadoop.ozone.web.exceptions.ErrorTable.BAD_DIGEST;
-import static org.apache.hadoop.ozone.web.exceptions.ErrorTable.INCOMPLETE_BODY;
-import static org.apache.hadoop.ozone.web.exceptions.ErrorTable.INVALID_BUCKET_NAME;
-import static org.apache.hadoop.ozone.web.exceptions.ErrorTable.INVALID_REQUEST;
-import static org.apache.hadoop.ozone.web.exceptions.ErrorTable.SERVER_ERROR;
-import static org.apache.hadoop.ozone.web.exceptions.ErrorTable.newError;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_COMPONENT;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RESOURCE;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_REQUEST;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_USER;
-
-
-/**
- * This class abstracts way the repetitive tasks in  Key handling code.
- */
-public abstract class KeyProcessTemplate {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(KeyProcessTemplate.class);
-
-  /**
-   * This function serves as the common error handling function for all Key
-   * related operations.
-   *
-   * @param bucket  bucket Name
-   * @param key     the object name
-   * @param headers Http headers
-   * @param is      Input XML stream
-   * @throws OzoneException
-   */
-  public Response handleCall(String volume, String bucket, String key,
-                             Request request, HttpHeaders headers, UriInfo info,
-                             InputStream is) throws OzoneException {
-
-    String reqID = OzoneUtils.getRequestID();
-    String hostName = OzoneUtils.getHostName();
-    MDC.put(OZONE_COMPONENT, "ozone");
-    MDC.put(OZONE_REQUEST, reqID);
-    UserArgs userArgs = null;
-    try {
-      userArgs = new UserArgs(reqID, hostName, request, info, headers);
-      OzoneUtils.validate(request, headers, reqID, bucket, hostName);
-      OzoneUtils.verifyResourceName(bucket);
-
-      UserAuth auth = UserHandlerBuilder.getAuthHandler();
-      userArgs.setUserName(auth.getUser(userArgs));
-      MDC.put(OZONE_USER, userArgs.getUserName());
-
-      KeyArgs args = new KeyArgs(volume, bucket, key, userArgs);
-      MDC.put(OZONE_RESOURCE, args.getResourceName());
-      Response response =  doProcess(args, is, request, headers, info);
-      LOG.debug("Success");
-      MDC.clear();
-      return response;
-
-    } catch (IllegalArgumentException argExp) {
-      LOG.error("Invalid bucket in key call.", argExp);
-      throw newError(INVALID_BUCKET_NAME, userArgs, argExp);
-    } catch (IOException fsExp) {
-      LOG.error("IOException:", fsExp);
-      // Map KEY_NOT_FOUND to INVALID_KEY
-      if (fsExp.getMessage().endsWith(
-          KeySpaceManagerProtocolProtos.Status.KEY_NOT_FOUND.name())) {
-        throw ErrorTable.newError(ErrorTable.INVALID_KEY, userArgs, fsExp);
-      }
-
-      // TODO : Handle errors from the FileSystem , let us map to server error
-      // for now.
-      throw ErrorTable.newError(ErrorTable.SERVER_ERROR, userArgs, fsExp);
-    } catch (NoSuchAlgorithmException algoEx) {
-      LOG.error("NoSuchAlgorithmException. Probably indicates an unusual java "
-          + "installation.", algoEx);
-      throw ErrorTable.newError(SERVER_ERROR, userArgs, algoEx);
-    }
-  }
-
-  /**
-   * Abstract function that gets implemented in the KeyHandler functions. This
-   * function will just deal with the core file system related logic and will
-   * rely on handleCall function for repetitive error checks
-   *
-   * @param args    - parsed bucket args, name, userName, ACLs etc
-   * @param input   - The body as an Input Stream
-   * @param request - Http request
-   * @param headers - Parsed http Headers.
-   * @param info    - UriInfo
-   * @return Response
-   * @throws IOException - From the file system operations
-   */
-  public abstract Response doProcess(KeyArgs args, InputStream input,
-                                     Request request, HttpHeaders headers,
-                                     UriInfo info)
-      throws IOException, OzoneException, NoSuchAlgorithmException;
-
-  /**
-   * checks if the File Content-MD5 we wrote matches the hash we computed from
-   * the stream. if it does match we delete the file and throw and exception to
-   * let the user know that we have a hash mismatch
-   *
-   * @param args           Object Args
-   * @param computedString MD5 hash value
-   * @param fs             Pointer to File System so we can delete the file
-   * @param contentHash    User Specified hash string
-   * @throws IOException
-   * @throws OzoneException
-   */
-  public void checkFileHashMatch(KeyArgs args, String computedString,
-                                 StorageHandler fs, String contentHash)
-      throws IOException, OzoneException {
-    if (contentHash != null) {
-      String contentString =
-          new String(Base64.decodeBase64(contentHash), OzoneUtils.ENCODING)
-              .trim();
-
-      if (!contentString.equals(computedString)) {
-        fs.deleteKey(args);
-        OzoneException ex = ErrorTable.newError(BAD_DIGEST, args);
-        String msg = String.format("MD5 Digest mismatch. Expected %s Found " +
-            "%s", contentString, computedString);
-        ex.setMessage(msg);
-        LOG.debug(msg);
-        throw ex;
-      }
-    }
-  }
-
-  /**
-   * check if the content-length matches the actual stream length. if we find a
-   * mismatch we will delete the file and throw an exception to let the user
-   * know that length mismatch detected
-   *
-   * @param args       Object Args
-   * @param fs         Pointer to File System Object, to delete the file that we
-   *                   wrote
-   * @param contentLen Http Content-Length Header
-   * @param bytesRead  Actual Bytes we read from the stream
-   * @throws IOException
-   * @throws OzoneException
-   */
-  public void checkFileLengthMatch(KeyArgs args, StorageHandler fs,
-                                   int contentLen, int bytesRead)
-      throws IOException, OzoneException {
-    if (bytesRead != contentLen) {
-      fs.deleteKey(args);
-      OzoneException ex = ErrorTable.newError(INCOMPLETE_BODY, args);
-      String msg = String.format("Body length mismatch. Expected length : %d" +
-          " Found %d", contentLen, bytesRead);
-      ex.setMessage(msg);
-      LOG.debug(msg);
-      throw ex;
-    }
-  }
-
-  /**
-   * Returns Content Length header value if available.
-   *
-   * @param headers - Http Headers
-   * @return - String or null
-   */
-  public String getContentLength(HttpHeaders headers, KeyArgs args)
-      throws OzoneException {
-    List<String> contentLengthList =
-        headers.getRequestHeader(HttpHeaders.CONTENT_LENGTH);
-    if ((contentLengthList != null) && (contentLengthList.size() > 0)) {
-      return contentLengthList.get(0);
-    }
-
-    OzoneException ex = ErrorTable.newError(INVALID_REQUEST, args);
-    ex.setMessage("Content-Length is a required header for putting a key.");
-    throw ex;
-
-  }
-
-  /**
-   * Returns Content MD5 value if available.
-   *
-   * @param headers - Http Headers
-   * @return - String or null
-   */
-  public String getContentMD5(HttpHeaders headers, KeyArgs args) {
-    List<String> contentLengthList =
-        headers.getRequestHeader(Header.CONTENT_MD5);
-    if ((contentLengthList != null) && (contentLengthList.size() > 0)) {
-      return contentLengthList.get(0);
-    }
-// TODO : Should we make this compulsory ?
-//    OzoneException ex = ErrorTable.newError(ErrorTable.invalidRequest, args);
-//    ex.setMessage("Content-MD5 is a required header for putting a key");
-//    throw ex;
-    return "";
-  }
-}
-

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/ListArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/ListArgs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/ListArgs.java
deleted file mode 100644
index 49ca4a4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/ListArgs.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.web.handlers;
-
-/**
- * Supports listing keys with pagination.
- */
-public class ListArgs<T extends UserArgs> {
-  private String prevKey;
-  private String prefix;
-  private int maxKeys;
-  private boolean rootScan;
-  private T args;
-
-  /**
-   * Constructor for ListArgs.
-   *
-   * @param args      - BucketArgs
-   * @param prefix    Prefix to start Query from
-   * @param maxKeys   Max result set
-   * @param prevKey - Page token
-   */
-  public ListArgs(T args, String prefix, int maxKeys,
-                  String prevKey) {
-    setArgs(args);
-    setPrefix(prefix);
-    setMaxKeys(maxKeys);
-    setPrevKey(prevKey);
-  }
-
-  /**
-   * Copy Constructor for ListArgs.
-   *
-   * @param args - List Args
-   */
-  public ListArgs(T args, ListArgs listArgs) {
-    this(args, listArgs.getPrefix(), listArgs.getMaxKeys(),
-        listArgs.getPrevKey());
-  }
-
-  /**
-   * Returns page token.
-   *
-   * @return String
-   */
-  public String getPrevKey() {
-    return prevKey;
-  }
-
-  /**
-   * Sets page token.
-   *
-   * @param prevKey - Page token
-   */
-  public void setPrevKey(String prevKey) {
-    this.prevKey = prevKey;
-  }
-
-  /**
-   * Gets max keys.
-   *
-   * @return int
-   */
-  public int getMaxKeys() {
-    return maxKeys;
-  }
-
-  /**
-   * Sets max keys.
-   *
-   * @param maxKeys - Maximum keys to return
-   */
-  public void setMaxKeys(int maxKeys) {
-    this.maxKeys = maxKeys;
-  }
-
-  /**
-   * Gets prefix.
-   *
-   * @return String
-   */
-  public String getPrefix() {
-    return prefix;
-  }
-
-  /**
-   * Sets prefix.
-   *
-   * @param prefix - The prefix that we are looking for
-   */
-  public void setPrefix(String prefix) {
-    this.prefix = prefix;
-  }
-
-  /**
-   * Gets args.
-   * @return  T
-   */
-  public T getArgs() {
-    return args;
-  }
-
-  /**
-   * Sets  args.
-   * @param args T
-   */
-  public void setArgs(T args) {
-    this.args = args;
-  }
-
-  /**
-   * Checks if we are doing a rootScan.
-   * @return - RootScan.
-   */
-  public boolean isRootScan() {
-    return rootScan;
-  }
-
-  /**
-   * Sets the RootScan property.
-   * @param rootScan - Boolean.
-   */
-  public void setRootScan(boolean rootScan) {
-    this.rootScan = rootScan;
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/ServiceFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/ServiceFilter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/ServiceFilter.java
deleted file mode 100644
index 76aa286..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/ServiceFilter.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import com.sun.jersey.spi.container.ContainerRequest;
-import com.sun.jersey.spi.container.ContainerRequestFilter;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-
-import javax.ws.rs.core.UriBuilder;
-import javax.ws.rs.ext.Provider;
-
-/**
- * This class is used to intercept root URL requests and route it to
- * Volume List functionality.
- */
-@Provider
-public class ServiceFilter implements ContainerRequestFilter {
-  /**
-   * Filter the request.
-   * <p>
-   * An implementation may modify the state of the request or
-   * create a new instance.
-   *
-   * @param request the request.
-   *
-   * @return the request.
-   */
-  @Override
-  public ContainerRequest filter(ContainerRequest request) {
-    if (request.getRequestUri().getPath().length() > 1) {
-      return request;
-    }
-
-    // Just re-route it to volume handler with some hypothetical volume name.
-    // volume name is ignored.
-
-    request.setUris(request.getBaseUri(),
-        UriBuilder.fromUri(request.getRequestUri())
-        .path("/service")
-        .queryParam("info", Header.OZONE_LIST_QUERY_SERVICE)
-        .build());
-
-    return request;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/StorageHandlerBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/StorageHandlerBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/StorageHandlerBuilder.java
deleted file mode 100644
index 64fe204..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/StorageHandlerBuilder.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.localstorage.LocalStorageHandler;
-
-/**
- * This class is responsible for providing a {@link StorageHandler}
- * implementation to object store web handlers.
- */
-@InterfaceAudience.Private
-public final class StorageHandlerBuilder {
-
-  private static final ThreadLocal<StorageHandler>
-      STORAGE_HANDLER_THREAD_LOCAL = new ThreadLocal<>();
-
-  /**
-   * Returns the configured StorageHandler from thread-local storage for this
-   * thread.
-   *
-   * @return StorageHandler from thread-local storage
-   */
-  public static StorageHandler getStorageHandler() {
-    StorageHandler storageHandler = STORAGE_HANDLER_THREAD_LOCAL.get();
-    if (storageHandler != null) {
-      return storageHandler;
-    } else {
-      // This only happens while using mvn jetty:run for testing.
-      Configuration conf = new OzoneConfiguration();
-      return new LocalStorageHandler(conf);
-    }
-  }
-
-  /**
-   * Removes the configured StorageHandler from thread-local storage for this
-   * thread.
-   */
-  public static void removeStorageHandler() {
-    STORAGE_HANDLER_THREAD_LOCAL.remove();
-  }
-
-  /**
-   * Sets the configured StorageHandler in thread-local storage for this thread.
-   *
-   * @param storageHandler StorageHandler to set in thread-local storage
-   */
-  public static void setStorageHandler(StorageHandler storageHandler) {
-    STORAGE_HANDLER_THREAD_LOCAL.set(storageHandler);
-  }
-
-  /**
-   * There is no reason to instantiate this class.
-   */
-  private StorageHandlerBuilder() {
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/UserArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/UserArgs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/UserArgs.java
deleted file mode 100644
index 07856d0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/UserArgs.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.UriInfo;
-
-/**
- * UserArgs is used to package caller info
- * and pass it down to file system.
- */
-@InterfaceAudience.Private
-public class UserArgs {
-  private String userName;
-  private final String requestID;
-  private final String hostName;
-  private final UriInfo uri;
-  private final Request request;
-  private final HttpHeaders headers;
-  private String[] groups;
-
-
-  /**
-   * Constructs  user args.
-   *
-   * @param userName - User name
-   * @param requestID - Request ID
-   * @param hostName - Host Name
-   * @param req - Request
-   * @param info - Uri Info
-   * @param httpHeaders - http headers
-   */
-  public UserArgs(String userName, String requestID, String hostName,
-                  Request req, UriInfo info, HttpHeaders httpHeaders) {
-    this.hostName = hostName;
-    this.userName = userName;
-    this.requestID = requestID;
-    this.uri = info;
-    this.request = req;
-    this.headers = httpHeaders;
-  }
-
-  /**
-   * Constructs  user args when we don't know the user name yet.
-   *
-   * @param requestID _ Request ID
-   * @param hostName - Host Name
-   * @param req - Request
-   * @param info - UriInfo
-   * @param httpHeaders - http headers
-   */
-  public UserArgs(String requestID, String hostName, Request req, UriInfo info,
-                  HttpHeaders httpHeaders) {
-    this.hostName = hostName;
-    this.requestID = requestID;
-    this.uri = info;
-    this.request = req;
-    this.headers = httpHeaders;
-  }
-
-  /**
-   * Returns hostname.
-   *
-   * @return String
-   */
-  public String getHostName() {
-    return hostName;
-  }
-
-  /**
-   * Returns RequestID.
-   *
-   * @return Long
-   */
-  public String getRequestID() {
-    return requestID;
-  }
-
-  /**
-   * Returns User Name.
-   *
-   * @return String
-   */
-  public String getUserName() {
-    return userName;
-  }
-
-  /**
-   * Sets the user name.
-   *
-   * @param userName Name of the user
-   */
-  public void setUserName(String userName) {
-    this.userName = userName;
-  }
-
-  /**
-   * Returns list of groups.
-   *
-   * @return String[]
-   */
-  public String[] getGroups() {
-    return this.groups;
-  }
-
-  /**
-   * Sets the group list.
-   *
-   * @param groups list of groups
-   */
-  public void setGroups(String[] groups) {
-    this.groups = groups;
-  }
-
-  /**
-   * Returns the resource Name.
-   *
-   * @return String Resource.
-   */
-  public String getResourceName() {
-    return getUserName();
-  }
-
-  /**
-   * Returns Http Headers for this call.
-   *
-   * @return httpHeaders
-   */
-  public HttpHeaders getHeaders() {
-    return headers;
-  }
-
-  /**
-   * Returns Request Object.
-   *
-   * @return Request
-   */
-  public Request getRequest() {
-    return request;
-  }
-
-  /**
-   * Returns UriInfo.
-   *
-   * @return UriInfo
-   */
-  public UriInfo getUri() {
-    return uri;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/UserHandlerBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/UserHandlerBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/UserHandlerBuilder.java
deleted file mode 100644
index d9051f3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/UserHandlerBuilder.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.web.interfaces.UserAuth;
-import org.apache.hadoop.ozone.web.userauth.Simple;
-
-/**
- * This class is responsible for providing a
- * {@link org.apache.hadoop.ozone.web.interfaces.UserAuth}
- * implementation to object store web handlers.
- */
-@InterfaceAudience.Private
-public final class UserHandlerBuilder {
-
-  private static final ThreadLocal<UserAuth> USER_AUTH_THREAD_LOCAL =
-      new ThreadLocal<UserAuth>();
-
-  /**
-   * Returns the configured UserAuth from thread-local storage for this
-   * thread.
-   *
-   * @return UserAuth from thread-local storage
-   */
-  public static UserAuth getAuthHandler() {
-    UserAuth authHandler = USER_AUTH_THREAD_LOCAL.get();
-    if (authHandler != null) {
-      return authHandler;
-    } else {
-      // This only happens while using mvn jetty:run for testing.
-      return new Simple();
-    }
-  }
-
-  /**
-   * Removes the configured UserAuth from thread-local storage for this
-   * thread.
-   */
-  public static void removeAuthHandler() {
-    USER_AUTH_THREAD_LOCAL.remove();
-  }
-
-  /**
-   * Sets the configured UserAuthHandler in thread-local storage for this
-   * thread.
-   *
-   * @param authHandler authHandler to set in thread-local storage
-   */
-  public static void setAuthHandler(UserAuth authHandler) {
-    USER_AUTH_THREAD_LOCAL.set(authHandler);
-  }
-
-  /**
-   * There is no reason to instantiate this class.
-   */
-  private UserHandlerBuilder() {
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeArgs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeArgs.java
deleted file mode 100644
index 1d67c67..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeArgs.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.web.request.OzoneQuota;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.UriInfo;
-
-/**
- * VolumeArgs is used to package all volume
- * related arguments in the call to underlying
- * file system.
- */
-@InterfaceAudience.Private
-public class VolumeArgs extends UserArgs {
-  private String adminName;
-  private final String volumeName;
-  private OzoneQuota quota;
-
-  /**
-   * Returns Quota Information.
-   *
-   * @return Quota
-   */
-  public OzoneQuota getQuota() {
-    return quota;
-  }
-
-  /**
-   * Returns volume name.
-   *
-   * @return String
-   */
-  public String getVolumeName() {
-    return volumeName;
-  }
-
-  /**
-   * Constructs  volume Args.
-   *
-   * @param userName - User name
-   * @param volumeName - volume Name
-   * @param requestID _ Request ID
-   * @param hostName - Host Name
-   * @param request  - Http Request
-   * @param info - URI info
-   * @param headers - http headers
-   * @param groups - list of groups allowed to access the volume
-   */
-  public VolumeArgs(String userName, String volumeName, String requestID,
-                    String hostName, Request request, UriInfo info,
-                    HttpHeaders headers, String[] groups) {
-    super(userName, requestID, hostName, request, info, headers);
-    super.setGroups(groups);
-    this.volumeName = volumeName;
-  }
-
-  /**
-   * Constructs  volume Args.
-   *
-   * @param volumeName - volume Name
-   * @param userArgs - userArgs
-   */
-  public VolumeArgs(String volumeName, UserArgs userArgs) {
-    this(userArgs.getUserName(), volumeName, userArgs.getRequestID(),
-         userArgs.getHostName(), userArgs.getRequest(), userArgs.getUri(),
-         userArgs.getHeaders(), userArgs.getGroups());
-  }
-
-  /**
-   * Creates VolumeArgs from another VolumeArgs.
-   */
-  public VolumeArgs(VolumeArgs volArgs) {
-    this(volArgs.getVolumeName(), volArgs);
-  }
-
-  /**
-   * Sets Quota information.
-   *
-   * @param quota - Quota Sting
-   * @throws IllegalArgumentException
-   */
-  public void setQuota(String quota) throws IllegalArgumentException {
-    this.quota = OzoneQuota.parseQuota(quota);
-  }
-
-  /**
-   * Sets quota information.
-   *
-   * @param quota - OzoneQuota
-   */
-  public void setQuota(OzoneQuota quota) {
-    this.quota = quota;
-  }
-
-  /**
-   * Gets admin Name.
-   *
-   * @return - Admin Name
-   */
-  public String getAdminName() {
-    return adminName;
-  }
-
-  /**
-   * Sets Admin Name.
-   *
-   * @param adminName - Admin Name
-   */
-  public void setAdminName(String adminName) {
-    this.adminName = adminName;
-  }
-
-  /**
-   * Returns UserName/VolumeName.
-   *
-   * @return String
-   */
-  @Override
-  public String getResourceName() {
-    return super.getResourceName() + "/" + getVolumeName();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeHandler.java
deleted file mode 100644
index 7c50859..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeHandler.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.interfaces.UserAuth;
-import org.apache.hadoop.ozone.web.interfaces.Volume;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-
-import static java.net.HttpURLConnection.HTTP_CREATED;
-import static java.net.HttpURLConnection.HTTP_OK;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_FUNCTION;
-
-/**
- * VolumeHandler handles volume specific HTTP calls.
- *
- * Most functions in this file follow a simple pattern.
- * All calls are handled by VolumeProcessTemplate.handleCall, which
- * calls back into doProcess function.
- *
- * Everything common to volume handling is abstracted out in handleCall function
- * For Example : Checking that volume name is sane, we have a supported
- * ozone version number and a valid date. That is everything common is in
- * handleCall and actions specific to a call is inside doProcess callback.
- */
-@InterfaceAudience.Private
-public class VolumeHandler implements Volume {
-  private static final Logger LOG = LoggerFactory.getLogger(VolumeHandler
-      .class);
-  /**
-   * Creates a volume.
-   *
-   * @param volume Volume Name, this has to be unique at Ozone cluster level
-   * @param quota Quota for this Storage Volume - <int>(<BYTES|MB|GB|TB>)
-   * @param req Request Object
-   * @param uriInfo URI info
-   * @param headers Http Headers
-   *
-   * @return Standard JAX-RS Response
-   *
-   * @throws OzoneException
-   */
-  @Override
-  public Response createVolume(String volume, final String quota, Request req,
-                               UriInfo uriInfo, HttpHeaders headers)
-      throws OzoneException {
-    MDC.put(OZONE_FUNCTION, "createVolume");
-    return new VolumeProcessTemplate() {
-      @Override
-      public Response doProcess(VolumeArgs args)
-          throws IOException, OzoneException {
-        UserAuth auth = UserHandlerBuilder.getAuthHandler();
-        if (auth.isAdmin(args)) {
-          args.setAdminName(args.getUserName());
-          String volumeOwner = auth.getOzoneUser(args);
-
-          if (volumeOwner == null) {
-            throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, args);
-          }
-
-          if (!auth.isUser(volumeOwner, args)) {
-            throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, args);
-          }
-
-          args.setUserName(volumeOwner);
-          args.setGroups(auth.getGroups(args));
-          if (!quota.equals(Header.OZONE_QUOTA_UNDEFINED)) {
-            setQuotaArgs(args, quota);
-          }
-          StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
-          fs.createVolume(args);
-          return OzoneUtils.getResponse(args, HTTP_CREATED, "");
-        } else {
-          throw ErrorTable.newError(ErrorTable.ACCESS_DENIED, args);
-        }
-      }
-    }.handleCall(volume, req, uriInfo, headers);
-  }
-
-  /**
-   * Updates  volume metadata.
-   *
-   * There are only two actions possible currently with updateVolume.
-   * Change the volume ownership or update quota. if you make a call
-   * with neither of these actions, update just returns 200 OK.
-   *
-   * @param volume Volume Name, this has to be unique at Ozone Level
-   * @param quota Quota for this volume - <int>(<BYTES|MB|GB|TB>)|remove
-   * @param req - Request Object
-   * @param uriInfo - URI info
-   * @param headers Http Headers
-   *
-   * @return Standard JAX-RS Response
-   *
-   * @throws OzoneException
-   */
-  @Override
-  public Response updateVolume(String volume, final String quota, Request req,
-                               UriInfo uriInfo, HttpHeaders headers)
-      throws OzoneException {
-    MDC.put(OZONE_FUNCTION, "updateVolume");
-    return new VolumeProcessTemplate() {
-      @Override
-      public Response doProcess(VolumeArgs args)
-          throws IOException, OzoneException {
-        UserAuth auth = UserHandlerBuilder.getAuthHandler();
-        if (auth.isAdmin(args)) {
-          StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
-          args.setAdminName(args.getUserName());
-          String newVolumeOwner = auth.getOzoneUser(args);
-
-          if (newVolumeOwner != null) {
-            if (!auth.isUser(newVolumeOwner, args)) {
-              throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, args);
-            }
-            args.setUserName(newVolumeOwner);
-            fs.setVolumeOwner(args);
-          }
-
-          if (!quota.equals(Header.OZONE_QUOTA_UNDEFINED)) {
-            if (quota.equals(Header.OZONE_QUOTA_REMOVE)) {
-              // if it is remove, just tell the file system to remove quota
-              fs.setVolumeQuota(args, true);
-            } else {
-              setQuotaArgs(args, quota);
-              fs.setVolumeQuota(args, false);
-            }
-          }
-          return OzoneUtils.getResponse(args, HTTP_OK, "");
-        } else {
-          // Only Admins are allowed to update volumes
-          throw ErrorTable.newError(ErrorTable.ACCESS_DENIED, args);
-        }
-      }
-    }.handleCall(volume, req, uriInfo, headers);
-  }
-
-
-  /**
-   * Deletes a volume if it is empty.
-   *
-   * @param volume Volume Name
-   * @param req - Http Request
-   * @param uriInfo - http URI
-   * @param headers - http headers
-   *
-   * @return Standard JAX-RS Response
-   *
-   * @throws OzoneException
-   */
-  @Override
-  public Response deleteVolume(String volume, Request req, UriInfo uriInfo,
-                               HttpHeaders headers) throws OzoneException {
-    MDC.put(OZONE_FUNCTION, "deleteVolume");
-
-    return new VolumeProcessTemplate() {
-      @Override
-      public Response doProcess(VolumeArgs args)
-          throws IOException, OzoneException {
-        UserAuth auth = UserHandlerBuilder.getAuthHandler();
-        if (auth.isAdmin(args)) {
-          StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
-          fs.deleteVolume(args);
-          return OzoneUtils.getResponse(args, HTTP_OK, "");
-        } else {
-          throw ErrorTable.newError(ErrorTable.ACCESS_DENIED, args);
-        }
-      }
-    }.handleCall(volume, req, uriInfo, headers);
-  }
-
-  /**
-   * Returns Volume info. This API can be invoked either by admin or the owner
-   *
-   * @param volume  - Storage Volume Name
-   * @param info    - Info attribute
-   * @param prefix  - Prefix key
-   * @param maxKeys - Max results
-   * @param prevKey - PrevKey
-   * @param req     - Http Req
-   * @param uriInfo - UriInfo.
-   * @param headers - Http headers
-   * @return
-   * @throws OzoneException
-   */
-  @Override
-  public Response getVolumeInfo(String volume, final String info,
-                                final String prefix,
-                                final int maxKeys,
-                                final String prevKey,
-                                final boolean rootScan,
-                                Request req,
-                                final UriInfo uriInfo, HttpHeaders headers)
-      throws OzoneException {
-
-    return new VolumeProcessTemplate() {
-      @Override
-      public Response doProcess(VolumeArgs args)
-          throws IOException, OzoneException {
-
-        switch (info) {
-        case Header.OZONE_INFO_QUERY_BUCKET:
-          MDC.put(OZONE_FUNCTION, "ListBucket");
-          return getBucketsInVolume(args, prefix, maxKeys, prevKey);
-        case Header.OZONE_INFO_QUERY_VOLUME:
-          MDC.put(OZONE_FUNCTION, "InfoVolume");
-          assertNoListParamPresent(uriInfo, args);
-          return getVolumeInfoResponse(args); // Return volume info
-        case Header.OZONE_LIST_QUERY_SERVICE:
-          MDC.put(OZONE_FUNCTION, "ListVolume");
-          return getVolumesByUser(args, prefix, maxKeys, prevKey, rootScan);
-        default:
-          LOG.debug("Unrecognized query param : {} ", info);
-          OzoneException ozoneException =
-              ErrorTable.newError(ErrorTable.INVALID_QUERY_PARAM, args);
-          ozoneException.setMessage("Unrecognized query param : " + info);
-          throw ozoneException;
-        }
-      }
-    }.handleCall(volume, req, uriInfo, headers);
-  }
-
-  /**
-   * Asserts no list query param is present during this call.
-   *
-   * @param uriInfo - UriInfo.   - UriInfo
-   * @param args    - Volume Args - VolumeArgs.
-   * @throws OzoneException
-   */
-  private void assertNoListParamPresent(final UriInfo uriInfo, VolumeArgs
-      args) throws
-      OzoneException {
-
-    String prefix = uriInfo.getQueryParameters().getFirst("prefix");
-    String maxKeys = uriInfo.getQueryParameters().getFirst("max_keys");
-    String prevKey = uriInfo.getQueryParameters().getFirst("prev_key");
-    if ((prefix != null && !prefix.equals(Header.OZONE_EMPTY_STRING)) ||
-        (maxKeys != null && !maxKeys.equals(Header.OZONE_DEFAULT_LIST_SIZE)) ||
-        (prevKey != null && !prevKey.equals(Header.OZONE_EMPTY_STRING))) {
-      throw ErrorTable.newError(ErrorTable.INVALID_QUERY_PARAM, args);
-    }
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[41/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyArgs.java
deleted file mode 100644
index abeac10..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyArgs.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-
-/**
- * Args for key. Client use this to specify key's attributes on  key creation
- * (putKey()).
- */
-public final class KsmKeyArgs {
-  private final String volumeName;
-  private final String bucketName;
-  private final String keyName;
-  private long dataSize;
-  private final ReplicationType type;
-  private final ReplicationFactor factor;
-
-  private KsmKeyArgs(String volumeName, String bucketName, String keyName,
-      long dataSize, ReplicationType type, ReplicationFactor factor) {
-    this.volumeName = volumeName;
-    this.bucketName = bucketName;
-    this.keyName = keyName;
-    this.dataSize = dataSize;
-    this.type = type;
-    this.factor = factor;
-  }
-
-  public ReplicationType getType() {
-    return type;
-  }
-
-  public ReplicationFactor getFactor() {
-    return factor;
-  }
-
-  public String getVolumeName() {
-    return volumeName;
-  }
-
-  public String getBucketName() {
-    return bucketName;
-  }
-
-  public String getKeyName() {
-    return keyName;
-  }
-
-  public long getDataSize() {
-    return dataSize;
-  }
-
-  public void setDataSize(long size) {
-    dataSize = size;
-  }
-
-  /**
-   * Builder class of KsmKeyArgs.
-   */
-  public static class Builder {
-    private String volumeName;
-    private String bucketName;
-    private String keyName;
-    private long dataSize;
-    private ReplicationType type;
-    private ReplicationFactor factor;
-
-
-    public Builder setVolumeName(String volume) {
-      this.volumeName = volume;
-      return this;
-    }
-
-    public Builder setBucketName(String bucket) {
-      this.bucketName = bucket;
-      return this;
-    }
-
-    public Builder setKeyName(String key) {
-      this.keyName = key;
-      return this;
-    }
-
-    public Builder setDataSize(long size) {
-      this.dataSize = size;
-      return this;
-    }
-
-    public Builder setType(ReplicationType replicationType) {
-      this.type = replicationType;
-      return this;
-    }
-
-    public Builder setFactor(ReplicationFactor replicationFactor) {
-      this.factor = replicationFactor;
-      return this;
-    }
-
-    public KsmKeyArgs build() {
-      return new KsmKeyArgs(volumeName, bucketName, keyName, dataSize,
-          type, factor);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
deleted file mode 100644
index 41d523c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo;
-import org.apache.hadoop.util.Time;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Args for key block. The block instance for the key requested in putKey.
- * This is returned from KSM to client, and client use class to talk to
- * datanode. Also, this is the metadata written to ksm.db on server side.
- */
-public final class KsmKeyInfo {
-  private final String volumeName;
-  private final String bucketName;
-  // name of key client specified
-  private final String keyName;
-  private long dataSize;
-  private List<KsmKeyLocationInfoGroup> keyLocationVersions;
-  private final long creationTime;
-  private long modificationTime;
-
-  private KsmKeyInfo(String volumeName, String bucketName, String keyName,
-      List<KsmKeyLocationInfoGroup> versions, long dataSize,
-      long creationTime, long modificationTime) {
-    this.volumeName = volumeName;
-    this.bucketName = bucketName;
-    this.keyName = keyName;
-    this.dataSize = dataSize;
-    // it is important that the versions are ordered from old to new.
-    // Do this sanity check when versions got loaded on creating KsmKeyInfo.
-    // TODO : this is not necessary, here only because versioning is still a
-    // work in-progress, remove this following check when versioning is
-    // complete and prove correctly functioning
-    long currentVersion = -1;
-    for (KsmKeyLocationInfoGroup version : versions) {
-      Preconditions.checkArgument(
-            currentVersion + 1 == version.getVersion());
-      currentVersion = version.getVersion();
-    }
-    this.keyLocationVersions = versions;
-    this.creationTime = creationTime;
-    this.modificationTime = modificationTime;
-  }
-
-  public String getVolumeName() {
-    return volumeName;
-  }
-
-  public String getBucketName() {
-    return bucketName;
-  }
-
-  public String getKeyName() {
-    return keyName;
-  }
-
-  public long getDataSize() {
-    return dataSize;
-  }
-
-  public void setDataSize(long size) {
-    this.dataSize = size;
-  }
-
-  public synchronized KsmKeyLocationInfoGroup getLatestVersionLocations()
-      throws IOException {
-    return keyLocationVersions.size() == 0? null :
-        keyLocationVersions.get(keyLocationVersions.size() - 1);
-  }
-
-  public List<KsmKeyLocationInfoGroup> getKeyLocationVersions() {
-    return keyLocationVersions;
-  }
-
-  public void updateModifcationTime() {
-    this.modificationTime = Time.monotonicNow();
-  }
-
-  /**
-   * Append a set of blocks to the latest version. Note that these blocks are
-   * part of the latest version, not a new version.
-   *
-   * @param newLocationList the list of new blocks to be added.
-   * @throws IOException
-   */
-  public synchronized void appendNewBlocks(
-      List<KsmKeyLocationInfo> newLocationList) throws IOException {
-    if (keyLocationVersions.size() == 0) {
-      throw new IOException("Appending new block, but no version exist");
-    }
-    KsmKeyLocationInfoGroup currentLatestVersion =
-        keyLocationVersions.get(keyLocationVersions.size() - 1);
-    currentLatestVersion.appendNewBlocks(newLocationList);
-    setModificationTime(Time.now());
-  }
-
-  /**
-   * Add a new set of blocks. The new blocks will be added as appending a new
-   * version to the all version list.
-   *
-   * @param newLocationList the list of new blocks to be added.
-   * @throws IOException
-   */
-  public synchronized long addNewVersion(
-      List<KsmKeyLocationInfo> newLocationList) throws IOException {
-    long latestVersionNum;
-    if (keyLocationVersions.size() == 0) {
-      // no version exist, these blocks are the very first version.
-      keyLocationVersions.add(new KsmKeyLocationInfoGroup(0, newLocationList));
-      latestVersionNum = 0;
-    } else {
-      // it is important that the new version are always at the tail of the list
-      KsmKeyLocationInfoGroup currentLatestVersion =
-          keyLocationVersions.get(keyLocationVersions.size() - 1);
-      // the new version is created based on the current latest version
-      KsmKeyLocationInfoGroup newVersion =
-          currentLatestVersion.generateNextVersion(newLocationList);
-      keyLocationVersions.add(newVersion);
-      latestVersionNum = newVersion.getVersion();
-    }
-    setModificationTime(Time.now());
-    return latestVersionNum;
-  }
-
-  public long getCreationTime() {
-    return creationTime;
-  }
-
-  public long getModificationTime() {
-    return modificationTime;
-  }
-
-  public void setModificationTime(long modificationTime) {
-    this.modificationTime = modificationTime;
-  }
-
-  /**
-   * Builder of KsmKeyInfo.
-   */
-  public static class Builder {
-    private String volumeName;
-    private String bucketName;
-    private String keyName;
-    private long dataSize;
-    private List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroups;
-    private long creationTime;
-    private long modificationTime;
-
-    public Builder setVolumeName(String volume) {
-      this.volumeName = volume;
-      return this;
-    }
-
-    public Builder setBucketName(String bucket) {
-      this.bucketName = bucket;
-      return this;
-    }
-
-    public Builder setKeyName(String key) {
-      this.keyName = key;
-      return this;
-    }
-
-    public Builder setKsmKeyLocationInfos(
-        List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoList) {
-      this.ksmKeyLocationInfoGroups = ksmKeyLocationInfoList;
-      return this;
-    }
-
-    public Builder setDataSize(long size) {
-      this.dataSize = size;
-      return this;
-    }
-
-    public Builder setCreationTime(long crTime) {
-      this.creationTime = crTime;
-      return this;
-    }
-
-    public Builder setModificationTime(long mTime) {
-      this.modificationTime = mTime;
-      return this;
-    }
-
-    public KsmKeyInfo build() {
-      return new KsmKeyInfo(
-          volumeName, bucketName, keyName, ksmKeyLocationInfoGroups,
-          dataSize, creationTime, modificationTime);
-    }
-  }
-
-  public KeyInfo getProtobuf() {
-    long latestVersion = keyLocationVersions.size() == 0 ? -1 :
-        keyLocationVersions.get(keyLocationVersions.size() - 1).getVersion();
-    return KeyInfo.newBuilder()
-        .setVolumeName(volumeName)
-        .setBucketName(bucketName)
-        .setKeyName(keyName)
-        .setDataSize(dataSize)
-        .addAllKeyLocationList(keyLocationVersions.stream()
-            .map(KsmKeyLocationInfoGroup::getProtobuf)
-            .collect(Collectors.toList()))
-        .setLatestVersion(latestVersion)
-        .setCreationTime(creationTime)
-        .setModificationTime(modificationTime)
-        .build();
-  }
-
-  public static KsmKeyInfo getFromProtobuf(KeyInfo keyInfo) {
-    return new KsmKeyInfo(
-        keyInfo.getVolumeName(),
-        keyInfo.getBucketName(),
-        keyInfo.getKeyName(),
-        keyInfo.getKeyLocationListList().stream()
-            .map(KsmKeyLocationInfoGroup::getFromProtobuf)
-            .collect(Collectors.toList()),
-        keyInfo.getDataSize(),
-        keyInfo.getCreationTime(),
-        keyInfo.getModificationTime());
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfo.java
deleted file mode 100644
index 9d24b30..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfo.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyLocation;
-
-/**
- * One key can be too huge to fit in one container. In which case it gets split
- * into a number of subkeys. This class represents one such subkey instance.
- */
-public final class KsmKeyLocationInfo {
-  private final String containerName;
-  // name of the block id SCM assigned for the key
-  private final String blockID;
-  private final boolean shouldCreateContainer;
-  // the id of this subkey in all the subkeys.
-  private final long length;
-  private final long offset;
-  // the version number indicating when this block was added
-  private long createVersion;
-
-  private KsmKeyLocationInfo(String containerName,
-      String blockID, boolean shouldCreateContainer,
-      long length, long offset) {
-    this.containerName = containerName;
-    this.blockID = blockID;
-    this.shouldCreateContainer = shouldCreateContainer;
-    this.length = length;
-    this.offset = offset;
-  }
-
-  public void setCreateVersion(long version) {
-    createVersion = version;
-  }
-
-  public long getCreateVersion() {
-    return createVersion;
-  }
-
-  public String getContainerName() {
-    return containerName;
-  }
-
-  public String getBlockID() {
-    return blockID;
-  }
-
-  public boolean getShouldCreateContainer() {
-    return shouldCreateContainer;
-  }
-
-  public long getLength() {
-    return length;
-  }
-
-  public long getOffset() {
-    return offset;
-  }
-
-  /**
-   * Builder of KsmKeyLocationInfo.
-   */
-  public static class Builder {
-    private String containerName;
-    private String blockID;
-    private boolean shouldCreateContainer;
-    private long length;
-    private long offset;
-
-    public Builder setContainerName(String container) {
-      this.containerName = container;
-      return this;
-    }
-
-    public Builder setBlockID(String block) {
-      this.blockID = block;
-      return this;
-    }
-
-    public Builder setShouldCreateContainer(boolean create) {
-      this.shouldCreateContainer = create;
-      return this;
-    }
-
-    public Builder setLength(long len) {
-      this.length = len;
-      return this;
-    }
-
-    public Builder setOffset(long off) {
-      this.offset = off;
-      return this;
-    }
-
-    public KsmKeyLocationInfo build() {
-      return new KsmKeyLocationInfo(containerName, blockID,
-          shouldCreateContainer, length, offset);
-    }
-  }
-
-  public KeyLocation getProtobuf() {
-    return KeyLocation.newBuilder()
-        .setContainerName(containerName)
-        .setBlockID(blockID)
-        .setShouldCreateContainer(shouldCreateContainer)
-        .setLength(length)
-        .setOffset(offset)
-        .setCreateVersion(createVersion)
-        .build();
-  }
-
-  public static KsmKeyLocationInfo getFromProtobuf(KeyLocation keyLocation) {
-    KsmKeyLocationInfo info = new KsmKeyLocationInfo(
-        keyLocation.getContainerName(),
-        keyLocation.getBlockID(),
-        keyLocation.getShouldCreateContainer(),
-        keyLocation.getLength(),
-        keyLocation.getOffset());
-    info.setCreateVersion(keyLocation.getCreateVersion());
-    return info;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfoGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfoGroup.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfoGroup.java
deleted file mode 100644
index bef65ec..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfoGroup.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyLocationList;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * A list of key locations. This class represents one single version of the
- * blocks of a key.
- */
-public class KsmKeyLocationInfoGroup {
-  private final long version;
-  private final List<KsmKeyLocationInfo> locationList;
-
-  public KsmKeyLocationInfoGroup(long version,
-      List<KsmKeyLocationInfo> locations) {
-    this.version = version;
-    this.locationList = locations;
-  }
-
-  /**
-   * Return only the blocks that are created in the most recent version.
-   *
-   * @return the list of blocks that are created in the latest version.
-   */
-  public List<KsmKeyLocationInfo> getBlocksLatestVersionOnly() {
-    List<KsmKeyLocationInfo> list = new ArrayList<>();
-    locationList.stream().filter(x -> x.getCreateVersion() == version)
-        .forEach(list::add);
-    return list;
-  }
-
-  public long getVersion() {
-    return version;
-  }
-
-  public List<KsmKeyLocationInfo> getLocationList() {
-    return locationList;
-  }
-
-  public KeyLocationList getProtobuf() {
-    return KeyLocationList.newBuilder()
-        .setVersion(version)
-        .addAllKeyLocations(
-            locationList.stream().map(KsmKeyLocationInfo::getProtobuf)
-                .collect(Collectors.toList()))
-        .build();
-  }
-
-  public static KsmKeyLocationInfoGroup getFromProtobuf(
-      KeyLocationList keyLocationList) {
-    return new KsmKeyLocationInfoGroup(
-        keyLocationList.getVersion(),
-        keyLocationList.getKeyLocationsList().stream()
-            .map(KsmKeyLocationInfo::getFromProtobuf)
-            .collect(Collectors.toList()));
-  }
-
-  /**
-   * Given a new block location, generate a new version list based upon this
-   * one.
-   *
-   * @param newLocationList a list of new location to be added.
-   * @return
-   */
-  KsmKeyLocationInfoGroup generateNextVersion(
-      List<KsmKeyLocationInfo> newLocationList) throws IOException {
-    // TODO : revisit if we can do this method more efficiently
-    // one potential inefficiency here is that later version always include
-    // older ones. e.g. v1 has B1, then v2, v3...will all have B1 and only add
-    // more
-    List<KsmKeyLocationInfo> newList = new ArrayList<>();
-    newList.addAll(locationList);
-    for (KsmKeyLocationInfo newInfo : newLocationList) {
-      // all these new blocks will have addVersion of current version + 1
-      newInfo.setCreateVersion(version + 1);
-      newList.add(newInfo);
-    }
-    return new KsmKeyLocationInfoGroup(version + 1, newList);
-  }
-
-  void appendNewBlocks(List<KsmKeyLocationInfo> newLocationList)
-      throws IOException {
-    for (KsmKeyLocationInfo info : newLocationList) {
-      info.setCreateVersion(version);
-      locationList.add(info);
-    }
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder();
-    sb.append("version:").append(version).append(" ");
-    for (KsmKeyLocationInfo kli : locationList) {
-      sb.append(kli.getBlockID()).append(" || ");
-    }
-    return sb.toString();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmOzoneAclMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmOzoneAclMap.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmOzoneAclMap.java
deleted file mode 100644
index 7d9efad..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmOzoneAclMap.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.ksm.helpers;
-
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.OzoneAclInfo;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclRights;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclType;
-
-import java.util.List;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.ArrayList;
-import java.util.HashMap;
-
-/**
- * This helper class keeps a map of all user and their permissions.
- */
-public class KsmOzoneAclMap {
-  // per Acl Type user:rights map
-  private ArrayList<Map<String, OzoneAclRights>> aclMaps;
-
-  KsmOzoneAclMap() {
-    aclMaps = new ArrayList<>();
-    for (OzoneAclType aclType : OzoneAclType.values()) {
-      aclMaps.add(aclType.ordinal(), new HashMap<>());
-    }
-  }
-
-  private Map<String, OzoneAclRights> getMap(OzoneAclType type) {
-    return aclMaps.get(type.ordinal());
-  }
-
-  // For a given acl type and user, get the stored acl
-  private OzoneAclRights getAcl(OzoneAclType type, String user) {
-    return getMap(type).get(user);
-  }
-
-  // Add a new acl to the map
-  public void addAcl(OzoneAclInfo acl) {
-    getMap(acl.getType()).put(acl.getName(), acl.getRights());
-  }
-
-  // for a given acl, check if the user has access rights
-  public boolean hasAccess(OzoneAclInfo acl) {
-    OzoneAclRights storedRights = getAcl(acl.getType(), acl.getName());
-    if (storedRights != null) {
-      switch (acl.getRights()) {
-      case READ:
-        return (storedRights == OzoneAclRights.READ)
-            || (storedRights == OzoneAclRights.READ_WRITE);
-      case WRITE:
-        return (storedRights == OzoneAclRights.WRITE)
-            || (storedRights == OzoneAclRights.READ_WRITE);
-      case READ_WRITE:
-        return (storedRights == OzoneAclRights.READ_WRITE);
-      default:
-        return false;
-      }
-    } else {
-      return false;
-    }
-  }
-
-  // Convert this map to OzoneAclInfo Protobuf List
-  public List<OzoneAclInfo> ozoneAclGetProtobuf() {
-    List<OzoneAclInfo> aclList = new LinkedList<>();
-    for (OzoneAclType type: OzoneAclType.values()) {
-      for (Map.Entry<String, OzoneAclRights> entry :
-          aclMaps.get(type.ordinal()).entrySet()) {
-        OzoneAclInfo aclInfo = OzoneAclInfo.newBuilder()
-            .setName(entry.getKey())
-            .setType(type)
-            .setRights(entry.getValue())
-            .build();
-        aclList.add(aclInfo);
-      }
-    }
-
-    return aclList;
-  }
-
-  // Create map from list of OzoneAclInfos
-  public static KsmOzoneAclMap ozoneAclGetFromProtobuf(
-      List<OzoneAclInfo> aclList) {
-    KsmOzoneAclMap aclMap = new KsmOzoneAclMap();
-    for (OzoneAclInfo acl : aclList) {
-      aclMap.addAcl(acl);
-    }
-    return aclMap;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmVolumeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmVolumeArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmVolumeArgs.java
deleted file mode 100644
index 6b1020c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmVolumeArgs.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.OzoneAclInfo;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.VolumeInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-
-/**
- * A class that encapsulates the KsmVolumeArgs Args.
- */
-public final class KsmVolumeArgs {
-  private final String adminName;
-  private final String ownerName;
-  private final String volume;
-  private final long creationTime;
-  private final long quotaInBytes;
-  private final Map<String, String> keyValueMap;
-  private final KsmOzoneAclMap aclMap;
-
-  /**
-   * Private constructor, constructed via builder.
-   * @param adminName  - Administrator's name.
-   * @param ownerName  - Volume owner's name
-   * @param volume - volume name
-   * @param quotaInBytes - Volume Quota in bytes.
-   * @param keyValueMap - keyValue map.
-   * @param aclMap - User to access rights map.
-   * @param creationTime - Volume creation time.
-   */
-  private KsmVolumeArgs(String adminName, String ownerName, String volume,
-                        long quotaInBytes, Map<String, String> keyValueMap,
-                        KsmOzoneAclMap aclMap, long creationTime) {
-    this.adminName = adminName;
-    this.ownerName = ownerName;
-    this.volume = volume;
-    this.quotaInBytes = quotaInBytes;
-    this.keyValueMap = keyValueMap;
-    this.aclMap = aclMap;
-    this.creationTime = creationTime;
-  }
-
-  /**
-   * Returns the Admin Name.
-   * @return String.
-   */
-  public String getAdminName() {
-    return adminName;
-  }
-
-  /**
-   * Returns the owner Name.
-   * @return String
-   */
-  public String getOwnerName() {
-    return ownerName;
-  }
-
-  /**
-   * Returns the volume Name.
-   * @return String
-   */
-  public String getVolume() {
-    return volume;
-  }
-
-  /**
-   * Returns creation time.
-   * @return long
-   */
-  public long getCreationTime() {
-    return creationTime;
-  }
-
-  /**
-   * Returns Quota in Bytes.
-   * @return long, Quota in bytes.
-   */
-  public long getQuotaInBytes() {
-    return quotaInBytes;
-  }
-
-  public Map<String, String> getKeyValueMap() {
-    return keyValueMap;
-  }
-
-  public KsmOzoneAclMap getAclMap() {
-    return aclMap;
-  }
-  /**
-   * Returns new builder class that builds a KsmVolumeArgs.
-   *
-   * @return Builder
-   */
-  public static Builder newBuilder() {
-    return new Builder();
-  }
-
-  /**
-   * Builder for KsmVolumeArgs.
-   */
-  public static class Builder {
-    private String adminName;
-    private String ownerName;
-    private String volume;
-    private long creationTime;
-    private long quotaInBytes;
-    private Map<String, String> keyValueMap;
-    private KsmOzoneAclMap aclMap;
-
-    /**
-     * Constructs a builder.
-     */
-    Builder() {
-      keyValueMap = new HashMap<>();
-      aclMap = new KsmOzoneAclMap();
-    }
-
-    public Builder setAdminName(String admin) {
-      this.adminName = admin;
-      return this;
-    }
-
-    public Builder setOwnerName(String owner) {
-      this.ownerName = owner;
-      return this;
-    }
-
-    public Builder setVolume(String volumeName) {
-      this.volume = volumeName;
-      return this;
-    }
-
-    public Builder setCreationTime(long createdOn) {
-      this.creationTime = createdOn;
-      return this;
-    }
-
-    public Builder setQuotaInBytes(long quota) {
-      this.quotaInBytes = quota;
-      return this;
-    }
-
-    public Builder addMetadata(String key, String value) {
-      keyValueMap.put(key, value); // overwrite if present.
-      return this;
-    }
-
-    public Builder addOzoneAcls(OzoneAclInfo acl) throws IOException {
-      aclMap.addAcl(acl);
-      return this;
-    }
-
-    /**
-     * Constructs a CreateVolumeArgument.
-     * @return CreateVolumeArgs.
-     */
-    public KsmVolumeArgs build() {
-      Preconditions.checkNotNull(adminName);
-      Preconditions.checkNotNull(ownerName);
-      Preconditions.checkNotNull(volume);
-      return new KsmVolumeArgs(adminName, ownerName, volume, quotaInBytes,
-          keyValueMap, aclMap, creationTime);
-    }
-  }
-
-  public VolumeInfo getProtobuf() {
-    List<KeyValue> metadataList = new LinkedList<>();
-    for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
-      metadataList.add(KeyValue.newBuilder().setKey(entry.getKey()).
-          setValue(entry.getValue()).build());
-    }
-    List<OzoneAclInfo> aclList = aclMap.ozoneAclGetProtobuf();
-
-    return VolumeInfo.newBuilder()
-        .setAdminName(adminName)
-        .setOwnerName(ownerName)
-        .setVolume(volume)
-        .setQuotaInBytes(quotaInBytes)
-        .addAllMetadata(metadataList)
-        .addAllVolumeAcls(aclList)
-        .setCreationTime(creationTime)
-        .build();
-  }
-
-  public static KsmVolumeArgs getFromProtobuf(VolumeInfo volInfo) {
-    Map<String, String> kvMap = volInfo.getMetadataList().stream()
-        .collect(Collectors.toMap(KeyValue::getKey,
-            KeyValue::getValue));
-    KsmOzoneAclMap aclMap =
-        KsmOzoneAclMap.ozoneAclGetFromProtobuf(volInfo.getVolumeAclsList());
-
-    return new KsmVolumeArgs(volInfo.getAdminName(), volInfo.getOwnerName(),
-        volInfo.getVolume(), volInfo.getQuotaInBytes(), kvMap, aclMap,
-        volInfo.getCreationTime());
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/OpenKeySession.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/OpenKeySession.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/OpenKeySession.java
deleted file mode 100644
index c19c04b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/OpenKeySession.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-
-/**
- * This class represents a open key "session". A session here means a key is
- * opened by a specific client, the client sends the handler to server, such
- * that servers can recognize this client, and thus know how to close the key.
- */
-public class OpenKeySession {
-  private final int id;
-  private final KsmKeyInfo keyInfo;
-  // the version of the key when it is being opened in this session.
-  // a block that has a create version equals to open version means it will
-  // be committed only when this open session is closed.
-  private long openVersion;
-
-  public OpenKeySession(int id, KsmKeyInfo info, long version) {
-    this.id = id;
-    this.keyInfo = info;
-    this.openVersion = version;
-  }
-
-  public long getOpenVersion() {
-    return this.openVersion;
-  }
-
-  public KsmKeyInfo getKeyInfo() {
-    return keyInfo;
-  }
-
-  public int getId() {
-    return id;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/ServiceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/ServiceInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/ServiceInfo.java
deleted file mode 100644
index 1c76b8a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/ServiceInfo.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.ksm.helpers;
-
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.client.rest.response.BucketInfo;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos
-    .ServicePort;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeType;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-/**
- * ServiceInfo holds the config details of Ozone services.
- */
-public final class ServiceInfo {
-
-  private static final ObjectReader READER =
-      new ObjectMapper().readerFor(ServiceInfo.class);
-  private static final ObjectWriter WRITER =
-      new ObjectMapper().writerWithDefaultPrettyPrinter();
-
-  /**
-   * Type of node/service.
-   */
-  private NodeType nodeType;
-  /**
-   * Hostname of the node in which the service is running.
-   */
-  private String hostname;
-
-  /**
-   * List of ports the service listens to.
-   */
-  private Map<ServicePort.Type, Integer> ports;
-
-  /**
-   * Default constructor for JSON deserialization.
-   */
-  public ServiceInfo() {}
-
-  /**
-   * Constructs the ServiceInfo for the {@code nodeType}.
-   * @param nodeType type of node/service
-   * @param hostname hostname of the service
-   * @param portList list of ports the service listens to
-   */
-  private ServiceInfo(
-      NodeType nodeType, String hostname, List<ServicePort> portList) {
-    Preconditions.checkNotNull(nodeType);
-    Preconditions.checkNotNull(hostname);
-    this.nodeType = nodeType;
-    this.hostname = hostname;
-    this.ports = new HashMap<>();
-    for (ServicePort port : portList) {
-      ports.put(port.getType(), port.getValue());
-    }
-  }
-
-  /**
-   * Returns the type of node/service.
-   * @return node type
-   */
-  public NodeType getNodeType() {
-    return nodeType;
-  }
-
-  /**
-   * Returns the hostname of the service.
-   * @return hostname
-   */
-  public String getHostname() {
-    return hostname;
-  }
-
-  /**
-   * Returns ServicePort.Type to port mappings.
-   * @return ports
-   */
-  public Map<ServicePort.Type, Integer> getPorts() {
-    return ports;
-  }
-
-  /**
-   * Returns the port for given type, null if the service doesn't support
-   * the type.
-   *
-   * @param type the type of port.
-   *             ex: RPC, HTTP, HTTPS, etc..
-   */
-  @JsonIgnore
-  public int getPort(ServicePort.Type type) {
-    return ports.get(type);
-  }
-
-  /**
-   * Converts {@link ServiceInfo} to KeySpaceManagerProtocolProtos.ServiceInfo.
-   *
-   * @return KeySpaceManagerProtocolProtos.ServiceInfo
-   */
-  @JsonIgnore
-  public KeySpaceManagerProtocolProtos.ServiceInfo getProtobuf() {
-    KeySpaceManagerProtocolProtos.ServiceInfo.Builder builder =
-        KeySpaceManagerProtocolProtos.ServiceInfo.newBuilder();
-    builder.setNodeType(nodeType)
-        .setHostname(hostname)
-        .addAllServicePorts(
-            ports.entrySet().stream()
-                .map(
-                    entry ->
-                        ServicePort.newBuilder()
-                            .setType(entry.getKey())
-                            .setValue(entry.getValue()).build())
-                .collect(Collectors.toList()));
-    return builder.build();
-  }
-
-  /**
-   * Converts KeySpaceManagerProtocolProtos.ServiceInfo to {@link ServiceInfo}.
-   *
-   * @return {@link ServiceInfo}
-   */
-  @JsonIgnore
-  public static ServiceInfo getFromProtobuf(
-      KeySpaceManagerProtocolProtos.ServiceInfo serviceInfo) {
-    return new ServiceInfo(serviceInfo.getNodeType(),
-        serviceInfo.getHostname(),
-        serviceInfo.getServicePortsList());
-  }
-
-  /**
-   * Returns a JSON string of this object.
-   *
-   * @return String - json string
-   * @throws IOException
-   */
-  public String toJsonString() throws IOException {
-    return WRITER.writeValueAsString(this);
-  }
-
-  /**
-   * Parse a JSON string into ServiceInfo Object.
-   *
-   * @param jsonString Json String
-   * @return BucketInfo
-   * @throws IOException
-   */
-  public static BucketInfo parse(String jsonString) throws IOException {
-    return READER.readValue(jsonString);
-  }
-
-  /**
-   * Creates a new builder to build {@link ServiceInfo}.
-   * @return {@link ServiceInfo.Builder}
-   */
-  public static Builder newBuilder() {
-    return new Builder();
-  }
-
-  /**
-   * Builder used to build/construct {@link ServiceInfo}.
-   */
-  public static class Builder {
-
-    private NodeType node;
-    private String host;
-    private List<ServicePort> portList = new ArrayList<>();
-
-
-    /**
-     * Sets the node/service type.
-     * @param nodeType type of node
-     * @return the builder
-     */
-    public Builder setNodeType(NodeType nodeType) {
-      node = nodeType;
-      return this;
-    }
-
-    /**
-     * Sets the hostname of the service.
-     * @param hostname service hostname
-     * @return the builder
-     */
-    public Builder setHostname(String hostname) {
-      host = hostname;
-      return this;
-    }
-
-    /**
-     * Adds the service port to the service port list.
-     * @param servicePort RPC port
-     * @return the builder
-     */
-    public Builder addServicePort(ServicePort servicePort) {
-      portList.add(servicePort);
-      return this;
-    }
-
-
-    /**
-     * Builds and returns {@link ServiceInfo} with the set values.
-     * @return {@link ServiceInfo}
-     */
-    public ServiceInfo build() {
-      return new ServiceInfo(node, host, portList);
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/VolumeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/VolumeArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/VolumeArgs.java
deleted file mode 100644
index 1a3d486..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/VolumeArgs.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-
-import com.google.common.base.Preconditions;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * A class that encapsulates the createVolume Args.
- */
-public final class VolumeArgs {
-  private final String adminName;
-  private final String ownerName;
-  private final String volume;
-  private final long quotaInBytes;
-  private final Map<String, String> extendedAttributes;
-
-  /**
-   * Private constructor, constructed via builder.
-   *
-   * @param adminName - Administrator name.
-   * @param ownerName - Volume owner's name
-   * @param volume - volume name
-   * @param quotaInBytes - Volume Quota in bytes.
-   * @param keyValueMap - keyValue map.
-   */
-  private VolumeArgs(String adminName, String ownerName, String volume,
-      long quotaInBytes, Map<String, String> keyValueMap) {
-    this.adminName = adminName;
-    this.ownerName = ownerName;
-    this.volume = volume;
-    this.quotaInBytes = quotaInBytes;
-    this.extendedAttributes = keyValueMap;
-  }
-
-  /**
-   * Returns the Admin Name.
-   *
-   * @return String.
-   */
-  public String getAdminName() {
-    return adminName;
-  }
-
-  /**
-   * Returns the owner Name.
-   *
-   * @return String
-   */
-  public String getOwnerName() {
-    return ownerName;
-  }
-
-  /**
-   * Returns the volume Name.
-   *
-   * @return String
-   */
-  public String getVolume() {
-    return volume;
-  }
-
-  /**
-   * Returns Quota in Bytes.
-   *
-   * @return long, Quota in bytes.
-   */
-  public long getQuotaInBytes() {
-    return quotaInBytes;
-  }
-
-  public Map<String, String> getExtendedAttributes() {
-    return extendedAttributes;
-  }
-
-  static class Builder {
-    private String adminName;
-    private String ownerName;
-    private String volume;
-    private long quotaInBytes;
-    private Map<String, String> extendedAttributes;
-
-    /**
-     * Constructs a builder.
-     */
-    Builder() {
-      extendedAttributes = new HashMap<>();
-    }
-
-    public void setAdminName(String adminName) {
-      this.adminName = adminName;
-    }
-
-    public void setOwnerName(String ownerName) {
-      this.ownerName = ownerName;
-    }
-
-    public void setVolume(String volume) {
-      this.volume = volume;
-    }
-
-    public void setQuotaInBytes(long quotaInBytes) {
-      this.quotaInBytes = quotaInBytes;
-    }
-
-    public void addMetadata(String key, String value) {
-      extendedAttributes.put(key, value); // overwrite if present.
-    }
-
-    /**
-     * Constructs a CreateVolumeArgument.
-     *
-     * @return CreateVolumeArgs.
-     */
-    public VolumeArgs build() {
-      Preconditions.checkNotNull(adminName);
-      Preconditions.checkNotNull(ownerName);
-      Preconditions.checkNotNull(volume);
-      return new VolumeArgs(adminName, ownerName, volume, quotaInBytes,
-          extendedAttributes);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/package-info.java
deleted file mode 100644
index ce627a5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java
deleted file mode 100644
index 7698ee1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm;
-/**
- This package contains client side protocol library to communicate with KSM.
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java
deleted file mode 100644
index 5da5a27..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.protocol;
-
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.OzoneAclInfo;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Protocol to talk to KSM.
- */
-public interface KeySpaceManagerProtocol {
-
-  /**
-   * Creates a volume.
-   * @param args - Arguments to create Volume.
-   * @throws IOException
-   */
-  void createVolume(KsmVolumeArgs args) throws IOException;
-
-  /**
-   * Changes the owner of a volume.
-   * @param volume  - Name of the volume.
-   * @param owner - Name of the owner.
-   * @throws IOException
-   */
-  void setOwner(String volume, String owner) throws IOException;
-
-  /**
-   * Changes the Quota on a volume.
-   * @param volume - Name of the volume.
-   * @param quota - Quota in bytes.
-   * @throws IOException
-   */
-  void setQuota(String volume, long quota) throws IOException;
-
-  /**
-   * Checks if the specified user can access this volume.
-   * @param volume - volume
-   * @param userAcl - user acls which needs to be checked for access
-   * @return true if the user has required access for the volume,
-   *         false otherwise
-   * @throws IOException
-   */
-  boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl)
-      throws IOException;
-
-  /**
-   * Gets the volume information.
-   * @param volume - Volume name.
-   * @return VolumeArgs or exception is thrown.
-   * @throws IOException
-   */
-  KsmVolumeArgs getVolumeInfo(String volume) throws IOException;
-
-  /**
-   * Deletes an existing empty volume.
-   * @param volume - Name of the volume.
-   * @throws IOException
-   */
-  void deleteVolume(String volume) throws IOException;
-
-  /**
-   * Lists volume owned by a specific user.
-   * @param userName - user name
-   * @param prefix  - Filter prefix -- Return only entries that match this.
-   * @param prevKey - Previous key -- List starts from the next from the prevkey
-   * @param maxKeys - Max number of keys to return.
-   * @return List of Volumes.
-   * @throws IOException
-   */
-  List<KsmVolumeArgs> listVolumeByUser(String userName, String prefix, String
-      prevKey, int maxKeys) throws IOException;
-
-  /**
-   * Lists volume all volumes in the cluster.
-   * @param prefix  - Filter prefix -- Return only entries that match this.
-   * @param prevKey - Previous key -- List starts from the next from the prevkey
-   * @param maxKeys - Max number of keys to return.
-   * @return List of Volumes.
-   * @throws IOException
-   */
-  List<KsmVolumeArgs> listAllVolumes(String prefix, String
-      prevKey, int maxKeys) throws IOException;
-
-  /**
-   * Creates a bucket.
-   * @param bucketInfo - BucketInfo to create Bucket.
-   * @throws IOException
-   */
-  void createBucket(KsmBucketInfo bucketInfo) throws IOException;
-
-  /**
-   * Gets the bucket information.
-   * @param volumeName - Volume name.
-   * @param bucketName - Bucket name.
-   * @return KsmBucketInfo or exception is thrown.
-   * @throws IOException
-   */
-  KsmBucketInfo getBucketInfo(String volumeName, String bucketName)
-      throws IOException;
-
-  /**
-   * Sets bucket property from args.
-   * @param args - BucketArgs.
-   * @throws IOException
-   */
-  void setBucketProperty(KsmBucketArgs args) throws IOException;
-
-  /**
-   * Open the given key and return an open key session.
-   *
-   * @param args the args of the key.
-   * @return OpenKeySession instance that client uses to talk to container.
-   * @throws IOException
-   */
-  OpenKeySession openKey(KsmKeyArgs args) throws IOException;
-
-  /**
-   * Commit a key. This will make the change from the client visible. The client
-   * is identified by the clientID.
-   *
-   * @param args the key to commit
-   * @param clientID the client identification
-   * @throws IOException
-   */
-  void commitKey(KsmKeyArgs args, int clientID) throws IOException;
-
-  /**
-   * Allocate a new block, it is assumed that the client is having an open key
-   * session going on. This block will be appended to this open key session.
-   *
-   * @param args the key to append
-   * @param clientID the client identification
-   * @return an allocated block
-   * @throws IOException
-   */
-  KsmKeyLocationInfo allocateBlock(KsmKeyArgs args, int clientID)
-      throws IOException;
-
-  /**
-   * Look up for the container of an existing key.
-   *
-   * @param args the args of the key.
-   * @return KsmKeyInfo isntacne that client uses to talk to container.
-   * @throws IOException
-   */
-  KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException;
-
-  /**
-   * Deletes an existing key.
-   *
-   * @param args the args of the key.
-   * @throws IOException
-   */
-  void deleteKey(KsmKeyArgs args) throws IOException;
-
-  /**
-   * Deletes an existing empty bucket from volume.
-   * @param volume - Name of the volume.
-   * @param bucket - Name of the bucket.
-   * @throws IOException
-   */
-  void deleteBucket(String volume, String bucket) throws IOException;
-
-  /**
-   * Returns a list of buckets represented by {@link KsmBucketInfo}
-   * in the given volume. Argument volumeName is required, others
-   * are optional.
-   *
-   * @param volumeName
-   *   the name of the volume.
-   * @param startBucketName
-   *   the start bucket name, only the buckets whose name is
-   *   after this value will be included in the result.
-   * @param bucketPrefix
-   *   bucket name prefix, only the buckets whose name has
-   *   this prefix will be included in the result.
-   * @param maxNumOfBuckets
-   *   the maximum number of buckets to return. It ensures
-   *   the size of the result will not exceed this limit.
-   * @return a list of buckets.
-   * @throws IOException
-   */
-  List<KsmBucketInfo> listBuckets(String volumeName,
-      String startBucketName, String bucketPrefix, int maxNumOfBuckets)
-      throws IOException;
-
-  /**
-   * Returns a list of keys represented by {@link KsmKeyInfo}
-   * in the given bucket. Argument volumeName, bucketName is required,
-   * others are optional.
-   *
-   * @param volumeName
-   *   the name of the volume.
-   * @param bucketName
-   *   the name of the bucket.
-   * @param startKeyName
-   *   the start key name, only the keys whose name is
-   *   after this value will be included in the result.
-   * @param keyPrefix
-   *   key name prefix, only the keys whose name has
-   *   this prefix will be included in the result.
-   * @param maxKeys
-   *   the maximum number of keys to return. It ensures
-   *   the size of the result will not exceed this limit.
-   * @return a list of keys.
-   * @throws IOException
-   */
-  List<KsmKeyInfo> listKeys(String volumeName,
-      String bucketName, String startKeyName, String keyPrefix, int maxKeys)
-      throws IOException;
-
-  /**
-   * Returns list of Ozone services with its configuration details.
-   *
-   * @return list of Ozone services
-   * @throws IOException
-   */
-  List<ServiceInfo> getServiceList() throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/package-info.java
deleted file mode 100644
index f77e5fd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.ksm.protocol;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
deleted file mode 100644
index cc215cf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
+++ /dev/null
@@ -1,744 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.protocolPB;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolTranslator;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
-import org.apache.hadoop.ozone.ksm.protocol.KeySpaceManagerProtocol;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.AllocateBlockRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.AllocateBlockResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.CommitKeyRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.CommitKeyResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.BucketArgs;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.BucketInfo;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.CreateBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.CreateBucketResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.InfoBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.InfoBucketResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.SetBucketPropertyRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.SetBucketPropertyResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.DeleteBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.DeleteBucketResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.CreateVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.CreateVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.LocateKeyRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.LocateKeyResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.KeyArgs;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.SetVolumePropertyRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.SetVolumePropertyResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.DeleteVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.DeleteVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.InfoVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.InfoVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.CheckVolumeAccessRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.CheckVolumeAccessResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ListBucketsRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ListBucketsResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ListKeysRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ListKeysResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.VolumeInfo;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.Status;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.OzoneAclInfo;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ListVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ListVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ServiceListRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ServiceListResponse;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.stream.Collectors;
-
-/**
- *  The client side implementation of KeySpaceManagerProtocol.
- */
-
-@InterfaceAudience.Private
-public final class KeySpaceManagerProtocolClientSideTranslatorPB
-    implements KeySpaceManagerProtocol, ProtocolTranslator, Closeable {
-
-  /**
-   * RpcController is not used and hence is set to null.
-   */
-  private static final RpcController NULL_RPC_CONTROLLER = null;
-
-  private final KeySpaceManagerProtocolPB rpcProxy;
-
-  /**
-   * Constructor for KeySpaceManger Client.
-   * @param rpcProxy
-   */
-  public KeySpaceManagerProtocolClientSideTranslatorPB(
-      KeySpaceManagerProtocolPB rpcProxy) {
-    this.rpcProxy = rpcProxy;
-  }
-
-  /**
-   * Closes this stream and releases any system resources associated
-   * with it. If the stream is already closed then invoking this
-   * method has no effect.
-   * <p>
-   * <p> As noted in {@link AutoCloseable#close()}, cases where the
-   * close may fail require careful attention. It is strongly advised
-   * to relinquish the underlying resources and to internally
-   * <em>mark</em> the {@code Closeable} as closed, prior to throwing
-   * the {@code IOException}.
-   *
-   * @throws IOException if an I/O error occurs
-   */
-  @Override
-  public void close() throws IOException {
-
-  }
-
-  /**
-   * Creates a volume.
-   *
-   * @param args - Arguments to create Volume.
-   * @throws IOException
-   */
-  @Override
-  public void createVolume(KsmVolumeArgs args) throws IOException {
-    CreateVolumeRequest.Builder req =
-        CreateVolumeRequest.newBuilder();
-    VolumeInfo volumeInfo = args.getProtobuf();
-    req.setVolumeInfo(volumeInfo);
-
-    final CreateVolumeResponse resp;
-    try {
-      resp = rpcProxy.createVolume(NULL_RPC_CONTROLLER,
-          req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-
-    if (resp.getStatus() != Status.OK) {
-      throw new
-          IOException("Volume creation failed, error:" + resp.getStatus());
-    }
-  }
-
-  /**
-   * Changes the owner of a volume.
-   *
-   * @param volume - Name of the volume.
-   * @param owner - Name of the owner.
-   * @throws IOException
-   */
-  @Override
-  public void setOwner(String volume, String owner) throws IOException {
-    SetVolumePropertyRequest.Builder req =
-        SetVolumePropertyRequest.newBuilder();
-    req.setVolumeName(volume).setOwnerName(owner);
-    final SetVolumePropertyResponse resp;
-    try {
-      resp = rpcProxy.setVolumeProperty(NULL_RPC_CONTROLLER, req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    if (resp.getStatus() != Status.OK) {
-      throw new
-          IOException("Volume owner change failed, error:" + resp.getStatus());
-    }
-  }
-
-  /**
-   * Changes the Quota on a volume.
-   *
-   * @param volume - Name of the volume.
-   * @param quota - Quota in bytes.
-   * @throws IOException
-   */
-  @Override
-  public void setQuota(String volume, long quota) throws IOException {
-    SetVolumePropertyRequest.Builder req =
-        SetVolumePropertyRequest.newBuilder();
-    req.setVolumeName(volume).setQuotaInBytes(quota);
-    final SetVolumePropertyResponse resp;
-    try {
-      resp = rpcProxy.setVolumeProperty(NULL_RPC_CONTROLLER, req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    if (resp.getStatus() != Status.OK) {
-      throw new
-          IOException("Volume quota change failed, error:" + resp.getStatus());
-    }
-  }
-
-  /**
-   * Checks if the specified user can access this volume.
-   *
-   * @param volume - volume
-   * @param userAcl - user acls which needs to be checked for access
-   * @return true if the user has required access for the volume,
-   *         false otherwise
-   * @throws IOException
-   */
-  @Override
-  public boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl) throws
-      IOException {
-    CheckVolumeAccessRequest.Builder req =
-        CheckVolumeAccessRequest.newBuilder();
-    req.setVolumeName(volume).setUserAcl(userAcl);
-    final CheckVolumeAccessResponse resp;
-    try {
-      resp = rpcProxy.checkVolumeAccess(NULL_RPC_CONTROLLER, req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-
-    if (resp.getStatus() == Status.ACCESS_DENIED) {
-      return false;
-    } else if (resp.getStatus() == Status.OK) {
-      return true;
-    } else {
-      throw new
-          IOException("Check Volume Access failed, error:" + resp.getStatus());
-    }
-  }
-
-  /**
-   * Gets the volume information.
-   *
-   * @param volume - Volume name.
-   * @return KsmVolumeArgs or exception is thrown.
-   * @throws IOException
-   */
-  @Override
-  public KsmVolumeArgs getVolumeInfo(String volume) throws IOException {
-    InfoVolumeRequest.Builder req = InfoVolumeRequest.newBuilder();
-    req.setVolumeName(volume);
-    final InfoVolumeResponse resp;
-    try {
-      resp = rpcProxy.infoVolume(NULL_RPC_CONTROLLER, req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    if (resp.getStatus() != Status.OK) {
-      throw new
-          IOException("Info Volume failed, error:" + resp.getStatus());
-    }
-    return KsmVolumeArgs.getFromProtobuf(resp.getVolumeInfo());
-  }
-
-  /**
-   * Deletes an existing empty volume.
-   *
-   * @param volume - Name of the volume.
-   * @throws IOException
-   */
-  @Override
-  public void deleteVolume(String volume) throws IOException {
-    DeleteVolumeRequest.Builder req = DeleteVolumeRequest.newBuilder();
-    req.setVolumeName(volume);
-    final DeleteVolumeResponse resp;
-    try {
-      resp = rpcProxy.deleteVolume(NULL_RPC_CONTROLLER, req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    if (resp.getStatus() != Status.OK) {
-      throw new
-          IOException("Delete Volume failed, error:" + resp.getStatus());
-    }
-  }
-
-  /**
-   * Lists volume owned by a specific user.
-   *
-   * @param userName - user name
-   * @param prefix - Filter prefix -- Return only entries that match this.
-   * @param prevKey - Previous key -- List starts from the next from the
-   * prevkey
-   * @param maxKeys - Max number of keys to return.
-   * @return List of Volumes.
-   * @throws IOException
-   */
-  @Override
-  public List<KsmVolumeArgs> listVolumeByUser(String userName, String prefix,
-                                              String prevKey, int maxKeys)
-      throws IOException {
-    ListVolumeRequest.Builder builder = ListVolumeRequest.newBuilder();
-    if (!Strings.isNullOrEmpty(prefix)) {
-      builder.setPrefix(prefix);
-    }
-    if (!Strings.isNullOrEmpty(prevKey)) {
-      builder.setPrevKey(prevKey);
-    }
-    builder.setMaxKeys(maxKeys);
-    builder.setUserName(userName);
-    builder.setScope(ListVolumeRequest.Scope.VOLUMES_BY_USER);
-    return listVolume(builder.build());
-  }
-
-  /**
-   * Lists volume all volumes in the cluster.
-   *
-   * @param prefix - Filter prefix -- Return only entries that match this.
-   * @param prevKey - Previous key -- List starts from the next from the
-   * prevkey
-   * @param maxKeys - Max number of keys to return.
-   * @return List of Volumes.
-   * @throws IOException
-   */
-  @Override
-  public List<KsmVolumeArgs> listAllVolumes(String prefix, String prevKey,
-      int maxKeys) throws IOException {
-    ListVolumeRequest.Builder builder = ListVolumeRequest.newBuilder();
-    if (!Strings.isNullOrEmpty(prefix)) {
-      builder.setPrefix(prefix);
-    }
-    if (!Strings.isNullOrEmpty(prevKey)) {
-      builder.setPrevKey(prevKey);
-    }
-    builder.setMaxKeys(maxKeys);
-    builder.setScope(ListVolumeRequest.Scope.VOLUMES_BY_CLUSTER);
-    return listVolume(builder.build());
-  }
-
-  private List<KsmVolumeArgs> listVolume(ListVolumeRequest request)
-      throws IOException {
-    final ListVolumeResponse resp;
-    try {
-      resp = rpcProxy.listVolumes(NULL_RPC_CONTROLLER, request);
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-
-    if (resp.getStatus() != Status.OK) {
-      throw new IOException("List volume failed, error: "
-          + resp.getStatus());
-    }
-
-    List<KsmVolumeArgs> result = Lists.newArrayList();
-    for (VolumeInfo volInfo : resp.getVolumeInfoList()) {
-      KsmVolumeArgs volArgs = KsmVolumeArgs.getFromProtobuf(volInfo);
-      result.add(volArgs);
-    }
-
-    return resp.getVolumeInfoList().stream()
-        .map(item -> KsmVolumeArgs.getFromProtobuf(item))
-        .collect(Collectors.toList());
-  }
-
-  /**
-   * Creates a bucket.
-   *
-   * @param bucketInfo - BucketInfo to create bucket.
-   * @throws IOException
-   */
-  @Override
-  public void createBucket(KsmBucketInfo bucketInfo) throws IOException {
-    CreateBucketRequest.Builder req =
-        CreateBucketRequest.newBuilder();
-    BucketInfo bucketInfoProtobuf = bucketInfo.getProtobuf();
-    req.setBucketInfo(bucketInfoProtobuf);
-
-    final CreateBucketResponse resp;
-    try {
-      resp = rpcProxy.createBucket(NULL_RPC_CONTROLLER,
-          req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    if (resp.getStatus() != Status.OK) {
-      throw new IOException("Bucket creation failed, error: "
-          + resp.getStatus());
-    }
-  }
-
-  /**
-   * Gets the bucket information.
-   *
-   * @param volume - Volume name.
-   * @param bucket - Bucket name.
-   * @return KsmBucketInfo or exception is thrown.
-   * @throws IOException
-   */
-  @Override
-  public KsmBucketInfo getBucketInfo(String volume, String bucket)
-      throws IOException {
-    InfoBucketRequest.Builder req =
-        InfoBucketRequest.newBuilder();
-    req.setVolumeName(volume);
-    req.setBucketName(bucket);
-
-    final InfoBucketResponse resp;
-    try {
-      resp = rpcProxy.infoBucket(NULL_RPC_CONTROLLER,
-          req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    if (resp.getStatus() == Status.OK) {
-      return KsmBucketInfo.getFromProtobuf(resp.getBucketInfo());
-    } else {
-      throw new IOException("Info Bucket failed, error: "
-          + resp.getStatus());
-    }
-  }
-
-  /**
-   * Sets bucket property from args.
-   * @param args - BucketArgs.
-   * @throws IOException
-   */
-  @Override
-  public void setBucketProperty(KsmBucketArgs args)
-      throws IOException {
-    SetBucketPropertyRequest.Builder req =
-        SetBucketPropertyRequest.newBuilder();
-    BucketArgs bucketArgs = args.getProtobuf();
-    req.setBucketArgs(bucketArgs);
-    final SetBucketPropertyResponse resp;
-    try {
-      resp = rpcProxy.setBucketProperty(NULL_RPC_CONTROLLER,
-          req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    if (resp.getStatus() != Status.OK) {
-      throw new IOException("Setting bucket property failed, error: "
-          + resp.getStatus());
-    }
-  }
-
-  /**
-   * List buckets in a volume.
-   *
-   * @param volumeName
-   * @param startKey
-   * @param prefix
-   * @param count
-   * @return
-   * @throws IOException
-   */
-  @Override
-  public List<KsmBucketInfo> listBuckets(String volumeName,
-      String startKey, String prefix, int count) throws IOException {
-    List<KsmBucketInfo> buckets = new ArrayList<>();
-    ListBucketsRequest.Builder reqBuilder = ListBucketsRequest.newBuilder();
-    reqBuilder.setVolumeName(volumeName);
-    reqBuilder.setCount(count);
-    if (startKey != null) {
-      reqBuilder.setStartKey(startKey);
-    }
-    if (prefix != null) {
-      reqBuilder.setPrefix(prefix);
-    }
-    ListBucketsRequest request = reqBuilder.build();
-    final ListBucketsResponse resp;
-    try {
-      resp = rpcProxy.listBuckets(NULL_RPC_CONTROLLER, request);
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-
-    if (resp.getStatus() == Status.OK) {
-      buckets.addAll(
-          resp.getBucketInfoList().stream()
-              .map(KsmBucketInfo::getFromProtobuf)
-              .collect(Collectors.toList()));
-      return buckets;
-    } else {
-      throw new IOException("List Buckets failed, error: "
-          + resp.getStatus());
-    }
-  }
-
-  /**
-   * Create a new open session of the key, then use the returned meta info to
-   * talk to data node to actually write the key.
-   * @param args the args for the key to be allocated
-   * @return a handler to the key, returned client
-   * @throws IOException
-   */
-  @Override
-  public OpenKeySession openKey(KsmKeyArgs args) throws IOException {
-    LocateKeyRequest.Builder req = LocateKeyRequest.newBuilder();
-    KeyArgs.Builder keyArgs = KeyArgs.newBuilder()
-        .setVolumeName(args.getVolumeName())
-        .setBucketName(args.getBucketName())
-        .setFactor(args.getFactor())
-        .setType(args.getType())
-        .setKeyName(args.getKeyName());
-    if (args.getDataSize() > 0) {
-      keyArgs.setDataSize(args.getDataSize());
-    }
-    req.setKeyArgs(keyArgs.build());
-
-    final LocateKeyResponse resp;
-    try {
-      resp = rpcProxy.createKey(NULL_RPC_CONTROLLER, req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    if (resp.getStatus() != Status.OK) {
-      throw new IOException("Create key failed, error:" + resp.getStatus());
-    }
-    return new OpenKeySession(resp.getID(),
-        KsmKeyInfo.getFromProtobuf(resp.getKeyInfo()), resp.getOpenVersion());
-  }
-
-  @Override
-  public KsmKeyLocationInfo allocateBlock(KsmKeyArgs args, int clientID)
-      throws IOException {
-    AllocateBlockRequest.Builder req = AllocateBlockRequest.newBuilder();
-    KeyArgs keyArgs = KeyArgs.newBuilder()
-        .setVolumeName(args.getVolumeName())
-        .setBucketName(args.getBucketName())
-        .setKeyName(args.getKeyName())
-        .setFactor(args.getFactor())
-        .setType(args.getType())
-        .setDataSize(args.getDataSize()).build();
-    req.setKeyArgs(keyArgs);
-    req.setClientID(clientID);
-
-    final AllocateBlockResponse resp;
-    try {
-      resp = rpcProxy.allocateBlock(NULL_RPC_CONTROLLER, req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    if (resp.getStatus() != Status.OK) {
-      throw new IOException("Allocate block failed, error:" +
-          resp.getStatus());
-    }
-    return KsmKeyLocationInfo.getFromProtobuf(resp.getKeyLocation());
-  }
-
-  @Override
-  public void commitKey(KsmKeyArgs args, int clientID)
-      throws IOException {
-    CommitKeyRequest.Builder req = CommitKeyRequest.newBuilder();
-    KeyArgs keyArgs = KeyArgs.newBuilder()
-        .setVolumeName(args.getVolumeName())
-        .setBucketName(args.getBucketName())
-        .setKeyName(args.getKeyName())
-        .setDataSize(args.getDataSize()).build();
-    req.setKeyArgs(keyArgs);
-    req.setClientID(clientID);
-
-    final CommitKeyResponse resp;
-    try {
-      resp = rpcProxy.commitKey(NULL_RPC_CONTROLLER, req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    if (resp.getStatus() != Status.OK) {
-      throw new IOException("Commit key failed, error:" +
-          resp.getStatus());
-    }
-  }
-
-
-  @Override
-  public KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException {
-    LocateKeyRequest.Builder req = LocateKeyRequest.newBuilder();
-    KeyArgs keyArgs = KeyArgs.newBuilder()
-        .setVolumeName(args.getVolumeName())
-        .setBucketName(args.getBucketName())
-        .setKeyName(args.getKeyName())
-        .setDataSize(args.getDataSize()).build();
-    req.setKeyArgs(keyArgs);
-
-    final LocateKeyResponse resp;
-    try {
-      resp = rpcProxy.lookupKey(NULL_RPC_CONTROLLER, req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    if (resp.getStatus() != Status.OK) {
-      throw new IOException("Lookup key failed, error:" +
-          resp.getStatus());
-    }
-    return KsmKeyInfo.getFromProtobuf(resp.getKeyInfo());
-  }
-
-  /**
-   * Deletes an existing key.
-   *
-   * @param args the args of the key.
-   * @throws IOException
-   */
-  @Override
-  public void deleteKey(KsmKeyArgs args) throws IOException {
-    LocateKeyRequest.Builder req = LocateKeyRequest.newBuilder();
-    KeyArgs keyArgs = KeyArgs.newBuilder()
-        .setVolumeName(args.getVolumeName())
-        .setBucketName(args.getBucketName())
-        .setKeyName(args.getKeyName()).build();
-    req.setKeyArgs(keyArgs);
-
-    final LocateKeyResponse resp;
-    try {
-      resp = rpcProxy.deleteKey(NULL_RPC_CONTROLLER, req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    if (resp.getStatus() != Status.OK) {
-      throw new IOException("Delete key failed, error:" +
-          resp.getStatus());
-    }
-  }
-
-  /**
-   * Deletes an existing empty bucket from volume.
-   * @param volume - Name of the volume.
-   * @param bucket - Name of the bucket.
-   * @throws IOException
-   */
-  public void deleteBucket(String volume, String bucket) throws IOException {
-    DeleteBucketRequest.Builder req = DeleteBucketRequest.newBuilder();
-    req.setVolumeName(volume);
-    req.setBucketName(bucket);
-    final DeleteBucketResponse resp;
-    try {
-      resp = rpcProxy.deleteBucket(NULL_RPC_CONTROLLER, req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    if (resp.getStatus() != Status.OK) {
-      throw new
-          IOException("Delete Bucket failed, error:" + resp.getStatus());
-    }
-  }
-
-  /**
-   * List keys in a bucket.
-   */
-  @Override
-  public List<KsmKeyInfo> listKeys(String volumeName, String bucketName,
-      String startKey, String prefix, int maxKeys) throws IOException {
-    List<KsmKeyInfo> keys = new ArrayList<>();
-    ListKeysRequest.Builder reqBuilder = ListKeysRequest.newBuilder();
-    reqBuilder.setVolumeName(volumeName);
-    reqBuilder.setBucketName(bucketName);
-    reqBuilder.setCount(maxKeys);
-
-    if (startKey != null) {
-      reqBuilder.setStartKey(startKey);
-    }
-
-    if (prefix != null) {
-      reqBuilder.setPrefix(prefix);
-    }
-
-    ListKeysRequest request = reqBuilder.build();
-    final ListKeysResponse resp;
-    try {
-      resp = rpcProxy.listKeys(NULL_RPC_CONTROLLER, request);
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-
-    if (resp.getStatus() == Status.OK) {
-      keys.addAll(
-          resp.getKeyInfoList().stream()
-              .map(KsmKeyInfo::getFromProtobuf)
-              .collect(Collectors.toList()));
-      return keys;
-    } else {
-      throw new IOException("List Keys failed, error: "
-          + resp.getStatus());
-    }
-  }
-
-  @Override
-  public List<ServiceInfo> getServiceList() throws IOException {
-    ServiceListRequest request = ServiceListRequest.newBuilder().build();
-    final ServiceListResponse resp;
-    try {
-      resp = rpcProxy.getServiceList(NULL_RPC_CONTROLLER, request);
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-
-    if (resp.getStatus() == Status.OK) {
-      return resp.getServiceInfoList().stream()
-              .map(ServiceInfo::getFromProtobuf)
-              .collect(Collectors.toList());
-    } else {
-      throw new IOException("Getting service list failed, error: "
-          + resp.getStatus());
-    }
-  }
-
-  /**
-   * Return the proxy object underlying this protocol translator.
-   *
-   * @return the proxy object underlying this protocol translator.
-   */
-  @Override
-  public Object getUnderlyingProxyObject() {
-    return null;
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[10/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto
deleted file mode 100644
index 7306b81..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto
+++ /dev/null
@@ -1,367 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are private and unstable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
- * for what changes are allowed for a *unstable* .proto interface.
- */
-
-option java_package = "org.apache.hadoop.ozone.protocol.proto";
-
-option java_outer_classname = "StorageContainerDatanodeProtocolProtos";
-
-option java_generic_services = true;
-
-option java_generate_equals_and_hash = true;
-
-package hadoop.hdfs;
-
-import "hdfs.proto";
-
-import "HdfsServer.proto";
-
-import "DatanodeProtocol.proto";
-
-import "Ozone.proto";
-
-
-/**
-* This message is send by data node to indicate that it is alive or it is
-* registering with the node manager.
-*/
-message SCMHeartbeatRequestProto {
-  required DatanodeIDProto datanodeID = 1;
-  optional SCMNodeReport nodeReport = 2;
-  optional ReportState containerReportState = 3;
-}
-
-enum DatanodeContainerState {
-  closed = 0;
-  open = 1;
-}
-
-/**
-NodeState contains messages from datanode to SCM saying that it has
-some information that SCM might be interested in.*/
-message ReportState {
-  enum states {
-    noContainerReports = 0;
-    completeContinerReport = 1;
-    deltaContainerReport = 2;
-  }
-  required states state = 1;
-  required int64 count = 2 [default = 0];
-}
-
-
-/**
-This message is used to persist the information about a container in the
-SCM database, This information allows SCM to startup faster and avoid having
-all container info in memory all the time.
-  */
-message ContainerPersistanceProto {
-  required DatanodeContainerState state = 1;
-  required hadoop.hdfs.ozone.Pipeline pipeline = 2;
-  required ContainerInfo info = 3;
-}
-
-/**
-This message is used to do a quick look up of which containers are effected
-if a node goes down
-*/
-message NodeContianerMapping {
-  repeated string contianerName = 1;
-}
-
-/**
-A container report contains the following information.
-*/
-message ContainerInfo {
-  required string containerName = 1;
-  optional string finalhash = 2;
-  optional int64 size = 3;
-  optional int64 used = 4;
-  optional int64 keyCount = 5;
-  // TODO: move the io count to separate message
-  optional int64 readCount = 6;
-  optional int64 writeCount = 7;
-  optional int64 readBytes = 8;
-  optional int64 writeBytes = 9;
-  required int64 containerID = 10;
-  optional hadoop.hdfs.ozone.LifeCycleState state = 11;
-}
-
-// The deleted blocks which are stored in deletedBlock.db of scm.
-message DeletedBlocksTransaction {
-  required int64 txID = 1;
-  required string containerName = 2;
-  repeated string blockID = 3;
-  // the retry time of sending deleting command to datanode.
-  required int32 count = 4;
-}
-
-/**
-A set of container reports, max count is generally set to
-8192 since that keeps the size of the reports under 1 MB.
-*/
-message ContainerReportsRequestProto {
-  enum reportType {
-    fullReport = 0;
-    deltaReport = 1;
-  }
-  required DatanodeIDProto datanodeID = 1;
-  repeated ContainerInfo reports = 2;
-  required reportType type = 3;
-}
-
-message ContainerReportsResponseProto {
-}
-
-/**
-* This message is send along with the heart beat to report datanode
-* storage utilization by SCM.
-*/
-message SCMNodeReport {
-  repeated SCMStorageReport storageReport = 1;
-}
-
-message SCMStorageReport {
-  required string storageUuid = 1;
-  optional uint64 capacity = 2 [default = 0];
-  optional uint64 scmUsed = 3 [default = 0];
-  optional uint64 remaining = 4 [default = 0];
-  optional StorageTypeProto storageType = 5 [default = DISK];
-}
-
-message SCMRegisterRequestProto {
-  required DatanodeIDProto datanodeID = 1;
-  optional SCMNodeAddressList addressList = 2;
-}
-
-/**
- * Request for version info of the software stack on the server.
- */
-message SCMVersionRequestProto {
-
-}
-
-/**
-* Generic response that is send to a version request. This allows keys to be
-* added on the fly and protocol to remain stable.
-*/
-message SCMVersionResponseProto {
-  required uint32 softwareVersion = 1;
-  repeated hadoop.hdfs.ozone.KeyValue keys = 2;
-}
-
-message SCMNodeAddressList {
-  repeated string addressList = 1;
-}
-
-/**
- * Datanode ID returned by the SCM. This is similar to name node
- * registeration of a datanode.
- */
-message SCMRegisteredCmdResponseProto {
-  enum ErrorCode {
-    success = 1;
-    errorNodeNotPermitted = 2;
-  }
-  required ErrorCode errorCode = 2;
-  optional string datanodeUUID = 3;
-  optional string clusterID = 4;
-  optional SCMNodeAddressList addressList = 5;
-}
-
-/**
- * SCM informs a datanode to register itself again.
- * With recieving this command, datanode will transit to REGISTER state.
- */
-message SCMReregisterCmdResponseProto {}
-
-/**
- * Container ID maintains the container's Identity along with cluster ID
- * after the registration is done.
- */
-message ContainerNodeIDProto {
-  required DatanodeIDProto datanodeID = 1;
-  optional string clusterID = 2;
-}
-
-
-
-/**
-This command tells the data node to send in the container report when possible
-*/
-message SendContainerReportProto {
-}
-
-/**
-This command asks the datanode to close a specific container.
-*/
-message SCMCloseContainerCmdResponseProto {
-  required string containerName = 1;
-}
-
-/**
-Type of commands supported by SCM to datanode protocol.
-*/
-enum Type {
-  versionCommand = 2;
-  registeredCommand = 3;
-  sendContainerReport = 4;
-  reregisterCommand = 5;
-  deleteBlocksCommand = 6;
-  closeContainerCommand = 7;
-}
-
-/*
- * These are commands returned by SCM for to the datanode to execute.
- */
-message SCMCommandResponseProto {
-  required Type cmdType = 2; // Type of the command
-  optional SCMRegisteredCmdResponseProto registeredProto = 3;
-  optional SCMVersionResponseProto versionProto = 4;
-  optional SendContainerReportProto sendReport = 5;
-  optional SCMReregisterCmdResponseProto reregisterProto = 6;
-  optional SCMDeleteBlocksCmdResponseProto deleteBlocksProto = 7;
-  required string datanodeUUID = 8;
-  optional SCMCloseContainerCmdResponseProto closeContainerProto = 9;
-}
-
-
-/*
- * A group of commands for the datanode to execute
- */
-message SCMHeartbeatResponseProto {
-  repeated SCMCommandResponseProto commands = 1;
-}
-
-// HB response from SCM, contains a list of block deletion transactions.
-message SCMDeleteBlocksCmdResponseProto {
-  repeated DeletedBlocksTransaction deletedBlocksTransactions = 1;
-}
-
-// SendACK response returned by datanode to SCM, currently empty.
-message ContainerBlocksDeletionACKResponseProto {
-}
-
-// ACK message datanode sent to SCM, contains the result of
-// block deletion transactions.
-message ContainerBlocksDeletionACKProto {
-  message DeleteBlockTransactionResult {
-    required int64 txID = 1;
-    required bool success = 2;
-  }
-  repeated DeleteBlockTransactionResult results = 1;
-}
-
-/**
- * Protocol used from a datanode to StorageContainerManager.
- *
- * Please see the request and response messages for details of the RPC calls.
- *
- * Here is a simple state diagram that shows how a datanode would boot up and
- * communicate with SCM.
- *
- *           -----------------------
- *          |         Start         |
- *           ---------- ------------
- *                     |
- *                     |
- *                     |
- *                     |
- *                     |
- *                     |
- *                     |
- *           ----------v-------------
- *          |   Searching for  SCM    ------------
- *           ---------- -------------             |
- *                     |                          |
- *                     |                          |
- *                     |                ----------v-------------
- *                     |               | Register if needed     |
- *                     |                ----------- ------------
- *                     |                           |
- *                     v                           |
- *            ----------- ----------------         |
- *  ---------   Heartbeat state           <--------
- * |          --------^-------------------
- * |                  |
- * |                  |
- * |                  |
- * |                  |
- * |                  |
- * |                  |
- * |                  |
- *  ------------------
- *
- *
- *
- * Here is how this protocol is used by the datanode. When a datanode boots up
- * it moves into a stated called SEARCHING_SCM. In this state datanode is
- * trying to establish communication with the SCM. The address of the SCMs are
- * retrieved from the configuration information.
- *
- * In the SEARCHING_SCM state, only rpc call made by datanode is a getVersion
- * call to SCM. Once any of the SCMs reply, datanode checks if it has a local
- * persisted datanode ID. If it has this means that this datanode is already
- * registered with some SCM. If this file is not found, datanode assumes that
- * it needs to do a registration.
- *
- * If registration is need datanode moves into REGISTER state. It will
- * send a register call with datanodeID data structure and presist that info.
- *
- * The response to the command contains clusterID. This information is
- * also persisted by the datanode and moves into heartbeat state.
- *
- * Once in the heartbeat state, datanode sends heartbeats and container reports
- * to SCM and process commands issued by SCM until it is shutdown.
- *
- */
-service StorageContainerDatanodeProtocolService {
-
-  /**
-  * Gets the version information from the SCM.
-  */
-  rpc getVersion (SCMVersionRequestProto) returns (SCMVersionResponseProto);
-
-  /**
-  * Registers a data node with SCM.
-  */
-  rpc register (SCMRegisterRequestProto) returns (SCMRegisteredCmdResponseProto);
-
-  /**
-   * Send heartbeat from datanode to SCM. HB's under SCM looks more
-   * like life line protocol than HB's under HDFS. In other words, it is
-   * extremely light weight and contains no data payload.
-   */
-  rpc sendHeartbeat (SCMHeartbeatRequestProto) returns (SCMHeartbeatResponseProto);
-
-  /**
-    send container reports sends the container report to SCM. This will
-    return a null command as response.
-  */
-  rpc sendContainerReport(ContainerReportsRequestProto) returns (ContainerReportsResponseProto);
-
-  /**
-   * Sends the block deletion ACK to SCM.
-   */
-  rpc sendContainerBlocksDeletionACK (ContainerBlocksDeletionACKProto) returns (ContainerBlocksDeletionACKResponseProto);
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/com.sun.jersey.spi.container.ContainerProvider
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/com.sun.jersey.spi.container.ContainerProvider b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/com.sun.jersey.spi.container.ContainerProvider
deleted file mode 100644
index 2e103fe..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/com.sun.jersey.spi.container.ContainerProvider
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-org.apache.hadoop.ozone.web.netty.ObjectStoreJerseyContainerProvider

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml
deleted file mode 100644
index 97eaeae..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml
+++ /dev/null
@@ -1,1308 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<!-- Do not modify this file directly.  Instead, copy entries that you -->
-<!-- wish to modify from this file into ozone-site.xml and change them -->
-<!-- there.  If ozone-site.xml does not already exist, create it.      -->
-
-<!--Tags supported are OZONE, CBLOCK, MANAGEMENT, SECURITY, PERFORMANCE,   -->
-<!--DEBUG, CLIENT, SERVER, KSM, SCM, CRITICAL, RATIS, CONTAINER, REQUIRED, -->
-<!--REST, STORAGE, PIPELINE, STANDALONE                                    -->
-
-<configuration>
-  <!--CBlock Settings-->
-  <property>
-    <name>dfs.cblock.block.buffer.flush.interval</name>
-    <value>60s</value>
-    <tag>CBLOCK, PERFORMANCE</tag>
-    <description>
-      Controls the frequency at this the local cache flushes the
-      blocks to the remote containers.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.cache.block.buffer.size</name>
-    <value>512</value>
-    <tag>CBLOCK, PERFORMANCE</tag>
-    <description>
-      Size of the local cache for blocks. So cache size will be block
-      size multiplied by this number.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.cache.core.min.pool.size</name>
-    <value>16</value>
-    <tag>CBLOCK, PERFORMANCE</tag>
-    <description>
-      A minimum number of threads in the pool that cBlock cache will
-      use for the background I/O to remote containers.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.cache.max.pool.size</name>
-    <value>256</value>
-    <tag>CBLOCK, PERFORMANCE</tag>
-    <description>
-      Maximum number of threads in the pool that cBlock cache will
-      use for background I/O to remote containers.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.cache.keep.alive</name>
-    <value>60s</value>
-    <tag>CBLOCK, PERFORMANCE</tag>
-    <description>
-      If the cblock cache has no I/O, then the threads in the cache
-      pool are kept idle for this amount of time before shutting down.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.cache.leveldb.cache.size.mb</name>
-    <value>256</value>
-    <tag>CBLOCK, PERFORMANCE</tag>
-    <description>
-      The amount of physical memory allocated to the local cache. The
-      SCSI driver will allocate this much RAM cache instances.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.cache.max.retry</name>
-    <value>65536</value>
-    <tag>CBLOCK, PERFORMANCE</tag>
-    <description>
-      If the local cache is enabled then, CBlock writes to the local
-      cache when I/O happens. Then the background I/O threads write this
-      block to the remote containers. This value controls how many times the
-      background thread should attempt to do I/O to the remote containers
-      before giving up.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.cache.queue.size.in.kb</name>
-    <value>256</value>
-    <tag>CBLOCK, PERFORMANCE</tag>
-    <description>
-      Size of the in memory cache queue, that is flushed to local
-      disk.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.cache.thread.priority</name>
-    <value>5</value>
-    <tag>CBLOCK, PERFORMANCE</tag>
-    <description>
-      Priority of cache flusher thread, affecting the relative performance of
-      write and read. Supported values are 1, 5, 10.
-      Use 10 for high priority and 1 for low priority.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.container.size.gb</name>
-    <value>5</value>
-    <tag>CBLOCK, MANAGEMENT</tag>
-    <description>
-      The size of ozone container in the number of GBs. Note that
-      this is not setting container size for ozone. This setting is
-      instructing CBlock to manage containers at a standard size.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.disk.cache.path</name>
-    <value>${hadoop.tmp.dir}/cblockCacheDB</value>
-    <tag>CBLOCK, REQUIRED</tag>
-    <description>
-      The default path for the cblock local cache. If the cblock
-      local cache is enabled, then it must be set to a valid path. This cache
-      *should* be mapped to the fastest disk on a given machine, For example,
-      an SSD drive would be a good idea. Currently, all mounted disk on a
-      data node is mapped to a single path, so having a large number of IOPS
-      is essential.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.jscsi-address</name>
-    <value/>
-    <tag>CBLOCK, MANAGEMENT</tag>
-    <description>
-      The address that cblock will be bind to, should be a host:port
-      format, This setting is required for cblock server to start.
-      This address to be used by jscsi to mount volume.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.jscsi.cblock.server.address</name>
-    <value>127.0.0.1</value>
-    <tag>CBLOCK, MANAGEMENT</tag>
-    <description>
-      The address local jscsi server will use to talk to cblock manager.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.jscsi.port</name>
-    <value>9811</value>
-    <tag>CBLOCK, MANAGEMENT</tag>
-    <description>
-      The port on CBlockManager node for jSCSI to talk to.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.jscsi.rpc-bind-host</name>
-    <value>0.0.0.0</value>
-    <tag>CBLOCK, MANAGEMENT</tag>
-    <description>
-      The actual address the cblock jscsi rpc server will bind to. If
-      this optional address is set, it overrides only the hostname portion of
-      dfs.cblock.jscsi-address.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.jscsi.server.address</name>
-    <value>0.0.0.0</value>
-    <tag>CBLOCK, MANAGEMENT</tag>
-    <description>
-      The address that jscsi server will be running, it is nice have one
-      local jscsi server for each client(Linux JSCSI client) that tries to
-      mount cblock.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.manager.pool.size</name>
-    <value>16</value>
-    <tag>CBLOCK, PERFORMANCE</tag>
-    <description>
-      Number of active threads that cblock manager will use for container
-      operations. The maximum number of the threads are limited to the
-      processor count * 2.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.rpc.timeout</name>
-    <value>300s</value>
-    <tag>CBLOCK, MANAGEMENT</tag>
-    <description>
-      RPC timeout used for cblock CLI operations. When you
-      create very large disks, like 5TB, etc. The number of containers
-      allocated in the system is huge. It is will 5TB/5GB, which is 1000
-      containers. The client CLI might timeout even though the cblock manager
-      creates the specified disk. This value allows the user to wait for a
-      longer period.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.scm.ipaddress</name>
-    <value>127.0.0.1</value>
-    <tag>CBLOCK, MANAGEMENT</tag>
-    <description>
-      IP address used by cblock to connect to SCM.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.scm.port</name>
-    <value>9860</value>
-    <tag>CBLOCK, MANAGEMENT</tag>
-    <description>
-      Port used by cblock to connect to SCM.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.service.handler.count</name>
-    <value>10</value>
-    <tag>CBLOCK, MANAGEMENT</tag>
-    <description>
-      Default number of handlers for CBlock service rpc.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.service.leveldb.path</name>
-    <value>${hadoop.tmp.dir}/cblock_server.dat</value>
-    <tag>CBLOCK, REQUIRED</tag>
-    <description>
-      Default path for the cblock meta data store.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.service.rpc-bind-host</name>
-    <value>0.0.0.0</value>
-    <tag>CBLOCK, MANAGEMENT</tag>
-    <description>
-      The actual address the cblock service RPC server will bind to.
-      If the optional address is set, it overrides only the hostname portion of
-      dfs.cblock.servicerpc-address.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.servicerpc-address</name>
-    <value/>
-    <tag>CBLOCK, MANAGEMENT, REQUIRED</tag>
-    <description>
-      The address that cblock will be bind to, should be a host:port
-      format, this setting is required for cblock server to start.
-      This address is used for cblock management operations like create, delete,
-      info and list volumes
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.short.circuit.io</name>
-    <value>false</value>
-    <tag>CBLOCK, PERFORMANCE</tag>
-    <description>
-      Enables use of the local cache in cblock. Enabling this allows
-      I/O against the local cache and background threads do actual I/O against
-      the
-      containers.
-    </description>
-  </property>
-  <property>
-    <name>dfs.cblock.trace.io</name>
-    <value>false</value>
-    <tag>CBLOCK, DEBUG</tag>
-    <description>Default flag for enabling trace io, Trace I/O logs all I/O with
-      hashes of
-      data. This is useful for detecting things like data corruption.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.cblock.iscsi.advertised.ip</name>
-    <value>0.0.0.0</value>
-    <tag>CBLOCK</tag>
-    <description>
-      IP address returned during the iscsi discovery.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.cblock.iscsi.advertised.port</name>
-    <value>3260</value>
-    <tag>CBLOCK</tag>
-    <description>
-      TCP port returned during the iscsi discovery.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.cblock.kubernetes.dynamic-provisioner.enabled</name>
-    <value>false</value>
-    <tag>CBLOCK, KUBERNETES</tag>
-    <description>Flag to enable automatic creation of cblocks and
-      kubernetes PersitentVolumes in kubernetes environment.</description>
-  </property>
-
-  <property>
-    <name>dfs.cblock.kubernetes.cblock-user</name>
-    <value>iqn.2001-04.org.apache.hadoop</value>
-    <tag>CBLOCK, KUBERNETES</tag>
-    <description>CBlock user to use for the dynamic provisioner.
-      This user will own all of the auto-created cblocks.</description>
-  </property>
-
-  <property>
-    <name>dfs.cblock.kubernetes.configfile</name>
-    <value></value>
-    <tag>CBLOCK, KUBERNETES</tag>
-    <description>Location of the kubernetes configuration file
-      to access the kubernetes cluster. Not required inside a pod
-      as the default service account will be if this value is
-      empty.</description>
-  </property>
-
-  <property>
-    <name>dfs.cblock.iscsi.advertised.ip</name>
-    <value></value>
-    <tag>CBLOCK, KUBERNETES</tag>
-    <description>IP where the cblock target server is available
-      from the kubernetes nodes. Usually it's a cluster ip address
-    which is defined by a deployed Service.</description>
-  </property>
-
-  <property>
-    <name>dfs.cblock.iscsi.advertised.port</name>
-    <value>3260</value>
-    <tag>CBLOCK, KUBERNETES</tag>
-    <description>Port where the cblock target server is available
-      from the kubernetes nodes. Could be different from the
-      listening port if jscsi is behind a Service.</description>
-  </property>
-
-  <!--Container Settings used by Datanode-->
-  <property>
-    <name>ozone.container.cache.size</name>
-    <value>1024</value>
-    <tag>PERFORMANCE, CONTAINER, STORAGE</tag>
-    <description>The open container is cached on the data node side. We maintain
-      an LRU
-      cache for caching the recently used containers. This setting controls the
-      size of that cache.
-    </description>
-  </property>
-  <property>
-    <name>dfs.container.ipc</name>
-    <value>9859</value>
-    <tag>OZONE, CONTAINER, MANAGEMENT</tag>
-    <description>The ipc port number of container.</description>
-  </property>
-  <property>
-    <name>dfs.container.ipc.random.port</name>
-    <value>false</value>
-    <tag>OZONE, DEBUG, CONTAINER</tag>
-    <description>Allocates a random free port for ozone container. This is used
-      only while
-      running unit tests.
-    </description>
-  </property>
-  <property>
-    <name>dfs.container.ratis.datanode.storage.dir</name>
-    <value/>
-    <tag>OZONE, CONTAINER, STORAGE, MANAGEMENT, RATIS</tag>
-    <description>This directory is used for storing Ratis metadata like logs. If
-      this is
-      not set then default metadata dirs is used. A warning will be logged if
-      this not set. Ideally, this should be mapped to a fast disk like an SSD.
-    </description>
-  </property>
-  <property>
-    <name>dfs.container.ratis.enabled</name>
-    <value>false</value>
-    <tag>OZONE, MANAGEMENT, PIPELINE, RATIS</tag>
-    <description>Ozone supports different kinds of replication pipelines. Ratis
-      is one of
-      the replication pipeline supported by ozone.
-    </description>
-  </property>
-  <property>
-    <name>dfs.container.ratis.ipc</name>
-    <value>9858</value>
-    <tag>OZONE, CONTAINER, PIPELINE, RATIS, MANAGEMENT</tag>
-    <description>The ipc port number of container.</description>
-  </property>
-  <property>
-    <name>dfs.container.ratis.ipc.random.port</name>
-    <value>false</value>
-    <tag>OZONE,DEBUG</tag>
-    <description>Allocates a random free port for ozone ratis port for the
-      container. This
-      is used only while running unit tests.
-    </description>
-  </property>
-  <property>
-    <name>dfs.container.ratis.rpc.type</name>
-    <value>GRPC</value>
-    <tag>OZONE, RATIS, MANAGEMENT</tag>
-    <description>Ratis supports different kinds of transports like netty, GRPC,
-      Hadoop RPC
-      etc. This picks one of those for this cluster.
-    </description>
-  </property>
-  <property>
-    <name>dfs.container.ratis.num.write.chunk.threads</name>
-    <value>60</value>
-    <tag>OZONE, RATIS, PERFORMANCE</tag>
-    <description>Maximum number of threads in the thread pool that Ratis
-      will use for writing chunks (60 by default).
-    </description>
-  </property>
-  <property>
-    <name>dfs.container.ratis.segment.size</name>
-    <value>1073741824</value>
-    <tag>OZONE, RATIS, PERFORMANCE</tag>
-    <description>The size of the raft segment used by Apache Ratis on datanodes.
-      (1 GB by default)
-    </description>
-  </property>
-  <property>
-    <name>dfs.container.ratis.segment.preallocated.size</name>
-    <value>134217728</value>
-    <tag>OZONE, RATIS, PERFORMANCE</tag>
-    <description>The size of the buffer which is preallocated for raft segment
-      used by Apache Ratis on datanodes.(128 MB by default)
-    </description>
-  </property>
-  <property>
-    <name>ozone.container.report.interval</name>
-    <value>60000ms</value>
-    <tag>OZONE, CONTAINER, MANAGEMENT</tag>
-    <description>Time interval of the datanode to send container report. Each
-      datanode periodically send container report upon receive
-      sendContainerReport from SCM. Unit could be defined with
-      postfix (ns,ms,s,m,h,d)</description>
-  </property>
-  <!--Ozone Settings-->
-  <property>
-    <name>ozone.administrators</name>
-    <value/>
-    <tag>OZONE, SECURITY</tag>
-    <description>Ozone administrator users delimited by the comma.
-      If not set, only the user who launches an ozone service will be the admin
-      user. This property must be set if ozone services are started by different
-      users. Otherwise, the RPC layer will reject calls from other servers which
-      are started by users not in the list.
-    </description>
-  </property>
-  <property>
-    <name>ozone.block.deleting.container.limit.per.interval</name>
-    <value>10</value>
-    <tag>OZONE, PERFORMANCE, SCM</tag>
-    <description>A maximum number of containers to be scanned by block deleting
-      service per
-      time interval. The block deleting service spawns a thread to handle block
-      deletions in a container. This property is used to throttle the number of
-      threads spawned for block deletions.
-    </description>
-  </property>
-  <property>
-    <name>ozone.block.deleting.limit.per.task</name>
-    <value>1000</value>
-    <tag>OZONE, PERFORMANCE, SCM</tag>
-    <description>A maximum number of blocks to be deleted by block deleting
-      service per
-      time interval. This property is used to throttle the actual number of
-      block deletions on a data node per container.
-    </description>
-  </property>
-  <property>
-    <name>ozone.block.deleting.service.interval</name>
-    <value>1m</value>
-    <tag>OZONE, PERFORMANCE, SCM</tag>
-    <description>Time interval of the block deleting service.
-      The block deleting service runs on each datanode periodically and
-      deletes blocks queued for deletion. Unit could be defined with
-      postfix (ns,ms,s,m,h,d)
-    </description>
-  </property>
-  <property>
-    <name>ozone.block.deleting.service.timeout</name>
-    <value>300000ms</value>
-    <tag>OZONE, PERFORMANCE, SCM</tag>
-    <description>A timeout value of block deletion service. If this is set
-      greater than 0,
-      the service will stop waiting for the block deleting completion after this
-      time. If timeout happens to a large proportion of block deletion, this
-      needs to be increased with ozone.block.deleting.limit.per.task. This
-      setting supports multiple time unit suffixes as described in
-      dfs.heartbeat.interval. If no suffix is specified, then milliseconds is
-      assumed.
-    </description>
-  </property>
-  <property>
-    <name>ozone.client.connection.timeout</name>
-    <value>5000ms</value>
-    <tag>OZONE, PERFORMANCE, CLIENT</tag>
-    <description>Connection timeout for Ozone client in milliseconds.
-    </description>
-  </property>
-  <property>
-    <name>ozone.client.protocol</name>
-    <value>org.apache.hadoop.ozone.client.rpc.RpcClient</value>
-    <tag>OZONE, CLIENT, MANAGEMENT</tag>
-    <description>Protocol class to be used by the client to connect to ozone
-      cluster.
-      The build-in implementation includes:
-      org.apache.hadoop.ozone.client.rpc.RpcClient for RPC
-      org.apache.hadoop.ozone.client.rest.RestClient for REST
-      The default is the RpClient. Please do not change this unless you have a
-      very good understanding of what you are doing.
-    </description>
-  </property>
-  <property>
-    <name>ozone.client.socket.timeout</name>
-    <value>5000ms</value>
-    <tag>OZONE, CLIENT</tag>
-    <description>Socket timeout for Ozone client. Unit could be defined with
-      postfix (ns,ms,s,m,h,d)</description>
-  </property>
-  <property>
-    <name>ozone.enabled</name>
-    <value>false</value>
-    <tag>OZONE, REQUIRED</tag>
-    <description>
-      Status of the Ozone Object Storage service is enabled.
-      Set to true to enable Ozone.
-      Set to false to disable Ozone.
-      Unless this value is set to true, Ozone services will not be started in
-      the cluster.
-
-      Please note: By default ozone is disabled on a hadoop cluster.
-    </description>
-  </property>
-  <property>
-    <name>ozone.handler.type</name>
-    <value>distributed</value>
-    <tag>OZONE, REST</tag>
-    <description>
-      Tells ozone which storage handler to use. The possible values are:
-      distributed - The Ozone distributed storage handler, which speaks to
-      KSM/SCM on the backend and provides REST services to clients.
-      local - Local Storage handler strictly for testing - To be removed.
-    </description>
-  </property>
-  <property>
-    <name>ozone.key.deleting.limit.per.task</name>
-    <value>1000</value>
-    <tag>KSM, PERFORMANCE</tag>
-    <description>
-      A maximum number of keys to be scanned by key deleting service
-      per time interval in KSM. Those keys are sent to delete metadata and
-      generate transactions in SCM for next async deletion between SCM
-      and DataNode.
-    </description>
-  </property>
-  <property>
-    <name>ozone.ksm.address</name>
-    <value/>
-    <tag>KSM, REQUIRED</tag>
-    <description>
-      The address of the Ozone KSM service. This allows clients to discover
-      the KSMs address.
-    </description>
-  </property>
-  <property>
-    <name>ozone.ksm.group.rights</name>
-    <value>READ_WRITE</value>
-    <tag>KSM, SECURITY</tag>
-    <description>
-      Default group permissions in Ozone KSM.
-    </description>
-  </property>
-  <property>
-    <name>ozone.ksm.handler.count.key</name>
-    <value>20</value>
-    <tag>KSM, PERFORMANCE</tag>
-    <description>
-      The number of RPC handler threads for KSM service endpoints.
-    </description>
-  </property>
-  <property>
-    <name>ozone.ksm.http-address</name>
-    <value>0.0.0.0:9874</value>
-    <tag>KSM, MANAGEMENT</tag>
-    <description>
-      The address and the base port where the KSM web UI will listen on.
-
-      If the port is 0, then the server will start on a free port. However, it
-      is best to specify a well-known port, so it is easy to connect and see
-      the KSM management UI.
-    </description>
-  </property>
-  <property>
-    <name>ozone.ksm.http-bind-host</name>
-    <value>0.0.0.0</value>
-    <tag>KSM, MANAGEMENT</tag>
-    <description>
-      The actual address the KSM web server will bind to. If this optional
-      the address is set, it overrides only the hostname portion of
-      ozone.ksm.http-address.
-    </description>
-  </property>
-  <property>
-    <name>ozone.ksm.http.enabled</name>
-    <value>true</value>
-    <tag>KSM, MANAGEMENT</tag>
-    <description>
-      Property to enable or disable KSM web user interface.
-    </description>
-  </property>
-  <property>
-    <name>ozone.ksm.https-address</name>
-    <value>0.0.0.0:9875</value>
-    <tag>KSM, MANAGEMENT, SECURITY</tag>
-    <description>
-      The address and the base port where the KSM web UI will listen
-      on using HTTPS.
-      If the port is 0 then the server will start on a free port.
-    </description>
-  </property>
-  <property>
-    <name>ozone.ksm.https-bind-host</name>
-    <value>0.0.0.0</value>
-    <tag>KSM, MANAGEMENT, SECURITY</tag>
-    <description>
-      The actual address the KSM web server will bind to using HTTPS.
-      If this optional address is set, it overrides only the hostname portion of
-      ozone.ksm.http-address.
-    </description>
-  </property>
-  <property>
-    <name>ozone.ksm.keytab.file</name>
-    <value/>
-    <tag>KSM, SECURITY</tag>
-    <description>
-      The keytab file for Kerberos authentication in KSM.
-    </description>
-  </property>
-  <property>
-    <name>ozone.ksm.db.cache.size.mb</name>
-    <value>128</value>
-    <tag>KSM, PERFORMANCE</tag>
-    <description>
-      The size of KSM DB cache in MB that used for caching files.
-      This value is set to an abnormally low value in the default configuration.
-      That is to make unit testing easy. Generally, this value should be set to
-      something like 16GB or more, if you intend to use Ozone at scale.
-
-      A large value for this key allows a proportionally larger amount of KSM
-      metadata to be cached in memory. This makes KSM operations faster.
-    </description>
-  </property>
-  <property>
-    <name>ozone.ksm.user.max.volume</name>
-    <value>1024</value>
-    <tag>KSM, MANAGEMENT</tag>
-    <description>
-      The maximum number of volumes a user can have on a cluster.Increasing or
-      decreasing this number has no real impact on ozone cluster. This is
-      defined only for operational purposes. Only an administrator can create a
-      volume, once a volume is created there are no restrictions on the number
-      of buckets or keys inside each bucket a user can create.
-    </description>
-  </property>
-  <property>
-    <name>ozone.ksm.user.rights</name>
-    <value>READ_WRITE</value>
-    <tag>KSM, SECURITY</tag>
-    <description>
-      Default user permissions used in KSM.
-    </description>
-  </property>
-  <property>
-    <name>ozone.localstorage.root</name>
-    <value>${hadoop.tmp.dir}/ozone</value>
-    <tag>OZONE, DEBUG</tag>
-    <description>
-      This is used only for testing purposes. This value is used by the local
-      storage handler to simulate a REST backend. This is useful only when
-      debugging the REST front end independent of KSM and SCM. To be removed.
-    </description>
-  </property>
-  <property>
-    <name>ozone.metadata.dirs</name>
-    <value/>
-    <tag>OZONE, KSM, SCM, CONTAINER, REQUIRED, STORAGE</tag>
-    <description>
-      Ozone metadata is shared among KSM, which acts as the namespace
-      manager for ozone, SCM which acts as the block manager and data nodes
-      which maintain the name of the key(Key Name and BlockIDs). This
-      replicated and distributed metadata store is maintained under the
-      directory pointed by this key. Since metadata can be I/O intensive, at
-      least on KSM and SCM we recommend having SSDs. If you have the luxury
-      of mapping this path to SSDs on all machines in the cluster, that will
-      be excellent.
-
-      If Ratis metadata directories are not specified, Ratis server will emit a
-      warning and use this path for storing its metadata too.
-    </description>
-  </property>
-  <property>
-    <name>ozone.metastore.impl</name>
-    <value>RocksDB</value>
-    <tag>OZONE, KSM, SCM, CONTAINER, STORAGE</tag>
-    <description>
-      Ozone metadata store implementation. Ozone metadata are well
-      distributed to multiple services such as ksm, scm. They are stored in
-      some local key-value databases. This property determines which database
-      library to use. Supported value is either LevelDB or RocksDB.
-    </description>
-  </property>
-
-  <property>
-    <name>ozone.metastore.rocksdb.statistics</name>
-    <value>ALL</value>
-    <tag>OZONE, KSM, SCM, STORAGE, PERFORMANCE</tag>
-    <description>
-      The statistics level of the rocksdb store. If you use any value from
-      org.rocksdb.StatsLevel (eg. ALL or EXCEPT_DETAILED_TIMERS), the rocksdb
-      statistics will be exposed over JMX bean with the choosed setting. Set
-      it to OFF to not initialize rocksdb statistics at all. Please note that
-      collection of statistics could have 5-10% performance penalty.
-      Check the rocksdb documentation for more details.
-    </description>
-  </property>
-
-  <property>
-    <name>ozone.scm.block.client.address</name>
-    <value/>
-    <tag>OZONE, SCM</tag>
-    <description>The address of the Ozone SCM block client service. If not
-      defined value of ozone.scm.client.address is used.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.block.client.bind.host</name>
-    <value>0.0.0.0</value>
-    <tag>OZONE, SCM</tag>
-    <description>
-      The hostname or IP address used by the SCM block client
-      endpoint to bind.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.block.client.port</name>
-    <value>9863</value>
-    <tag>OZONE, SCM</tag>
-    <description>
-      The port number of the Ozone SCM block client service.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.block.deletion.max.retry</name>
-    <value>4096</value>
-    <tag>OZONE, SCM</tag>
-    <description>
-      SCM wraps up many blocks in a deletion transaction and sends that to data
-      node for physical deletion periodically. This property determines how many
-      times SCM is going to retry sending a deletion operation to the data node.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.block.size.in.mb</name>
-    <value>256</value>
-    <tag>OZONE, SCM</tag>
-    <description>
-      The default size of a scm block in bytes. This is maps to the default
-      Ozone block size.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.chunk.size</name>
-    <value>16777216</value>
-    <tag>OZONE, SCM, CONTAINER, PERFORMANCE</tag>
-    <description>
-      The chunk size for reading/writing chunk operations in bytes.
-
-      The chunk size defaults to 8MB. If the value configured is more than the
-      maximum size (16MB), it will be reset to the maximum size. This maps to
-      the network packet sizes and file write operations in the client to
-      datanode protocol.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.client.address</name>
-    <value/>
-    <tag>OZONE, SCM, REQUIRED</tag>
-    <description>
-      The address of the Ozone SCM client service. This is a required setting.
-
-      It is a string in the host:port format. The port number is optional
-      and defaults to 9860.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.client.bind.host</name>
-    <value>0.0.0.0</value>
-    <tag>OZONE, SCM, MANAGEMENT</tag>
-    <description>The hostname or IP address used by the SCM client endpoint to
-      bind.
-      This setting is used by the SCM only and never used by clients.
-
-      The setting can be useful in multi-homed setups to restrict the
-      availability of the SCM client service to a specific interface.
-
-      The default is appropriate for most clusters.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.client.port</name>
-    <value>9860</value>
-    <tag>OZONE, SCM, MANAGEMENT</tag>
-    <description>The port number of the Ozone SCM client service.</description>
-  </property>
-  <property>
-    <name>ozone.scm.container.deletion-choosing.policy</name>
-    <value>
-      org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy
-    </value>
-    <tag>OZONE, MANAGEMENT</tag>
-    <description>
-      The policy used for choosing desire containers for block deletion.
-      Datanode selects some containers to process block deletion
-      in a certain interval defined by ozone.block.deleting.service.interval.
-      The number of containers to process in each interval is defined
-      by ozone.block.deleting.container.limit.per.interval. This property is
-      used to configure the policy applied while selecting containers.
-      There are two policies supporting now:
-      RandomContainerDeletionChoosingPolicy and
-      TopNOrderedContainerDeletionChoosingPolicy.
-      org.apache.hadoop.ozone.container.common.impl.RandomContainerDeletionChoosingPolicy
-      implements a simply random policy that to return a random list of
-      containers.
-      org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy
-      implements a policy that choosing top count number of containers in a
-      pending-deletion-blocks's num
-      based descending order.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.container.placement.impl</name>
-    <value>
-      org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementRandom
-    </value>
-    <tag>OZONE, MANAGEMENT</tag>
-    <description>Placement policy class for containers.
-      Defaults to SCMContainerPlacementRandom.class
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.container.provision_batch_size</name>
-    <value>20</value>
-    <tag>OZONE, PERFORMANCE</tag>
-    <description>Pre-provision specified number of containers for block
-      allocation.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.container.report.processing.interval</name>
-    <value>60s</value>
-    <tag>OZONE, PERFORMANCE</tag>
-    <description>Time interval for scm to process container reports
-      for a node pool. Scm handles node pool reports in a cyclic clock
-      manner, it fetches pools periodically with this time interval.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.container.reports.wait.timeout</name>
-    <value>300s</value>
-    <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
-    <description>Maximum time to wait in seconds for processing all container
-      reports from
-      a node pool. It determines the timeout for a
-      node pool report.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.container.size.gb</name>
-    <value>5</value>
-    <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
-    <description>
-      Default container size used by Ozone. This value is specified
-      in GB.
-      There are two considerations while picking this number. The speed at which
-      a container can be replicated, determined by the network speed and the
-      metadata that each container generates. So selecting a large number
-      creates less SCM metadata, but recovery time will be more. 5GB is a number
-      that maps to quick replication times in gigabit networks, but still
-      balances the amount of metadata.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.datanode.address</name>
-    <value/>
-    <tag>OZONE, MANAGEMENT</tag>
-    <description>
-      The address of the Ozone SCM service used for internal
-      communication between the DataNodes and the SCM.
-
-      It is a string in the host:port format. The port number is optional
-      and defaults to 9861.
-
-      This setting is optional. If unspecified then the hostname portion
-      is picked from the ozone.scm.client.address setting and the
-      default service port of 9861 is chosen.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.datanode.bind.host</name>
-    <value/>
-    <tag>OZONE, MANAGEMENT</tag>
-    <description>
-      The hostname or IP address used by the SCM service endpoint to
-      bind.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.datanode.id</name>
-    <value/>
-    <tag>OZONE, MANAGEMENT</tag>
-    <description>The path that datanodes will use to store the datanode ID.
-      If this value is not set, then datanode ID is created under the
-      metadata directory.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.datanode.port</name>
-    <value>9861</value>
-    <tag>OZONE, MANAGEMENT</tag>
-    <description>
-      The port number of the Ozone SCM service.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.db.cache.size.mb</name>
-    <value>128</value>
-    <tag>OZONE, PERFORMANCE</tag>
-    <description>SCM keeps track of the Containers in the cluster. This DB holds
-      the container metadata. This value is set to a small value to make the
-      unit
-      testing runs smooth. In production, we recommend a value of 16GB or
-      higher. This allows SCM to avoid disk I/O's while looking up the container
-      location.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.dead.node.interval</name>
-    <value>10m</value>
-    <tag>OZONE, MANAGEMENT</tag>
-    <description>
-      The interval between heartbeats before a node is tagged as dead.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.handler.count.key</name>
-    <value>10</value>
-    <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
-    <description>
-      The number of RPC handler threads for each SCM service
-      endpoint.
-
-      The default is appropriate for small clusters (tens of nodes).
-
-      Set a value that is appropriate for the cluster size. Generally, HDFS
-      recommends RPC handler count is set to 20 * log2(Cluster Size) with an
-      upper limit of 200. However, SCM will not have the same amount of
-      traffic as Namenode, so a value much smaller than that will work well too.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.heartbeat.interval</name>
-    <value>30s</value>
-    <tag>OZONE, MANAGEMENT</tag>
-    <description>
-      The heartbeat interval from a data node to SCM. Yes,
-      it is not three but 30, since most data nodes will heart beating via Ratis
-      heartbeats. If a client is not able to talk to a data node, it will notify
-      KSM/SCM eventually. So a 30 second HB seems to work. This assumes that
-      replication strategy used is Ratis if not, this value should be set to
-      something smaller like 3 seconds.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.heartbeat.log.warn.interval.count</name>
-    <value>10</value>
-    <tag>OZONE, MANAGEMENT</tag>
-    <description>
-      Defines how frequently we will log the missing of a heartbeat to SCM.
-      For example in the default case, we will write a warning message for each
-      ten consecutive heartbeats that we miss to SCM. This helps in reducing
-      clutter in a data node log, but trade off is that logs will have less of
-      this statement.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.heartbeat.rpc-timeout</name>
-    <value>1000</value>
-    <tag>OZONE, MANAGEMENT</tag>
-    <description>
-      Timeout value for the RPC from Datanode to SCM in milliseconds.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.heartbeat.thread.interval</name>
-    <value>3s</value>
-    <tag>OZONE, MANAGEMENT</tag>
-    <description>
-      When a heartbeat from the data node arrives on SCM, It is queued for
-      processing with the time stamp of when the heartbeat arrived. There is a
-      heartbeat processing thread inside SCM that runs at a specified interval.
-      This value controls how frequently this thread is run.
-
-      There are some assumptions build into SCM such as this value should allow
-      the heartbeat processing thread to run at least three times more
-      frequently than heartbeats and at least five times more than stale node
-      detection time. If you specify a wrong value, SCM will gracefully refuse
-      to run. For more info look at the node manager tests in SCM.
-
-      In short, you don't need to change this.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.http-address</name>
-    <value>0.0.0.0:9876</value>
-    <tag>OZONE, MANAGEMENT</tag>
-    <description>
-      The address and the base port where the SCM web ui will listen on.
-
-      If the port is 0 then the server will start on a free port.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.http-bind-host</name>
-    <value>0.0.0.0</value>
-    <tag>OZONE, MANAGEMENT</tag>
-    <description>
-      The actual address the SCM web server will bind to. If this
-      optional address is set, it overrides only the hostname portion of
-      ozone.scm.http-address.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.http.enabled</name>
-    <value>true</value>
-    <tag>OZONE, MANAGEMENT</tag>
-    <description>
-      Property to enable or disable SCM web ui.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.https-address</name>
-    <value>0.0.0.0:9877</value>
-    <tag>OZONE, MANAGEMENT</tag>
-    <description>
-      The address and the base port where the SCM web UI will listen
-      on using HTTPS.
-
-      If the port is 0 then the server will start on a free port.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.https-bind-host</name>
-    <value>0.0.0.0</value>
-    <tag>OZONE, MANAGEMENT</tag>
-    <description>
-      The actual address the SCM web server will bind to using HTTPS.
-      If this optional address is set, it overrides only the hostname portion of
-      ozone.scm.http-address.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.keytab.file</name>
-    <value/>
-    <tag>OZONE, SECURITY</tag>
-    <description>
-      The keytab file for Kerberos authentication in SCM.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.max.container.report.threads</name>
-    <value>100</value>
-    <tag>OZONE, PERFORMANCE</tag>
-    <description>
-      Maximum number of threads to process container reports in scm.
-      Each container report from a data node is processed by scm in a worker
-      thread, fetched from a thread pool. This property is used to control the
-      maximum size of the thread pool.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.max.hb.count.to.process</name>
-    <value>5000</value>
-    <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
-    <description>
-      The maximum number of heartbeat to process per loop of the
-      heartbeat process thread. Please see
-      ozone.scm.heartbeat.thread.interval
-      for more info.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.max.nodepool.processing.threads</name>
-    <value>1</value>
-    <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
-    <description>
-      Number of node pools to process in parallel.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.names</name>
-    <value/>
-    <tag>OZONE</tag>
-    <description>
-      The value of this property is a set of DNS | DNS:PORT | IP
-      Address | IP:PORT. Written as a comma separated string. e.g. scm1,
-      scm2:8020, 7.7.7.7:7777.
-      This property allows datanodes to discover where SCM is, so that
-      datanodes can send heartbeat to SCM.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.stale.node.interval</name>
-    <value>90s</value>
-    <tag>OZONE, MANAGEMENT</tag>
-    <description>
-      The interval for stale node flagging. Please
-      see ozone.scm.heartbeat.thread.interval before changing this value.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.max.nodepool.processing.threads</name>
-    <value>1</value>
-    <tag>OZONE, SCM</tag>
-    <description>
-      Controls the number of node pools that can be processed in parallel by
-      Container Supervisor.
-    </description>
-  </property>
-  <property>
-    <name>ozone.trace.enabled</name>
-    <value>false</value>
-    <tag>OZONE, DEBUG</tag>
-    <description>
-      Setting this flag to true dumps the HTTP request/ response in
-      the logs. Very useful when debugging REST protocol.
-    </description>
-  </property>
-  <property>
-    <name>ozone.web.authentication.kerberos.principal</name>
-    <value/>
-    <tag>OZONE, SECURITY</tag>
-    <description>
-      The server principal used by the SCM and KSM for web UI SPNEGO
-      authentication when Kerberos security is enabled. This is typically set to
-      HTTP/_HOST@REALM.TLD The SPNEGO server principal begins with the prefix
-      HTTP/ by convention.
-
-      If the value is '*', the web server will attempt to login with
-      every principal specified in the keytab file.
-    </description>
-  </property>
-
-  <!--Client Settings-->
-  <property>
-    <name>scm.container.client.idle.threshold</name>
-    <value>10s</value>
-    <tag>OZONE, PERFORMANCE</tag>
-    <description>
-      In the standalone pipelines, the SCM clients use netty to
-      communicate with the container. It also uses connection pooling to
-      reduce client side overheads. This allows a connection to stay idle for
-      a while before the connection is closed.
-    </description>
-  </property>
-  <property>
-    <name>scm.container.client.max.size</name>
-    <value>256</value>
-    <tag>OZONE, PERFORMANCE</tag>
-    <description>
-      Controls the maximum number of connections that we cached via
-      clientconnection pooling. If the number of connection
-      exceed this count then the oldest idle connection is evicted.
-    </description>
-  </property>
-
-  <property>
-    <name>scm.container.client.max.outstanding.requests</name>
-    <value>100</value>
-    <tag>OZONE, PERFORMANCE</tag>
-    <description>
-      Controls the maximum number of outstanding async requests that can be
-      handled by the Standalone as well as Ratis client.
-    </description>
-  </property>
-
-  <property>
-    <name>ozone.scm.container.creation.lease.timeout</name>
-    <value>60s</value>
-    <tag>OZONE, SCM</tag>
-    <description>
-      Container creation timeout in milliseconds to be used by SCM. When
-      BEGIN_CREATE event happens the container is moved from ALLOCATED to
-      CREATING state, SCM will now wait for the configured amount of time
-      to get COMPLETE_CREATE event if it doesn't receive it will move the
-      container to DELETING.
-    </description>
-  </property>
-
-  <property>
-    <name>ozone.key.preallocation.maxsize</name>
-    <value>134217728</value>
-    <tag>OZONE, KSM, PERFORMANCE</tag>
-    <description>
-      When a new key write request is sent to KSM, if a size is requested, at most
-      128MB of size is allocated at request time. If client needs more space for the
-      write, separate block allocation requests will be made.
-    </description>
-  </property>
-
-  <property>
-    <name>ozone.client.list.cache</name>
-    <value>1000</value>
-    <tag>OZONE, PERFORMANCE</tag>
-    <description>
-      Configuration property to configure the cache size of client list calls.
-    </description>
-  </property>
-
-  <property>
-    <name>ozone.replication</name>
-    <value>3</value>
-    <tag>OZONE, CLIENT</tag>
-    <description>
-      Default replication value. The actual number of replications can be
-      specified when writing the key. The default is used if replication
-      is not specified. Supported values: 1 and 3.
-    </description>
-  </property>
-
-  <property>
-    <name>ozone.replication.type</name>
-    <value>RATIS</value>
-    <tag>OZONE, CLIENT</tag>
-    <description>
-      Default replication type to be used while writing key into ozone. The
-      value can be specified when writing the key, default is used when
-      nothing is specified. Supported values: RATIS, STAND_ALONE and CHAINED.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.container.close.threshold</name>
-    <value>0.9f</value>
-    <tag>OZONE, SCM</tag>
-    <description>
-      This determines the threshold to be used for closing a container.
-      When the container used percentage reaches this threshold,
-      the container will be closed. Value should be a positive, non-zero
-      percentage in float notation (X.Yf), with 1.0f meaning 100%.
-    </description>
-  </property>
-  <property>
-    <name>ozone.rest.client.http.connection.max</name>
-    <value>100</value>
-    <tag>OZONE, CLIENT</tag>
-    <description>
-      This defines the overall connection limit for the connection pool used in
-      RestClient.
-    </description>
-  </property>
-  <property>
-    <name>ozone.rest.client.http.connection.per-route.max</name>
-    <value>20</value>
-    <tag>OZONE, CLIENT</tag>
-    <description>
-      This defines the connection limit per one HTTP route/host. Total max
-      connection is limited by ozone.rest.client.http.connection.max property.
-    </description>
-  </property>
-
-  <property>
-    <name>ozone.open.key.cleanup.service.interval.seconds</name>
-    <value>86400</value>
-    <tag>OZONE, KSM, PERFORMANCE</tag>
-    <description>
-      A background job periodically checks open key entries and delete the expired ones. This entry controls the
-      interval of this cleanup check.
-    </description>
-  </property>
-
-  <property>
-    <name>ozone.open.key.expire.threshold</name>
-    <value>86400</value>
-    <tag>OZONE, KSM, PERFORMANCE</tag>
-    <description>
-      Controls how long an open key operation is considered active. Specifically, if a key
-      has been open longer than the value of this config entry, that open key is considered as
-      expired (e.g. due to client crash). Default to 24 hours.
-    </description>
-  </property>
-</configuration>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/dn.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/dn.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/dn.js
index 3b67167..9e2732c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/dn.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/dn.js
@@ -18,11 +18,11 @@
 (function () {
   "use strict";
 
-  var data = {ozone: {enabled: false}};
+  var data = {};
 
   dust.loadSource(dust.compile($('#tmpl-dn').html(), 'dn'));
 
-  function loadDatanodeInfo() {
+  function load() {
     $.get('/jmx?qry=Hadoop:service=DataNode,name=DataNodeInfo', function(resp) {
       data.dn = workaround(resp.beans[0]);
       data.dn.HostName = resp.beans[0]['DatanodeHostname'];
@@ -30,26 +30,6 @@
     }).fail(show_err_msg);
   }
 
-  function loadOzoneScmInfo() {
-        $.get('/jmx?qry=Hadoop:service=OzoneDataNode,name=SCMConnectionManager', function (resp) {
-            if (resp.beans.length > 0) {
-                data.ozone.SCMServers = resp.beans[0].SCMServers;
-                data.ozone.enabled = true;
-                render();
-            }
-        }).fail(show_err_msg);
-  }
-
-  function loadOzoneStorageInfo() {
-        $.get('/jmx?qry=Hadoop:service=OzoneDataNode,name=ContainerLocationManager', function (resp) {
-            if (resp.beans.length > 0) {
-                data.ozone.LocationReport = resp.beans[0].LocationReport;
-                data.ozone.enabled = true;
-                render();
-            }
-        }).fail(show_err_msg);
-    }
-
   function workaround(dn) {
     function node_map_to_array(nodes) {
       var res = [];
@@ -85,8 +65,6 @@
     $('#alert-panel').show();
   }
 
-    loadDatanodeInfo();
-    loadOzoneScmInfo();
-    loadOzoneStorageInfo();
+  load();
 
 })();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/index.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/index.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/index.html
deleted file mode 100644
index bee42bf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/index.html
+++ /dev/null
@@ -1,70 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
-        "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<html lang="en">
-<head>
-    <meta charset="utf-8">
-    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
-    <meta name="description" content="HDFS Key Space Manager">
-
-    <title>HDFS Key Space Manager</title>
-
-    <link href="static/bootstrap-3.0.2/css/bootstrap.min.css" rel="stylesheet">
-    <link href="static/hadoop.css" rel="stylesheet">
-    <link href="static/nvd3-1.8.5.min.css" rel="stylesheet">
-
-    <link href="static/ozone.css" rel="stylesheet">
-
-</head>
-
-<body ng-app="ksm">
-
-<header class="navbar navbar-inverse navbar-fixed-top bs-docs-nav">
-    <div class="container-fluid">
-        <div class="navbar-header">
-            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar"
-                    aria-expanded="false" aria-controls="navbar">
-                <span class="sr-only">Toggle navigation</span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-            </button>
-            <a class="navbar-brand" href="#">HDFS KSM</a>
-        </div>
-        <navmenu
-                metrics="{ 'Ksm metrics' : '#!/metrics/ksm', 'Rpc metrics' : '#!/metrics/rpc'}"></navmenu>
-    </div>
-</header>
-
-<div class="container-fluid">
-    <ng-view></ng-view>
-</div><!-- /.container -->
-
-<script src="static/jquery-1.10.2.min.js"></script>
-<script src="static/angular-1.6.4.min.js"></script>
-<script src="static/angular-route-1.6.4.min.js"></script>
-<script src="static/d3-3.5.17.min.js"></script>
-<script src="static/nvd3-1.8.5.min.js"></script>
-<script src="static/angular-nvd3-1.0.9.min.js"></script>
-<script src="static/ozone.js"></script>
-<script src="ksm.js"></script>
-<script src="static/bootstrap-3.0.2/js/bootstrap.min.js"></script>
-</body>
-</html>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/ksm-metrics.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/ksm-metrics.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/ksm-metrics.html
deleted file mode 100644
index e63fb00..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/ksm-metrics.html
+++ /dev/null
@@ -1,44 +0,0 @@
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<h1>KSM Metrics</h1>
-
-<div ng-repeat="(type,numbers) in $ctrl.metrics.nums">
-    <h2>{{type}}</h2>
-    <div class="container">
-        <div class="col-md-6">
-            <h3>Requests ({{numbers.ops}} ops)</h3>
-            <nvd3 options="$ctrl.graphOptions"
-                  data="numbers.all"></nvd3>
-        </div>
-        <div class="col-md-6">
-            <h3>Failures</h3>
-            <nvd3 options="$ctrl.graphOptions"
-                  data="numbers.failures"></nvd3>
-        </div>
-    </div>
-</div>
-
-<div ng-show="$ctrl.metrics.others.length > 0">
-    <h2>Other JMX properties</h2>
-
-    <table class="table">
-        <tr ng-repeat="metric in $ctrl.metrics.others">
-            <td>{{metric.key}}</td>
-            <td>{{metric.value}}</td>
-        </tr>
-    </table>
-</div>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/ksm.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/ksm.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/ksm.js
deleted file mode 100644
index 7fb52b1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/ksm.js
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-(function () {
-    "use strict";
-
-    var isIgnoredJmxKeys = function (key) {
-        return key == 'name' || key == 'modelerType' || key.match(/tag.*/);
-    };
-
-    angular.module('ksm', ['ozone', 'nvd3']);
-    angular.module('ksm').config(function ($routeProvider) {
-        $routeProvider
-            .when("/metrics/ksm", {
-                template: "<ksm-metrics></ksm-metrics>"
-            });
-    });
-    angular.module('ksm').component('ksmMetrics', {
-        templateUrl: 'ksm-metrics.html',
-        controller: function ($http) {
-            var ctrl = this;
-
-            ctrl.graphOptions = {
-                chart: {
-                    type: 'pieChart',
-                    height: 500,
-                    x: function (d) {
-                        return d.key;
-                    },
-                    y: function (d) {
-                        return d.value;
-                    },
-                    showLabels: true,
-                    labelType: 'value',
-                    duration: 500,
-                    labelThreshold: 0.01,
-                    labelSunbeamLayout: true,
-                    legend: {
-                        margin: {
-                            top: 5,
-                            right: 35,
-                            bottom: 5,
-                            left: 0
-                        }
-                    }
-                }
-            };
-
-
-            $http.get("jmx?qry=Hadoop:service=KeySpaceManager,name=KSMMetrics")
-                .then(function (result) {
-
-                    var groupedMetrics = {others: [], nums: {}};
-                    var metrics = result.data.beans[0]
-                    for (var key in metrics) {
-                        var numericalStatistic = key.match(/Num([A-Z][a-z]+)(.+?)(Fails)?$/);
-                        if (numericalStatistic) {
-                            var type = numericalStatistic[1];
-                            var name = numericalStatistic[2];
-                            var failed = numericalStatistic[3];
-                            groupedMetrics.nums[type] = groupedMetrics.nums[type] || {
-                                    failures: [],
-                                    all: []
-                                };
-                            if (failed) {
-                                groupedMetrics.nums[type].failures.push({
-                                    key: name,
-                                    value: metrics[key]
-                                })
-                            } else {
-                                if (name == "Ops") {
-                                    groupedMetrics.nums[type].ops = metrics[key]
-                                } else {
-                                    groupedMetrics.nums[type].all.push({
-                                        key: name,
-                                        value: metrics[key]
-                                    })
-                                }
-                            }
-                        } else if (isIgnoredJmxKeys(key)) {
-                            //ignore
-                        } else {
-                            groupedMetrics.others.push({
-                                'key': key,
-                                'value': metrics[key]
-                            });
-                        }
-                    }
-                    ctrl.metrics = groupedMetrics;
-                })
-        }
-    });
-
-})();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/main.css
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/main.css b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/main.css
deleted file mode 100644
index e442adc..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/main.css
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- *   Licensed to the Apache Software Foundation (ASF) under one or more
- *  contributor license agreements.  See the NOTICE file distributed with
- *  this work for additional information regarding copyright ownership.
- *  The ASF licenses this file to You under the Apache License, Version 2.0
- *  (the "License"); you may not use this file except in compliance with
- *  the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
-*/
-body {
-  padding-top: 50px;
-}
-.starter-template {
-  padding: 40px 15px;
-  text-align: center;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/main.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/main.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/main.html
deleted file mode 100644
index 0821899..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/main.html
+++ /dev/null
@@ -1,18 +0,0 @@
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-  -->
-<overview>
-</overview>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/index.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/index.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/index.html
deleted file mode 100644
index 3407f51..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/index.html
+++ /dev/null
@@ -1,76 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
-        "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<html lang="en">
-<head>
-    <meta charset="utf-8">
-    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
-    <meta name="description" content="HDFS Storage Container Manager">
-
-    <title>HDFS Storage Container Manager</title>
-
-    <link href="static/bootstrap-3.0.2/css/bootstrap.min.css" rel="stylesheet">
-    <link href="static/hadoop.css" rel="stylesheet">
-    <link href="static/nvd3-1.8.5.min.css" rel="stylesheet">
-
-    <link href="static/ozone.css" rel="stylesheet">
-
-</head>
-
-<body ng-app="scm">
-
-<header class="navbar navbar-inverse navbar-fixed-top bs-docs-nav">
-    <div class="container-fluid">
-        <div class="navbar-header">
-            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar"
-                    aria-expanded="false" aria-controls="navbar">
-                <span class="sr-only">Toggle navigation</span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-            </button>
-            <a class="navbar-brand" href="#">HDFS SCM</a>
-        </div>
-
-
-        <navmenu
-                metrics="{ 'Rpc metrics' : '#!/metrics/rpc'}"></navmenu>
-
-
-    </div>
-</header>
-
-<div class="container-fluid" style="margin: 12pt">
-
-    <ng-view></ng-view>
-
-</div><!-- /.container -->
-
-<script src="static/jquery-1.10.2.min.js"></script>
-<script src="static/angular-1.6.4.min.js"></script>
-<script src="static/angular-route-1.6.4.min.js"></script>
-<script src="static/d3-3.5.17.min.js"></script>
-<script src="static/nvd3-1.8.5.min.js"></script>
-<script src="static/angular-nvd3-1.0.9.min.js"></script>
-<script src="static/ozone.js"></script>
-<script src="scm.js"></script>
-<script src="static/bootstrap-3.0.2/js/bootstrap.min.js"></script>
-</body>
-</html>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/main.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/main.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/main.html
deleted file mode 100644
index 2666f81..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/main.html
+++ /dev/null
@@ -1,20 +0,0 @@
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-  -->
-<overview>
-    <scm-overview>
-    </scm-overview>
-</overview>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/scm-overview.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/scm-overview.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/scm-overview.html
deleted file mode 100644
index fca23ba..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/scm-overview.html
+++ /dev/null
@@ -1,60 +0,0 @@
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<h2>Node counts</h2>
-
-<table class="table table-bordered table-striped" class="col-md-6">
-    <tbody>
-    <tr ng-repeat="typestat in $ctrl.nodemanagermetrics.NodeCount | orderBy:'key':false:$ctrl.nodeOrder">
-        <td>{{typestat.key}}</td>
-        <td>{{typestat.value}}</td>
-    </tr>
-    </tbody>
-</table>
-
-<h2>Status</h2>
-<table class="table table-bordered table-striped" class="col-md-6">
-    <tbody>
-    <tr>
-        <td>Client Rpc port</td>
-        <td>{{$ctrl.overview.jmx.ClientRpcPort}}</td>
-    </tr>
-    <tr>
-        <td>Datanode Rpc port</td>
-        <td>{{$ctrl.overview.jmx.DatanodeRpcPort}}</td>
-    </tr>
-    <tr>
-        <td>Block Manager: Open containers</td>
-        <td>{{$ctrl.blockmanagermetrics.OpenContainersNo}}</td>
-    </tr>
-    <tr>
-        <td>Node Manager: Minimum chill mode nodes</td>
-        <td>{{$ctrl.nodemanagermetrics.MinimumChillModeNodes}}</td>
-    </tr>
-    <tr>
-        <td>Node Manager: Out-of-node chill mode</td>
-        <td>{{$ctrl.nodemanagermetrics.OutOfNodeChillMode}}</td>
-    </tr>
-    <tr>
-        <td>Node Manager: Chill mode status</td>
-        <td>{{$ctrl.nodemanagermetrics.ChillModeStatus}}</td>
-    </tr>
-    <tr>
-        <td>Node Manager: Manual chill mode</td>
-        <td>{{$ctrl.nodemanagermetrics.InManualChillMode}}</td>
-    </tr>
-    </tbody>
-</table>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/scm.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/scm.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/scm.js
deleted file mode 100644
index bcfa8b7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/scm.js
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-(function () {
-    "use strict";
-    angular.module('scm', ['ozone', 'nvd3']);
-
-    angular.module('scm').component('scmOverview', {
-        templateUrl: 'scm-overview.html',
-        require: {
-            overview: "^overview"
-        },
-        controller: function ($http) {
-            var ctrl = this;
-            $http.get("jmx?qry=Hadoop:service=BlockManager,name=*")
-                .then(function (result) {
-                    ctrl.blockmanagermetrics = result.data.beans[0];
-                });
-            $http.get("jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo")
-                .then(function (result) {
-                    ctrl.nodemanagermetrics = result.data.beans[0];
-                });
-
-            var statusSortOrder = {
-                "HEALTHY": "a",
-                "STALE": "b",
-                "DEAD": "c",
-                "UNKNOWN": "z",
-                "DECOMMISSIONING": "x",
-                "DECOMMISSIONED": "y"
-            };
-            ctrl.nodeOrder = function (v1, v2) {
-                //status with non defined sort order will be "undefined"
-                return ("" + statusSortOrder[v1.value]).localeCompare("" + statusSortOrder[v2.value])
-            }
-
-        }
-    });
-
-})();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[19/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/InProgressPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/InProgressPool.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/InProgressPool.java
deleted file mode 100644
index 833d1a8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/InProgressPool.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container.replication;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.commands.SendContainerCommand;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerInfo;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.scm.node.NodePoolManager;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-
-import static com.google.common.util.concurrent.Uninterruptibles
-    .sleepUninterruptibly;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos
-    .NodeState.HEALTHY;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos
-    .NodeState.STALE;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos
-    .NodeState.UNKNOWN;
-
-/**
- * These are pools that are actively checking for replication status of the
- * containers.
- */
-public final class InProgressPool {
-  public static final Logger LOG =
-      LoggerFactory.getLogger(InProgressPool.class);
-
-  private final PeriodicPool pool;
-  private final NodeManager nodeManager;
-  private final NodePoolManager poolManager;
-  private final ExecutorService executorService;
-  private final Map<String, Integer> containerCountMap;
-  private final Map<String, Boolean> processedNodeSet;
-  private final long startTime;
-  private ProgressStatus status;
-  private AtomicInteger nodeCount;
-  private AtomicInteger nodeProcessed;
-  private AtomicInteger containerProcessedCount;
-  private long maxWaitTime;
-  /**
-   * Constructs an pool that is being processed.
-   *  @param maxWaitTime - Maximum wait time in milliseconds.
-   * @param pool - Pool that we are working against
-   * @param nodeManager - Nodemanager
-   * @param poolManager - pool manager
-   * @param executorService - Shared Executor service.
-   */
-  InProgressPool(long maxWaitTime, PeriodicPool pool,
-      NodeManager nodeManager, NodePoolManager poolManager,
-                 ExecutorService executorService) {
-    Preconditions.checkNotNull(pool);
-    Preconditions.checkNotNull(nodeManager);
-    Preconditions.checkNotNull(poolManager);
-    Preconditions.checkNotNull(executorService);
-    Preconditions.checkArgument(maxWaitTime > 0);
-    this.pool = pool;
-    this.nodeManager = nodeManager;
-    this.poolManager = poolManager;
-    this.executorService = executorService;
-    this.containerCountMap = new ConcurrentHashMap<>();
-    this.processedNodeSet = new ConcurrentHashMap<>();
-    this.maxWaitTime = maxWaitTime;
-    startTime = Time.monotonicNow();
-  }
-
-  /**
-   * Returns periodic pool.
-   *
-   * @return PeriodicPool
-   */
-  public PeriodicPool getPool() {
-    return pool;
-  }
-
-  /**
-   * We are done if we have got reports from all nodes or we have
-   * done waiting for the specified time.
-   *
-   * @return true if we are done, false otherwise.
-   */
-  public boolean isDone() {
-    return (nodeCount.get() == nodeProcessed.get()) ||
-        (this.startTime + this.maxWaitTime) > Time.monotonicNow();
-  }
-
-  /**
-   * Gets the number of containers processed.
-   *
-   * @return int
-   */
-  public int getContainerProcessedCount() {
-    return containerProcessedCount.get();
-  }
-
-  /**
-   * Returns the start time in milliseconds.
-   *
-   * @return - Start Time.
-   */
-  public long getStartTime() {
-    return startTime;
-  }
-
-  /**
-   * Get the number of nodes in this pool.
-   *
-   * @return - node count
-   */
-  public int getNodeCount() {
-    return nodeCount.get();
-  }
-
-  /**
-   * Get the number of nodes that we have already processed container reports
-   * from.
-   *
-   * @return - Processed count.
-   */
-  public int getNodeProcessed() {
-    return nodeProcessed.get();
-  }
-
-  /**
-   * Returns the current status.
-   *
-   * @return Status
-   */
-  public ProgressStatus getStatus() {
-    return status;
-  }
-
-  /**
-   * Starts the reconciliation process for all the nodes in the pool.
-   */
-  public void startReconciliation() {
-    List<DatanodeID> datanodeIDList =
-        this.poolManager.getNodes(pool.getPoolName());
-    if (datanodeIDList.size() == 0) {
-      LOG.error("Datanode list for {} is Empty. Pool with no nodes ? ",
-          pool.getPoolName());
-      this.status = ProgressStatus.Error;
-      return;
-    }
-
-    nodeProcessed = new AtomicInteger(0);
-    containerProcessedCount = new AtomicInteger(0);
-    nodeCount = new AtomicInteger(0);
-    /*
-       Ask each datanode to send us commands.
-     */
-    SendContainerCommand cmd = SendContainerCommand.newBuilder().build();
-    for (DatanodeID id : datanodeIDList) {
-      NodeState currentState = getNodestate(id);
-      if (currentState == HEALTHY || currentState == STALE) {
-        nodeCount.incrementAndGet();
-        // Queue commands to all datanodes in this pool to send us container
-        // report. Since we ignore dead nodes, it is possible that we would have
-        // over replicated the container if the node comes back.
-        nodeManager.addDatanodeCommand(id, cmd);
-      }
-    }
-    this.status = ProgressStatus.InProgress;
-    this.getPool().setLastProcessedTime(Time.monotonicNow());
-  }
-
-  /**
-   * Gets the node state.
-   *
-   * @param id - datanode ID.
-   * @return NodeState.
-   */
-  private NodeState getNodestate(DatanodeID id) {
-    NodeState  currentState = UNKNOWN;
-    int maxTry = 100;
-    // We need to loop to make sure that we will retry if we get
-    // node state unknown. This can lead to infinite loop if we send
-    // in unknown node ID. So max try count is used to prevent it.
-
-    int currentTry = 0;
-    while (currentState == UNKNOWN && currentTry < maxTry) {
-      // Retry to make sure that we deal with the case of node state not
-      // known.
-      currentState = nodeManager.getNodeState(id);
-      currentTry++;
-      if (currentState == UNKNOWN) {
-        // Sleep to make sure that this is not a tight loop.
-        sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
-      }
-    }
-    if (currentState == UNKNOWN) {
-      LOG.error("Not able to determine the state of Node: {}, Exceeded max " +
-          "try and node manager returns UNKNOWN state. This indicates we " +
-          "are dealing with a node that we don't know about.", id);
-    }
-    return currentState;
-  }
-
-  /**
-   * Queues a container Report for handling. This is done in a worker thread
-   * since decoding a container report might be compute intensive . We don't
-   * want to block since we have asked for bunch of container reports
-   * from a set of datanodes.
-   *
-   * @param containerReport - ContainerReport
-   */
-  public void handleContainerReport(
-      ContainerReportsRequestProto containerReport) {
-    if (status == ProgressStatus.InProgress) {
-      executorService.submit(processContainerReport(containerReport));
-    } else {
-      LOG.debug("Cannot handle container report when the pool is in {} status.",
-          status);
-    }
-  }
-
-  private Runnable processContainerReport(
-      ContainerReportsRequestProto reports) {
-    return () -> {
-      DatanodeID datanodeID =
-          DatanodeID.getFromProtoBuf(reports.getDatanodeID());
-      if (processedNodeSet.computeIfAbsent(datanodeID.getDatanodeUuid(),
-          (k) -> true)) {
-        nodeProcessed.incrementAndGet();
-        LOG.debug("Total Nodes processed : {} Node Name: {} ", nodeProcessed,
-            datanodeID.getDatanodeUuid());
-        for (ContainerInfo info : reports.getReportsList()) {
-          containerProcessedCount.incrementAndGet();
-          LOG.debug("Total Containers processed: {} Container Name: {}",
-              containerProcessedCount.get(), info.getContainerName());
-
-          // Update the container map with count + 1 if the key exists or
-          // update the map with 1. Since this is a concurrentMap the
-          // computation and update is atomic.
-          containerCountMap.merge(info.getContainerName(), 1, Integer::sum);
-        }
-      }
-    };
-  }
-
-  /**
-   * Filter the containers based on specific rules.
-   *
-   * @param predicate -- Predicate to filter by
-   * @return A list of map entries.
-   */
-  public List<Map.Entry<String, Integer>> filterContainer(
-      Predicate<Map.Entry<String, Integer>> predicate) {
-    return containerCountMap.entrySet().stream()
-        .filter(predicate).collect(Collectors.toList());
-  }
-
-  /**
-   * Used only for testing, calling this will abort container report
-   * processing. This is very dangerous call and should not be made by any users
-   */
-  @VisibleForTesting
-  public void setDoneProcessing() {
-    nodeProcessed.set(nodeCount.get());
-  }
-
-  /**
-   * Returns the pool name.
-   *
-   * @return Name of the pool.
-   */
-  String getPoolName() {
-    return pool.getPoolName();
-  }
-
-  public void finalizeReconciliation() {
-    status = ProgressStatus.Done;
-    //TODO: Add finalizing logic. This is where actual reconciliation happens.
-  }
-
-  /**
-   * Current status of the computing replication status.
-   */
-  public enum ProgressStatus {
-    InProgress, Done, Error
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/PeriodicPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/PeriodicPool.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/PeriodicPool.java
deleted file mode 100644
index 35b1e76..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/PeriodicPool.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container.replication;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Periodic pool is a pool with a time stamp, this allows us to process pools
- * based on a cyclic clock.
- */
-public class PeriodicPool implements Comparable<PeriodicPool> {
-  private final String poolName;
-  private long lastProcessedTime;
-  private AtomicLong totalProcessedCount;
-
-  /**
-   * Constructs a periodic pool.
-   *
-   * @param poolName - Name of the pool
-   */
-  public PeriodicPool(String poolName) {
-    this.poolName = poolName;
-    lastProcessedTime = 0;
-    totalProcessedCount = new AtomicLong(0);
-  }
-
-  /**
-   * Get pool Name.
-   * @return PoolName
-   */
-  public String getPoolName() {
-    return poolName;
-  }
-
-  /**
-   * Compares this object with the specified object for order.  Returns a
-   * negative integer, zero, or a positive integer as this object is less
-   * than, equal to, or greater than the specified object.
-   *
-   * @param o the object to be compared.
-   * @return a negative integer, zero, or a positive integer as this object is
-   * less than, equal to, or greater than the specified object.
-   * @throws NullPointerException if the specified object is null
-   * @throws ClassCastException   if the specified object's type prevents it
-   *                              from being compared to this object.
-   */
-  @Override
-  public int compareTo(PeriodicPool o) {
-    return Long.compare(this.lastProcessedTime, o.lastProcessedTime);
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-
-    PeriodicPool that = (PeriodicPool) o;
-
-    return poolName.equals(that.poolName);
-  }
-
-  @Override
-  public int hashCode() {
-    return poolName.hashCode();
-  }
-
-  /**
-   * Returns the Total Times we have processed this pool.
-   *
-   * @return processed count.
-   */
-  public long getTotalProcessedCount() {
-    return totalProcessedCount.get();
-  }
-
-  /**
-   * Gets the last time we processed this pool.
-   * @return time in milliseconds
-   */
-  public long getLastProcessedTime() {
-    return this.lastProcessedTime;
-  }
-
-
-  /**
-   * Sets the last processed time.
-   *
-   * @param lastProcessedTime - Long in milliseconds.
-   */
-
-  public void setLastProcessedTime(long lastProcessedTime) {
-    this.lastProcessedTime = lastProcessedTime;
-  }
-
-  /*
-   * Increments the total processed count.
-   */
-  public void incTotalProcessedCount() {
-    this.totalProcessedCount.incrementAndGet();
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/package-info.java
deleted file mode 100644
index 82e4202..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container.replication;
-/*
- This package contains routines that manage replication of a container. This
- relies on container reports to understand the replication level of a
- container - UnderReplicated, Replicated, OverReplicated -- and manages the
- replication level based on that.
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/exceptions/SCMException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/exceptions/SCMException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/exceptions/SCMException.java
deleted file mode 100644
index 4fb50e7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/exceptions/SCMException.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.exceptions;
-
-import java.io.IOException;
-
-/**
- * Exception thrown by SCM.
- */
-public class SCMException extends IOException {
-  private final ResultCodes result;
-
-  /**
-   * Constructs an {@code IOException} with {@code null}
-   * as its error detail message.
-   */
-  public SCMException(ResultCodes result) {
-    this.result = result;
-  }
-
-  /**
-   * Constructs an {@code IOException} with the specified detail message.
-   *
-   * @param message The detail message (which is saved for later retrieval by
-   * the
-   * {@link #getMessage()} method)
-   */
-  public SCMException(String message, ResultCodes result) {
-    super(message);
-    this.result = result;
-  }
-
-  /**
-   * Constructs an {@code IOException} with the specified detail message
-   * and cause.
-   * <p>
-   * <p> Note that the detail message associated with {@code cause} is
-   * <i>not</i> automatically incorporated into this exception's detail
-   * message.
-   *
-   * @param message The detail message (which is saved for later retrieval by
-   * the
-   * {@link #getMessage()} method)
-   * @param cause The cause (which is saved for later retrieval by the {@link
-   * #getCause()} method).  (A null value is permitted, and indicates that the
-   * cause is nonexistent or unknown.)
-   * @since 1.6
-   */
-  public SCMException(String message, Throwable cause, ResultCodes result) {
-    super(message, cause);
-    this.result = result;
-  }
-
-  /**
-   * Constructs an {@code IOException} with the specified cause and a
-   * detail message of {@code (cause==null ? null : cause.toString())}
-   * (which typically contains the class and detail message of {@code cause}).
-   * This constructor is useful for IO exceptions that are little more
-   * than wrappers for other throwables.
-   *
-   * @param cause The cause (which is saved for later retrieval by the {@link
-   * #getCause()} method).  (A null value is permitted, and indicates that the
-   * cause is nonexistent or unknown.)
-   * @since 1.6
-   */
-  public SCMException(Throwable cause, ResultCodes result) {
-    super(cause);
-    this.result = result;
-  }
-
-  /**
-   * Returns resultCode.
-   * @return ResultCode
-   */
-  public ResultCodes getResult() {
-    return result;
-  }
-
-  /**
-   * Error codes to make it easy to decode these exceptions.
-   */
-  public enum ResultCodes {
-    SUCCEESS,
-    FAILED_TO_LOAD_NODEPOOL,
-    FAILED_TO_FIND_NODE_IN_POOL,
-    FAILED_TO_FIND_HEALTHY_NODES,
-    FAILED_TO_FIND_NODES_WITH_SPACE,
-    FAILED_TO_FIND_SUITABLE_NODE,
-    INVALID_CAPACITY,
-    INVALID_BLOCK_SIZE,
-    CHILL_MODE_EXCEPTION,
-    FAILED_TO_LOAD_OPEN_CONTAINER,
-    FAILED_TO_ALLOCATE_CONTAINER,
-    FAILED_TO_CHANGE_CONTAINER_STATE,
-    CONTAINER_EXISTS,
-    FAILED_TO_FIND_CONTAINER,
-    FAILED_TO_FIND_CONTAINER_WITH_SPACE,
-    BLOCK_EXISTS,
-    FAILED_TO_FIND_BLOCK,
-    IO_EXCEPTION,
-    UNEXPECTED_CONTAINER_STATE,
-    SCM_NOT_INITIALIZED
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/exceptions/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/exceptions/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/exceptions/package-info.java
deleted file mode 100644
index 0922382..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/exceptions/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.exceptions;
-// Exceptions thrown by SCM.
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/CommandQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/CommandQueue.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/CommandQueue.java
deleted file mode 100644
index bbf319b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/CommandQueue.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.node;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.util.Time;
-
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-/**
- * Command Queue is queue of commands for the datanode.
- * <p>
- * Node manager, container Manager and key space managers can queue commands for
- * datanodes into this queue. These commands will be send in the order in which
- * there where queued.
- */
-public class CommandQueue {
-  // This list is used as default return value.
-  private static final List<SCMCommand> DEFAULT_LIST = new LinkedList<>();
-  private final Map<DatanodeID, Commands> commandMap;
-  private final Lock lock;
-  private long commandsInQueue;
-
-  /**
-   * Returns number of commands in queue.
-   * @return Command Count.
-   */
-  public long getCommandsInQueue() {
-    return commandsInQueue;
-  }
-
-  /**
-   * Constructs a Command Queue.
-   * TODO : Add a flusher thread that throws away commands older than a certain
-   * time period.
-   */
-  public CommandQueue() {
-    commandMap = new HashMap<>();
-    lock = new ReentrantLock();
-    commandsInQueue = 0;
-  }
-
-  /**
-   * This function is used only for test purposes.
-   */
-  @VisibleForTesting
-  public void clear() {
-    lock.lock();
-    try {
-      commandMap.clear();
-      commandsInQueue = 0;
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  /**
-   * Returns  a list of Commands for the datanode to execute, if we have no
-   * commands returns a empty list otherwise the current set of
-   * commands are returned and command map set to empty list again.
-   *
-   * @param datanodeID DatanodeID
-   * @return List of SCM Commands.
-   */
-  @SuppressWarnings("unchecked")
-  List<SCMCommand> getCommand(final DatanodeID datanodeID) {
-    lock.lock();
-    try {
-      Commands cmds = commandMap.remove(datanodeID);
-      List<SCMCommand> cmdList = null;
-      if(cmds != null) {
-        cmdList = cmds.getCommands();
-        commandsInQueue -= cmdList.size() > 0 ? cmdList.size() : 0;
-        // A post condition really.
-        Preconditions.checkState(commandsInQueue >= 0);
-      }
-      return cmds == null ? DEFAULT_LIST : cmdList;
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  /**
-   * Adds a Command to the SCM Queue to send the command to container.
-   *
-   * @param datanodeID DatanodeID
-   * @param command    - Command
-   */
-  public void addCommand(final DatanodeID datanodeID, final SCMCommand
-      command) {
-    lock.lock();
-    try {
-      if (commandMap.containsKey(datanodeID)) {
-        commandMap.get(datanodeID).add(command);
-      } else {
-        commandMap.put(datanodeID, new Commands(command));
-      }
-      commandsInQueue++;
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  /**
-   * Class that stores commands for a datanode.
-   */
-  private static class Commands {
-    private long updateTime;
-    private long readTime;
-    private List<SCMCommand> commands;
-
-    /**
-     * Constructs a Commands class.
-     */
-    Commands() {
-      commands = new LinkedList<>();
-      updateTime = 0;
-      readTime = 0;
-    }
-
-    /**
-     * Creates the object and populates with the command.
-     * @param command command to add to queue.
-     */
-    Commands(SCMCommand command) {
-      this();
-      this.add(command);
-    }
-
-    /**
-     * Gets the last time the commands for this node was updated.
-     * @return Time stamp
-     */
-    public long getUpdateTime() {
-      return updateTime;
-    }
-
-    /**
-     * Gets the last read time.
-     * @return last time when these commands were read from this queue.
-     */
-    public long getReadTime() {
-      return readTime;
-    }
-
-    /**
-     * Adds a command to the list.
-     *
-     * @param command SCMCommand
-     */
-    public void add(SCMCommand command) {
-      this.commands.add(command);
-      updateTime = Time.monotonicNow();
-    }
-
-    /**
-     * Returns the commands for this datanode.
-     * @return command list.
-     */
-    public List<SCMCommand> getCommands() {
-      List<SCMCommand> temp = this.commands;
-      this.commands = new LinkedList<>();
-      readTime = Time.monotonicNow();
-      return temp;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/HeartbeatQueueItem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/HeartbeatQueueItem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/HeartbeatQueueItem.java
deleted file mode 100644
index 1e755b2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/HeartbeatQueueItem.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.node;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-
-import static org.apache.hadoop.util.Time.monotonicNow;
-
-/**
- * This class represents the item in SCM heartbeat queue.
- */
-public class HeartbeatQueueItem {
-  private DatanodeID datanodeID;
-  private long recvTimestamp;
-  private SCMNodeReport nodeReport;
-  private ReportState containerReportState;
-
-  /**
-   *
-   * @param datanodeID - datanode ID of the heartbeat.
-   * @param recvTimestamp - heartbeat receive timestamp.
-   * @param nodeReport - node report associated with the heartbeat if any.
-   * @param containerReportState - container report state.
-   */
-  HeartbeatQueueItem(DatanodeID datanodeID, long recvTimestamp,
-      SCMNodeReport nodeReport, ReportState containerReportState) {
-    this.datanodeID = datanodeID;
-    this.recvTimestamp = recvTimestamp;
-    this.nodeReport = nodeReport;
-    this.containerReportState = containerReportState;
-  }
-
-  /**
-   * @return datanode ID.
-   */
-  public DatanodeID getDatanodeID() {
-    return datanodeID;
-  }
-
-  /**
-   * @return node report.
-   */
-  public SCMNodeReport getNodeReport() {
-    return nodeReport;
-  }
-
-  /**
-   * @return container report state.
-   */
-  public ReportState getContainerReportState() {
-    return containerReportState;
-  }
-
-  /**
-   * @return heartbeat receive timestamp.
-   */
-  public long getRecvTimestamp() {
-    return recvTimestamp;
-  }
-
-  /**
-   * Builder for HeartbeatQueueItem.
-   */
-  public static class Builder {
-    private DatanodeID datanodeID;
-    private SCMNodeReport nodeReport;
-    private ReportState containerReportState;
-    private long recvTimestamp = monotonicNow();
-
-    public Builder setDatanodeID(DatanodeID datanodeId) {
-      this.datanodeID = datanodeId;
-      return this;
-    }
-
-    public Builder setNodeReport(SCMNodeReport scmNodeReport) {
-      this.nodeReport = scmNodeReport;
-      return this;
-    }
-
-    public Builder setContainerReportState(ReportState crs) {
-      this.containerReportState = crs;
-      return this;
-    }
-
-    @VisibleForTesting
-    public Builder setRecvTimestamp(long recvTime) {
-      this.recvTimestamp = recvTime;
-      return this;
-    }
-
-    public HeartbeatQueueItem build() {
-      return new HeartbeatQueueItem(datanodeID, recvTimestamp, nodeReport,
-          containerReportState);
-    }
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManager.java
deleted file mode 100644
index 77b3bff..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManager.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.node;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
-import org.apache.hadoop.ozone.protocol.StorageContainerNodeProtocol;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState;
-import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeMetric;
-import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeStat;
-
-import java.io.Closeable;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A node manager supports a simple interface for managing a datanode.
- * <p/>
- * 1. A datanode registers with the NodeManager.
- * <p/>
- * 2. If the node is allowed to register, we add that to the nodes that we need
- * to keep track of.
- * <p/>
- * 3. A heartbeat is made by the node at a fixed frequency.
- * <p/>
- * 4. A node can be in any of these 4 states: {HEALTHY, STALE, DEAD,
- * DECOMMISSIONED}
- * <p/>
- * HEALTHY - It is a datanode that is regularly heartbeating us.
- *
- * STALE - A datanode for which we have missed few heart beats.
- *
- * DEAD - A datanode that we have not heard from for a while.
- *
- * DECOMMISSIONED - Someone told us to remove this node from the tracking
- * list, by calling removeNode. We will throw away this nodes info soon.
- */
-public interface NodeManager extends StorageContainerNodeProtocol,
-    NodeManagerMXBean, Closeable, Runnable {
-  /**
-   * Removes a data node from the management of this Node Manager.
-   *
-   * @param node - DataNode.
-   * @throws UnregisteredNodeException
-   */
-  void removeNode(DatanodeID node) throws UnregisteredNodeException;
-
-  /**
-   * Gets all Live Datanodes that is currently communicating with SCM.
-   * @param nodeState - State of the node
-   * @return List of Datanodes that are Heartbeating SCM.
-   */
-  List<DatanodeID> getNodes(NodeState nodeState);
-
-  /**
-   * Returns the Number of Datanodes that are communicating with SCM.
-   * @param nodeState - State of the node
-   * @return int -- count
-   */
-  int getNodeCount(NodeState nodeState);
-
-  /**
-   * Get all datanodes known to SCM.
-   *
-   * @return List of DatanodeIDs known to SCM.
-   */
-  List<DatanodeID> getAllNodes();
-
-  /**
-   * Chill mode is the period when node manager waits for a minimum
-   * configured number of datanodes to report in. This is called chill mode
-   * to indicate the period before node manager gets into action.
-   *
-   * Forcefully exits the chill mode, even if we have not met the minimum
-   * criteria of the nodes reporting in.
-   */
-  void forceExitChillMode();
-
-  /**
-   * Puts the node manager into manual chill mode.
-   */
-  void enterChillMode();
-
-  /**
-   * Brings node manager out of manual chill mode.
-   */
-  void exitChillMode();
-
-  /**
-   * Returns the aggregated node stats.
-   * @return the aggregated node stats.
-   */
-  SCMNodeStat getStats();
-
-  /**
-   * Return a map of node stats.
-   * @return a map of individual node stats (live/stale but not dead).
-   */
-  Map<String, SCMNodeStat> getNodeStats();
-
-  /**
-   * Return the node stat of the specified datanode.
-   * @param datanodeID - datanode ID.
-   * @return node stat if it is live/stale, null if it is dead or does't exist.
-   */
-  SCMNodeMetric getNodeStat(DatanodeID datanodeID);
-
-  /**
-   * Returns the NodePoolManager associated with the NodeManager.
-   * @return NodePoolManager
-   */
-  NodePoolManager getNodePoolManager();
-
-  /**
-   * Wait for the heartbeat is processed by NodeManager.
-   * @return true if heartbeat has been processed.
-   */
-  @VisibleForTesting
-  boolean waitForHeartbeatProcessed();
-
-  /**
-   * Returns the node state of a specific node.
-   * @param id - DatanodeID
-   * @return Healthy/Stale/Dead.
-   */
-  NodeState getNodeState(DatanodeID id);
-
-  /**
-   * Add a {@link SCMCommand} to the command queue, which are
-   * handled by HB thread asynchronously.
-   * @param id
-   * @param command
-   */
-  void addDatanodeCommand(DatanodeID id, SCMCommand command);
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManagerMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManagerMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManagerMXBean.java
deleted file mode 100644
index 46881c9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManagerMXBean.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.node;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import java.util.Map;
-
-/**
- *
- * This is the JMX management interface for node manager information.
- */
-@InterfaceAudience.Private
-public interface NodeManagerMXBean {
-  /**
-   * Get the minimum number of nodes to get out of chill mode.
-   *
-   * @return int
-   */
-  int getMinimumChillModeNodes();
-
-  /**
-   * Returns a chill mode status string.
-   * @return String
-   */
-  String getChillModeStatus();
-
-
-  /**
-   * Returns true if node manager is out of chill mode, else false.
-   * @return true if out of chill mode, else false
-   */
-  boolean isOutOfChillMode();
-
-  /**
-   * Get the number of data nodes that in all states.
-   *
-   * @return A state to number of nodes that in this state mapping
-   */
-  Map<String, Integer> getNodeCount();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodePoolManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodePoolManager.java
deleted file mode 100644
index d3218b7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodePoolManager.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.node;
-
-
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Interface that defines SCM NodePoolManager.
- */
-public interface NodePoolManager extends Closeable {
-
-  /**
-   * Add a node to a node pool.
-   * @param pool - name of the node pool.
-   * @param node - data node.
-   */
-  void addNode(String pool, DatanodeID node) throws IOException;
-
-  /**
-   * Remove a node from a node pool.
-   * @param pool - name of the node pool.
-   * @param node - data node.
-   * @throws SCMException
-   */
-  void removeNode(String pool, DatanodeID node)
-      throws SCMException;
-
-  /**
-   * Get a list of known node pools.
-   * @return a list of known node pool names or an empty list if not node pool
-   * is defined.
-   */
-  List<String> getNodePools();
-
-  /**
-   * Get all nodes of a node pool given the name of the node pool.
-   * @param pool - name of the node pool.
-   * @return a list of datanode ids or an empty list if the node pool was not
-   *  found.
-   */
-  List<DatanodeID> getNodes(String pool);
-
-  /**
-   * Get the node pool name if the node has been added to a node pool.
-   * @param datanodeID - datanode ID.
-   * @return node pool name if it has been assigned.
-   * null if the node has not been assigned to any node pool yet.
-   */
-  String getNodePool(DatanodeID datanodeID) throws SCMException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/SCMNodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/SCMNodeManager.java
deleted file mode 100644
index 129e65d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/SCMNodeManager.java
+++ /dev/null
@@ -1,883 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.node;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.protocol.StorageContainerNodeProtocol;
-import org.apache.hadoop.ozone.protocol.VersionResponse;
-import org.apache.hadoop.ozone.protocol.commands.ReregisterCommand;
-import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.ozone.protocol.commands.SendContainerCommand;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMRegisteredCmdResponseProto
-    .ErrorCode;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
-import org.apache.hadoop.ozone.protocol
-    .proto.StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-import org.apache.hadoop.ozone.protocol
-    .proto.StorageContainerDatanodeProtocolProtos.SCMStorageReport;
-import org.apache.hadoop.ozone.scm.StorageContainerManager;
-import org.apache.hadoop.ozone.scm.VersionInfo;
-import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeMetric;
-import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeStat;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.ObjectName;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState.DEAD;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState.HEALTHY;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState.STALE;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState.UNKNOWN;
-import static org.apache.hadoop.util.Time.monotonicNow;
-
-/**
- * Maintains information about the Datanodes on SCM side.
- * <p>
- * Heartbeats under SCM is very simple compared to HDFS heartbeatManager.
- * <p>
- * Here we maintain 3 maps, and we propagate a node from healthyNodesMap to
- * staleNodesMap to deadNodesMap. This moving of a node from one map to another
- * is controlled by 4 configuration variables. These variables define how many
- * heartbeats must go missing for the node to move from one map to another.
- * <p>
- * Each heartbeat that SCMNodeManager receives is  put into heartbeatQueue. The
- * worker thread wakes up and grabs that heartbeat from the queue. The worker
- * thread will lookup the healthynodes map and set the timestamp if the entry
- * is there. if not it will look up stale and deadnodes map.
- * <p>
- * The getNode(byState) functions make copy of node maps and then creates a list
- * based on that. It should be assumed that these get functions always report
- * *stale* information. For example, getting the deadNodeCount followed by
- * getNodes(DEAD) could very well produce totally different count. Also
- * getNodeCount(HEALTHY) + getNodeCount(DEAD) + getNodeCode(STALE), is not
- * guaranteed to add up to the total nodes that we know off. Please treat all
- * get functions in this file as a snap-shot of information that is inconsistent
- * as soon as you read it.
- */
-public class SCMNodeManager
-    implements NodeManager, StorageContainerNodeProtocol {
-
-  @VisibleForTesting
-  static final Logger LOG =
-      LoggerFactory.getLogger(SCMNodeManager.class);
-
-  /**
-   * Key = NodeID, value = timestamp.
-   */
-  private final ConcurrentHashMap<String, Long> healthyNodes;
-  private final ConcurrentHashMap<String, Long> staleNodes;
-  private final ConcurrentHashMap<String, Long> deadNodes;
-  private final Queue<HeartbeatQueueItem> heartbeatQueue;
-  private final ConcurrentHashMap<String, DatanodeID> nodes;
-  // Individual live node stats
-  private final ConcurrentHashMap<String, SCMNodeStat> nodeStats;
-  // Aggregated node stats
-  private SCMNodeStat scmStat;
-  // TODO: expose nodeStats and scmStat as metrics
-  private final AtomicInteger healthyNodeCount;
-  private final AtomicInteger staleNodeCount;
-  private final AtomicInteger deadNodeCount;
-  private final AtomicInteger totalNodes;
-  private long staleNodeIntervalMs;
-  private final long deadNodeIntervalMs;
-  private final long heartbeatCheckerIntervalMs;
-  private final long datanodeHBIntervalSeconds;
-  private final ScheduledExecutorService executorService;
-  private long lastHBcheckStart;
-  private long lastHBcheckFinished = 0;
-  private long lastHBProcessedCount;
-  private int chillModeNodeCount;
-  private final int maxHBToProcessPerLoop;
-  private final String clusterID;
-  private final VersionInfo version;
-  /**
-   * During start up of SCM, it will enter into chill mode and will be there
-   * until number of Datanodes registered reaches {@code chillModeNodeCount}.
-   * This flag is for tracking startup chill mode.
-   */
-  private AtomicBoolean inStartupChillMode;
-  /**
-   * Administrator can put SCM into chill mode manually.
-   * This flag is for tracking manual chill mode.
-   */
-  private AtomicBoolean inManualChillMode;
-  private final CommandQueue commandQueue;
-  // Node manager MXBean
-  private ObjectName nmInfoBean;
-
-  // Node pool manager.
-  private final SCMNodePoolManager nodePoolManager;
-  private final StorageContainerManager scmManager;
-
-  /**
-   * Constructs SCM machine Manager.
-   */
-  public SCMNodeManager(OzoneConfiguration conf, String clusterID,
-      StorageContainerManager scmManager) throws IOException {
-    heartbeatQueue = new ConcurrentLinkedQueue<>();
-    healthyNodes = new ConcurrentHashMap<>();
-    deadNodes = new ConcurrentHashMap<>();
-    staleNodes = new ConcurrentHashMap<>();
-    nodes = new ConcurrentHashMap<>();
-    nodeStats = new ConcurrentHashMap();
-    scmStat = new SCMNodeStat();
-
-    healthyNodeCount = new AtomicInteger(0);
-    staleNodeCount = new AtomicInteger(0);
-    deadNodeCount = new AtomicInteger(0);
-    totalNodes = new AtomicInteger(0);
-    this.clusterID = clusterID;
-    this.version = VersionInfo.getLatestVersion();
-    commandQueue = new CommandQueue();
-
-    // TODO: Support this value as a Percentage of known machines.
-    chillModeNodeCount = 1;
-
-    staleNodeIntervalMs = OzoneClientUtils.getStaleNodeInterval(conf);
-    deadNodeIntervalMs = OzoneClientUtils.getDeadNodeInterval(conf);
-    heartbeatCheckerIntervalMs =
-        OzoneClientUtils.getScmheartbeatCheckerInterval(conf);
-    datanodeHBIntervalSeconds = OzoneClientUtils.getScmHeartbeatInterval(conf);
-    maxHBToProcessPerLoop = OzoneClientUtils.getMaxHBToProcessPerLoop(conf);
-
-    executorService = HadoopExecutors.newScheduledThreadPool(1,
-        new ThreadFactoryBuilder().setDaemon(true)
-            .setNameFormat("SCM Heartbeat Processing Thread - %d").build());
-
-    LOG.info("Entering startup chill mode.");
-    this.inStartupChillMode = new AtomicBoolean(true);
-    this.inManualChillMode = new AtomicBoolean(false);
-
-    Preconditions.checkState(heartbeatCheckerIntervalMs > 0);
-    executorService.schedule(this, heartbeatCheckerIntervalMs,
-        TimeUnit.MILLISECONDS);
-
-    registerMXBean();
-
-    this.nodePoolManager = new SCMNodePoolManager(conf);
-    this.scmManager = scmManager;
-  }
-
-  private void registerMXBean() {
-    this.nmInfoBean = MBeans.register("SCMNodeManager",
-        "SCMNodeManagerInfo", this);
-  }
-
-  private void unregisterMXBean() {
-    if(this.nmInfoBean != null) {
-      MBeans.unregister(this.nmInfoBean);
-      this.nmInfoBean = null;
-    }
-  }
-
-  /**
-   * Removes a data node from the management of this Node Manager.
-   *
-   * @param node - DataNode.
-   * @throws UnregisteredNodeException
-   */
-  @Override
-  public void removeNode(DatanodeID node) throws UnregisteredNodeException {
-    // TODO : Fix me when adding the SCM CLI.
-
-  }
-
-  /**
-   * Gets all datanodes that are in a certain state. This function works by
-   * taking a snapshot of the current collection and then returning the list
-   * from that collection. This means that real map might have changed by the
-   * time we return this list.
-   *
-   * @return List of Datanodes that are known to SCM in the requested state.
-   */
-  @Override
-  public List<DatanodeID> getNodes(NodeState nodestate)
-      throws IllegalArgumentException {
-    Map<String, Long> set;
-    switch (nodestate) {
-    case HEALTHY:
-      synchronized (this) {
-        set = Collections.unmodifiableMap(new HashMap<>(healthyNodes));
-      }
-      break;
-    case STALE:
-      synchronized (this) {
-        set = Collections.unmodifiableMap(new HashMap<>(staleNodes));
-      }
-      break;
-    case DEAD:
-      synchronized (this) {
-        set = Collections.unmodifiableMap(new HashMap<>(deadNodes));
-      }
-      break;
-    default:
-      throw new IllegalArgumentException("Unknown node state requested.");
-    }
-
-    return set.entrySet().stream().map(entry -> nodes.get(entry.getKey()))
-        .collect(Collectors.toList());
-  }
-
-  /**
-   * Returns all datanodes that are known to SCM.
-   *
-   * @return List of DatanodeIDs
-   */
-  @Override
-  public List<DatanodeID> getAllNodes() {
-    Map<String, DatanodeID> set;
-    synchronized (this) {
-      set = Collections.unmodifiableMap(new HashMap<>(nodes));
-    }
-    return set.entrySet().stream().map(entry -> nodes.get(entry.getKey()))
-        .collect(Collectors.toList());
-  }
-
-  /**
-   * Get the minimum number of nodes to get out of Chill mode.
-   *
-   * @return int
-   */
-  @Override
-  public int getMinimumChillModeNodes() {
-    return chillModeNodeCount;
-  }
-
-  /**
-   * Sets the Minimum chill mode nodes count, used only in testing.
-   *
-   * @param count - Number of nodes.
-   */
-  @VisibleForTesting
-  public void setMinimumChillModeNodes(int count) {
-    chillModeNodeCount = count;
-  }
-
-  /**
-   * Returns chill mode Status string.
-   * @return String
-   */
-  @Override
-  public String getChillModeStatus() {
-    if (inStartupChillMode.get()) {
-      return "Still in chill mode, waiting on nodes to report in." +
-          String.format(" %d nodes reported, minimal %d nodes required.",
-              totalNodes.get(), getMinimumChillModeNodes());
-    }
-    if (inManualChillMode.get()) {
-      return "Out of startup chill mode, but in manual chill mode." +
-          String.format(" %d nodes have reported in.", totalNodes.get());
-    }
-    return "Out of chill mode." +
-        String.format(" %d nodes have reported in.", totalNodes.get());
-  }
-
-  /**
-   * Forcefully exits the chill mode even if we have not met the minimum
-   * criteria of exiting the chill mode. This will exit from both startup
-   * and manual chill mode.
-   */
-  @Override
-  public void forceExitChillMode() {
-    if(inStartupChillMode.get()) {
-      LOG.info("Leaving startup chill mode.");
-      inStartupChillMode.set(false);
-    }
-    if(inManualChillMode.get()) {
-      LOG.info("Leaving manual chill mode.");
-      inManualChillMode.set(false);
-    }
-  }
-
-  /**
-   * Puts the node manager into manual chill mode.
-   */
-  @Override
-  public void enterChillMode() {
-    LOG.info("Entering manual chill mode.");
-    inManualChillMode.set(true);
-  }
-
-  /**
-   * Brings node manager out of manual chill mode.
-   */
-  @Override
-  public void exitChillMode() {
-    LOG.info("Leaving manual chill mode.");
-    inManualChillMode.set(false);
-  }
-
-  /**
-   * Returns true if node manager is out of chill mode, else false.
-   * @return true if out of chill mode, else false
-   */
-  @Override
-  public boolean isOutOfChillMode() {
-    return !(inStartupChillMode.get() || inManualChillMode.get());
-  }
-
-  /**
-   * Returns the Number of Datanodes by State they are in.
-   *
-   * @return int -- count
-   */
-  @Override
-  public int getNodeCount(NodeState nodestate) {
-    switch (nodestate) {
-    case HEALTHY:
-      return healthyNodeCount.get();
-    case STALE:
-      return staleNodeCount.get();
-    case DEAD:
-      return deadNodeCount.get();
-    case UNKNOWN:
-      // This is unknown due to the fact that some nodes can be in
-      // transit between the other states. Returning a count for that is not
-      // possible. The fact that we have such state is to deal with the fact
-      // that this information might not be consistent always.
-      return 0;
-    default:
-      return 0;
-    }
-  }
-
-  /**
-   * Used for testing.
-   *
-   * @return true if the HB check is done.
-   */
-  @VisibleForTesting
-  @Override
-  public boolean waitForHeartbeatProcessed() {
-    return lastHBcheckFinished != 0;
-  }
-
-  /**
-   * Returns the node state of a specific node.
-   *
-   * @param id - DatanodeID
-   * @return Healthy/Stale/Dead/Unknown.
-   */
-  @Override
-  public NodeState getNodeState(DatanodeID id) {
-    // There is a subtle race condition here, hence we also support
-    // the NODEState.UNKNOWN. It is possible that just before we check the
-    // healthyNodes, we have removed the node from the healthy list but stil
-    // not added it to Stale Nodes list.
-    // We can fix that by adding the node to stale list before we remove, but
-    // then the node is in 2 states to avoid this race condition. Instead we
-    // just deal with the possibilty of getting a state called unknown.
-
-    if(healthyNodes.containsKey(id.getDatanodeUuid())) {
-      return HEALTHY;
-    }
-
-    if(staleNodes.containsKey(id.getDatanodeUuid())) {
-      return STALE;
-    }
-
-    if(deadNodes.containsKey(id.getDatanodeUuid())) {
-      return DEAD;
-    }
-
-    return UNKNOWN;
-  }
-
-  /**
-   * This is the real worker thread that processes the HB queue. We do the
-   * following things in this thread.
-   * <p>
-   * Process the Heartbeats that are in the HB Queue. Move Stale or Dead node to
-   * healthy if we got a heartbeat from them. Move Stales Node to dead node
-   * table if it is needed. Move healthy nodes to stale nodes if it is needed.
-   * <p>
-   * if it is a new node, we call register node and add it to the list of nodes.
-   * This will be replaced when we support registration of a node in SCM.
-   *
-   * @see Thread#run()
-   */
-  @Override
-  public void run() {
-    lastHBcheckStart = monotonicNow();
-    lastHBProcessedCount = 0;
-
-    // Process the whole queue.
-    while (!heartbeatQueue.isEmpty() &&
-        (lastHBProcessedCount < maxHBToProcessPerLoop)) {
-      HeartbeatQueueItem hbItem = heartbeatQueue.poll();
-      synchronized (this) {
-        handleHeartbeat(hbItem);
-      }
-      // we are shutting down or something give up processing the rest of
-      // HBs. This will terminate the HB processing thread.
-      if (Thread.currentThread().isInterrupted()) {
-        LOG.info("Current Thread is isInterrupted, shutting down HB " +
-            "processing thread for Node Manager.");
-        return;
-      }
-    }
-
-    if (lastHBProcessedCount >= maxHBToProcessPerLoop) {
-      LOG.error("SCM is being flooded by heartbeats. Not able to keep up with" +
-          " the heartbeat counts. Processed {} heartbeats. Breaking out of" +
-          " loop. Leaving rest to be processed later. ", lastHBProcessedCount);
-    }
-
-    // Iterate over the Stale nodes and decide if we need to move any node to
-    // dead State.
-    long currentTime = monotonicNow();
-    for (Map.Entry<String, Long> entry : staleNodes.entrySet()) {
-      if (currentTime - entry.getValue() > deadNodeIntervalMs) {
-        synchronized (this) {
-          moveStaleNodeToDead(entry);
-        }
-      }
-    }
-
-    // Iterate over the healthy nodes and decide if we need to move any node to
-    // Stale State.
-    currentTime = monotonicNow();
-    for (Map.Entry<String, Long> entry : healthyNodes.entrySet()) {
-      if (currentTime - entry.getValue() > staleNodeIntervalMs) {
-        synchronized (this) {
-          moveHealthyNodeToStale(entry);
-        }
-      }
-    }
-    lastHBcheckFinished = monotonicNow();
-
-    monitorHBProcessingTime();
-
-    // we purposefully make this non-deterministic. Instead of using a
-    // scheduleAtFixedFrequency  we will just go to sleep
-    // and wake up at the next rendezvous point, which is currentTime +
-    // heartbeatCheckerIntervalMs. This leads to the issue that we are now
-    // heart beating not at a fixed cadence, but clock tick + time taken to
-    // work.
-    //
-    // This time taken to work can skew the heartbeat processor thread.
-    // The reason why we don't care is because of the following reasons.
-    //
-    // 1. checkerInterval is general many magnitudes faster than datanode HB
-    // frequency.
-    //
-    // 2. if we have too much nodes, the SCM would be doing only HB
-    // processing, this could lead to SCM's CPU starvation. With this
-    // approach we always guarantee that  HB thread sleeps for a little while.
-    //
-    // 3. It is possible that we will never finish processing the HB's in the
-    // thread. But that means we have a mis-configured system. We will warn
-    // the users by logging that information.
-    //
-    // 4. And the most important reason, heartbeats are not blocked even if
-    // this thread does not run, they will go into the processing queue.
-
-    if (!Thread.currentThread().isInterrupted() &&
-        !executorService.isShutdown()) {
-      executorService.schedule(this, heartbeatCheckerIntervalMs, TimeUnit
-          .MILLISECONDS);
-    } else {
-      LOG.info("Current Thread is interrupted, shutting down HB processing " +
-          "thread for Node Manager.");
-    }
-  }
-
-  /**
-   * If we have taken too much time for HB processing, log that information.
-   */
-  private void monitorHBProcessingTime() {
-    if (TimeUnit.MILLISECONDS.toSeconds(lastHBcheckFinished -
-        lastHBcheckStart) > datanodeHBIntervalSeconds) {
-      LOG.error("Total time spend processing datanode HB's is greater than " +
-              "configured values for datanode heartbeats. Please adjust the" +
-              " heartbeat configs. Time Spend on HB processing: {} seconds " +
-              "Datanode heartbeat Interval: {} seconds , heartbeats " +
-              "processed: {}",
-          TimeUnit.MILLISECONDS
-              .toSeconds(lastHBcheckFinished - lastHBcheckStart),
-          datanodeHBIntervalSeconds, lastHBProcessedCount);
-    }
-  }
-
-  /**
-   * Moves a Healthy node to a Stale node state.
-   *
-   * @param entry - Map Entry
-   */
-  private void moveHealthyNodeToStale(Map.Entry<String, Long> entry) {
-    LOG.trace("Moving healthy node to stale: {}", entry.getKey());
-    healthyNodes.remove(entry.getKey());
-    healthyNodeCount.decrementAndGet();
-    staleNodes.put(entry.getKey(), entry.getValue());
-    staleNodeCount.incrementAndGet();
-
-    if (scmManager != null) {
-      // remove stale node's container report
-      scmManager.removeContainerReport(entry.getKey());
-    }
-  }
-
-  /**
-   * Moves a Stale node to a dead node state.
-   *
-   * @param entry - Map Entry
-   */
-  private void moveStaleNodeToDead(Map.Entry<String, Long> entry) {
-    LOG.trace("Moving stale node to dead: {}", entry.getKey());
-    staleNodes.remove(entry.getKey());
-    staleNodeCount.decrementAndGet();
-    deadNodes.put(entry.getKey(), entry.getValue());
-    deadNodeCount.incrementAndGet();
-
-    // Update SCM node stats
-    SCMNodeStat deadNodeStat = nodeStats.get(entry.getKey());
-    scmStat.subtract(deadNodeStat);
-    nodeStats.remove(entry.getKey());
-  }
-
-  /**
-   * Handles a single heartbeat from a datanode.
-   *
-   * @param hbItem - heartbeat item from a datanode.
-   */
-  private void handleHeartbeat(HeartbeatQueueItem hbItem) {
-    lastHBProcessedCount++;
-
-    DatanodeID datanodeID = hbItem.getDatanodeID();
-    String datanodeUuid = datanodeID.getDatanodeUuid();
-    SCMNodeReport nodeReport = hbItem.getNodeReport();
-    long recvTimestamp = hbItem.getRecvTimestamp();
-    long processTimestamp = Time.monotonicNow();
-    if (LOG.isTraceEnabled()) {
-      //TODO: add average queue time of heartbeat request as metrics
-      LOG.trace("Processing Heartbeat from datanode {}: queueing time {}",
-          datanodeUuid, processTimestamp - recvTimestamp);
-    }
-
-    // If this node is already in the list of known and healthy nodes
-    // just set the last timestamp and return.
-    if (healthyNodes.containsKey(datanodeUuid)) {
-      healthyNodes.put(datanodeUuid, processTimestamp);
-      updateNodeStat(datanodeUuid, nodeReport);
-      updateCommandQueue(datanodeID,
-          hbItem.getContainerReportState().getState());
-      return;
-    }
-
-    // A stale node has heartbeat us we need to remove the node from stale
-    // list and move to healthy list.
-    if (staleNodes.containsKey(datanodeUuid)) {
-      staleNodes.remove(datanodeUuid);
-      healthyNodes.put(datanodeUuid, processTimestamp);
-      healthyNodeCount.incrementAndGet();
-      staleNodeCount.decrementAndGet();
-      updateNodeStat(datanodeUuid, nodeReport);
-      updateCommandQueue(datanodeID,
-          hbItem.getContainerReportState().getState());
-      return;
-    }
-
-    // A dead node has heartbeat us, we need to remove that node from dead
-    // node list and move it to the healthy list.
-    if (deadNodes.containsKey(datanodeUuid)) {
-      deadNodes.remove(datanodeUuid);
-      healthyNodes.put(datanodeUuid, processTimestamp);
-      deadNodeCount.decrementAndGet();
-      healthyNodeCount.incrementAndGet();
-      updateNodeStat(datanodeUuid, nodeReport);
-      updateCommandQueue(datanodeID,
-          hbItem.getContainerReportState().getState());
-      return;
-    }
-
-    LOG.warn("SCM receive heartbeat from unregistered datanode {}",
-        datanodeUuid);
-    this.commandQueue.addCommand(hbItem.getDatanodeID(),
-        new ReregisterCommand());
-  }
-
-  private void updateNodeStat(String datanodeUuid, SCMNodeReport nodeReport) {
-    SCMNodeStat stat = nodeStats.get(datanodeUuid);
-    if (stat == null) {
-      LOG.debug("SCM updateNodeStat based on heartbeat from previous" +
-          "dead datanode {}", datanodeUuid);
-      stat = new SCMNodeStat();
-    }
-
-    if (nodeReport != null && nodeReport.getStorageReportCount() > 0) {
-      long totalCapacity = 0;
-      long totalRemaining = 0;
-      long totalScmUsed = 0;
-      List<SCMStorageReport> storageReports = nodeReport.getStorageReportList();
-      for (SCMStorageReport report : storageReports) {
-        totalCapacity += report.getCapacity();
-        totalRemaining +=  report.getRemaining();
-        totalScmUsed+= report.getScmUsed();
-      }
-      scmStat.subtract(stat);
-      stat.set(totalCapacity, totalScmUsed, totalRemaining);
-      nodeStats.put(datanodeUuid, stat);
-      scmStat.add(stat);
-    }
-  }
-
-  private void updateCommandQueue(DatanodeID datanodeID,
-                                  ReportState.states containerReportState) {
-    if (containerReportState != null) {
-      switch (containerReportState) {
-      case completeContinerReport:
-        commandQueue.addCommand(datanodeID,
-            SendContainerCommand.newBuilder().build());
-        return;
-      case deltaContainerReport:
-      case noContainerReports:
-      default:
-        // do nothing
-      }
-    }
-  }
-
-  /**
-   * Closes this stream and releases any system resources associated with it. If
-   * the stream is already closed then invoking this method has no effect.
-   *
-   * @throws IOException if an I/O error occurs
-   */
-  @Override
-  public void close() throws IOException {
-    unregisterMXBean();
-    nodePoolManager.close();
-    executorService.shutdown();
-    try {
-      if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
-        executorService.shutdownNow();
-      }
-
-      if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
-        LOG.error("Unable to shutdown NodeManager properly.");
-      }
-    } catch (InterruptedException e) {
-      executorService.shutdownNow();
-      Thread.currentThread().interrupt();
-    }
-  }
-
-  @VisibleForTesting
-  long getLastHBProcessedCount() {
-    return lastHBProcessedCount;
-  }
-
-  /**
-   * Gets the version info from SCM.
-   *
-   * @param versionRequest - version Request.
-   * @return - returns SCM version info and other required information needed by
-   * datanode.
-   */
-  @Override
-  public VersionResponse getVersion(SCMVersionRequestProto versionRequest) {
-    return VersionResponse.newBuilder()
-        .setVersion(this.version.getVersion())
-        .build();
-  }
-
-  /**
-   * Register the node if the node finds that it is not registered with any
-   * SCM.
-   *
-   * @param datanodeID - Send datanodeID with Node info. This function
-   *                   generates and assigns new datanode ID for the datanode.
-   *                   This allows SCM to be run independent of Namenode if
-   *                   required.
-   *
-   * @return SCMHeartbeatResponseProto
-   */
-  @Override
-  public SCMCommand register(DatanodeID datanodeID) {
-
-    SCMCommand responseCommand = verifyDatanodeUUID(datanodeID);
-    if (responseCommand != null) {
-      return responseCommand;
-    }
-
-    nodes.put(datanodeID.getDatanodeUuid(), datanodeID);
-    totalNodes.incrementAndGet();
-    healthyNodes.put(datanodeID.getDatanodeUuid(), monotonicNow());
-    healthyNodeCount.incrementAndGet();
-    nodeStats.put(datanodeID.getDatanodeUuid(), new SCMNodeStat());
-
-    if(inStartupChillMode.get() &&
-        totalNodes.get() >= getMinimumChillModeNodes()) {
-      inStartupChillMode.getAndSet(false);
-      LOG.info("Leaving startup chill mode.");
-    }
-
-    // TODO: define node pool policy for non-default node pool.
-    // For now, all nodes are added to the "DefaultNodePool" upon registration
-    // if it has not been added to any node pool yet.
-    try {
-      if (nodePoolManager.getNodePool(datanodeID) == null) {
-        nodePoolManager.addNode(SCMNodePoolManager.DEFAULT_NODEPOOL,
-            datanodeID);
-      }
-    } catch (IOException e) {
-      // TODO: make sure registration failure is handled correctly.
-      return RegisteredCommand.newBuilder()
-          .setErrorCode(ErrorCode.errorNodeNotPermitted)
-          .build();
-    }
-    LOG.info("Data node with ID: {} Registered.",
-        datanodeID.getDatanodeUuid());
-    return RegisteredCommand.newBuilder()
-        .setErrorCode(ErrorCode.success)
-        .setDatanodeUUID(datanodeID.getDatanodeUuid())
-        .setClusterID(this.clusterID)
-        .build();
-  }
-
-  /**
-   * Verifies the datanode does not have a valid UUID already.
-   *
-   * @param datanodeID - Datanode UUID.
-   * @return SCMCommand
-   */
-  private SCMCommand verifyDatanodeUUID(DatanodeID datanodeID) {
-    if (datanodeID.getDatanodeUuid() != null &&
-        nodes.containsKey(datanodeID.getDatanodeUuid())) {
-      LOG.trace("Datanode is already registered. Datanode: {}",
-          datanodeID.toString());
-      return RegisteredCommand.newBuilder()
-          .setErrorCode(ErrorCode.success)
-          .setClusterID(this.clusterID)
-          .setDatanodeUUID(datanodeID.getDatanodeUuid())
-          .build();
-    }
-    return null;
-  }
-
-  /**
-   * Send heartbeat to indicate the datanode is alive and doing well.
-   *
-   * @param datanodeID - Datanode ID.
-   * @param nodeReport - node report.
-   * @param containerReportState - container report state.
-   * @return SCMheartbeat response.
-   * @throws IOException
-   */
-  @Override
-  public List<SCMCommand> sendHeartbeat(DatanodeID datanodeID,
-      SCMNodeReport nodeReport, ReportState containerReportState) {
-
-    // Checking for NULL to make sure that we don't get
-    // an exception from ConcurrentList.
-    // This could be a problem in tests, if this function is invoked via
-    // protobuf, transport layer will guarantee that this is not null.
-    if (datanodeID != null) {
-      heartbeatQueue.add(
-          new HeartbeatQueueItem.Builder()
-              .setDatanodeID(datanodeID)
-              .setNodeReport(nodeReport)
-              .setContainerReportState(containerReportState)
-              .build());
-    } else {
-      LOG.error("Datanode ID in heartbeat is null");
-    }
-
-    return commandQueue.getCommand(datanodeID);
-  }
-
-  /**
-   * Returns the aggregated node stats.
-   * @return the aggregated node stats.
-   */
-  @Override
-  public SCMNodeStat getStats() {
-    return new SCMNodeStat(this.scmStat);
-  }
-
-  /**
-   * Return a map of node stats.
-   * @return a map of individual node stats (live/stale but not dead).
-   */
-  @Override
-  public Map<String, SCMNodeStat> getNodeStats() {
-    return Collections.unmodifiableMap(nodeStats);
-  }
-
-  /**
-   * Return the node stat of the specified datanode.
-   * @param datanodeID - datanode ID.
-   * @return node stat if it is live/stale, null if it is dead or does't exist.
-   */
-  @Override
-  public SCMNodeMetric getNodeStat(DatanodeID datanodeID) {
-    return new SCMNodeMetric(nodeStats.get(datanodeID.getDatanodeUuid()));
-  }
-
-  @Override
-  public NodePoolManager getNodePoolManager() {
-    return nodePoolManager;
-  }
-
-  @Override
-  public Map<String, Integer> getNodeCount() {
-    Map<String, Integer> nodeCountMap = new HashMap<String, Integer>();
-    for(NodeState state : NodeState.values()) {
-      nodeCountMap.put(state.toString(), getNodeCount(state));
-    }
-    return nodeCountMap;
-  }
-
-  @Override
-  public void addDatanodeCommand(DatanodeID id, SCMCommand command) {
-    this.commandQueue.addCommand(id, command);
-  }
-
-  @VisibleForTesting
-  public void setStaleNodeIntervalMs(long interval) {
-    this.staleNodeIntervalMs = interval;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/SCMNodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/SCMNodePoolManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/SCMNodePoolManager.java
deleted file mode 100644
index 55bf1c4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/SCMNodePoolManager.java
+++ /dev/null
@@ -1,266 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.node;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
-import static org.apache.hadoop.ozone.scm
-    .exceptions.SCMException.ResultCodes.FAILED_TO_LOAD_NODEPOOL;
-import static org.apache.hadoop.ozone.scm
-    .exceptions.SCMException.ResultCodes.FAILED_TO_FIND_NODE_IN_POOL;
-import static org.apache.hadoop.scm
-    .ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.scm
-    .ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB;
-
-/**
- * SCM node pool manager that manges node pools.
- */
-public final class SCMNodePoolManager implements NodePoolManager {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(SCMNodePoolManager.class);
-  private static final List<DatanodeID> EMPTY_NODE_LIST = new ArrayList<>();
-  private static final List<String> EMPTY_NODEPOOL_LIST = new ArrayList<>();
-  public static final String DEFAULT_NODEPOOL = "DefaultNodePool";
-
-  // DB that saves the node to node pool mapping.
-  private MetadataStore nodePoolStore;
-
-  // In-memory node pool to nodes mapping
-  private HashMap<String, Set<DatanodeID>> nodePools;
-
-  // Read-write lock for nodepool operations
-  private ReadWriteLock lock;
-
-  /**
-   * Construct SCMNodePoolManager class that manages node to node pool mapping.
-   * @param conf - configuration.
-   * @throws IOException
-   */
-  public SCMNodePoolManager(final OzoneConfiguration conf)
-      throws IOException {
-    final int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
-        OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
-    File metaDir = OzoneUtils.getOzoneMetaDirPath(conf);
-    String scmMetaDataDir = metaDir.getPath();
-    File nodePoolDBPath = new File(scmMetaDataDir, NODEPOOL_DB);
-    nodePoolStore = MetadataStoreBuilder.newBuilder()
-        .setConf(conf)
-        .setDbFile(nodePoolDBPath)
-        .setCacheSize(cacheSize * OzoneConsts.MB)
-        .build();
-    nodePools = new HashMap<>();
-    lock = new ReentrantReadWriteLock();
-    init();
-  }
-
-  /**
-   * Initialize the in-memory store based on persist store from level db.
-   * No lock is needed as init() is only invoked by constructor.
-   * @throws SCMException
-   */
-  private void init() throws SCMException {
-    try {
-      nodePoolStore.iterate(null, (key, value) -> {
-        try {
-          DatanodeID nodeId = DatanodeID.getFromProtoBuf(
-              HdfsProtos.DatanodeIDProto.PARSER.parseFrom(key));
-          String poolName = DFSUtil.bytes2String(value);
-
-          Set<DatanodeID> nodePool = null;
-          if (nodePools.containsKey(poolName)) {
-            nodePool = nodePools.get(poolName);
-          } else {
-            nodePool = new HashSet<>();
-            nodePools.put(poolName, nodePool);
-          }
-          nodePool.add(nodeId);
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Adding node: {} to node pool: {}",
-                nodeId, poolName);
-          }
-        } catch (IOException e) {
-          LOG.warn("Can't add a datanode to node pool, continue next...");
-        }
-        return true;
-      });
-    } catch (IOException e) {
-      LOG.error("Loading node pool error " + e);
-      throw new SCMException("Failed to load node pool",
-          FAILED_TO_LOAD_NODEPOOL);
-    }
-  }
-
-  /**
-   * Add a datanode to a node pool.
-   * @param pool - name of the node pool.
-   * @param node - name of the datanode.
-   */
-  @Override
-  public void addNode(final String pool, final DatanodeID node)
-      throws IOException {
-    Preconditions.checkNotNull(pool, "pool name is null");
-    Preconditions.checkNotNull(node, "node is null");
-    lock.writeLock().lock();
-    try {
-      // add to the persistent store
-      nodePoolStore.put(node.getProtoBufMessage().toByteArray(),
-          DFSUtil.string2Bytes(pool));
-
-      // add to the in-memory store
-      Set<DatanodeID> nodePool = null;
-      if (nodePools.containsKey(pool)) {
-        nodePool = nodePools.get(pool);
-      } else {
-        nodePool = new HashSet<DatanodeID>();
-        nodePools.put(pool, nodePool);
-      }
-      nodePool.add(node);
-    } finally {
-      lock.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Remove a datanode from a node pool.
-   * @param pool - name of the node pool.
-   * @param node - datanode id.
-   * @throws SCMException
-   */
-  @Override
-  public void removeNode(final String pool, final DatanodeID node)
-      throws SCMException {
-    Preconditions.checkNotNull(pool, "pool name is null");
-    Preconditions.checkNotNull(node, "node is null");
-    lock.writeLock().lock();
-    try {
-      // Remove from the persistent store
-      byte[] kName = node.getProtoBufMessage().toByteArray();
-      byte[] kData = nodePoolStore.get(kName);
-      if (kData == null) {
-        throw new SCMException(String.format("Unable to find node %s from" +
-            " pool %s in DB.", DFSUtil.bytes2String(kName), pool),
-            FAILED_TO_FIND_NODE_IN_POOL);
-      }
-      nodePoolStore.delete(kName);
-
-      // Remove from the in-memory store
-      if (nodePools.containsKey(pool)) {
-        Set<DatanodeID> nodePool = nodePools.get(pool);
-        nodePool.remove(node);
-      } else {
-        throw new SCMException(String.format("Unable to find node %s from" +
-            " pool %s in MAP.", DFSUtil.bytes2String(kName), pool),
-            FAILED_TO_FIND_NODE_IN_POOL);      }
-    } catch (IOException e) {
-      throw new SCMException("Failed to remove node " + node.toString()
-          + " from node pool " + pool, e,
-          SCMException.ResultCodes.IO_EXCEPTION);
-    } finally {
-      lock.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Get all the node pools.
-   * @return all the node pools.
-   */
-  @Override
-  public List<String> getNodePools() {
-    lock.readLock().lock();
-    try {
-      if (!nodePools.isEmpty()) {
-        return nodePools.keySet().stream().collect(Collectors.toList());
-      } else {
-        return EMPTY_NODEPOOL_LIST;
-      }
-    } finally {
-      lock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Get all datanodes of a specific node pool.
-   * @param pool - name of the node pool.
-   * @return all datanodes of the specified node pool.
-   */
-  @Override
-  public List<DatanodeID> getNodes(final String pool) {
-    Preconditions.checkNotNull(pool, "pool name is null");
-    if (nodePools.containsKey(pool)) {
-      return nodePools.get(pool).stream().collect(Collectors.toList());
-    } else {
-      return EMPTY_NODE_LIST;
-    }
-  }
-
-  /**
-   * Get the node pool name if the node has been added to a node pool.
-   * @param datanodeID - datanode ID.
-   * @return node pool name if it has been assigned.
-   * null if the node has not been assigned to any node pool yet.
-   * TODO: Put this in a in-memory map if performance is an issue.
-   */
-  @Override
-  public String getNodePool(final DatanodeID datanodeID) throws SCMException {
-    Preconditions.checkNotNull(datanodeID, "node is null");
-    try {
-      byte[]  result = nodePoolStore.get(
-          datanodeID.getProtoBufMessage().toByteArray());
-      return result == null ? null : DFSUtil.bytes2String(result);
-    } catch (IOException e) {
-      throw new SCMException("Failed to get node pool for node "
-          + datanodeID.toString(), e,
-          SCMException.ResultCodes.IO_EXCEPTION);
-    }
-  }
-
-  /**
-   * Close node pool level db store.
-   * @throws IOException
-   */
-  @Override
-  public void close() throws IOException {
-    nodePoolStore.close();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/package-info.java
deleted file mode 100644
index f1efe79..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/package-info.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.node;
-
-/**
- * The node package deals with node management.
- * <p/>
- * The node manager takes care of node registrations, removal of node and
- * handling of heartbeats.
- * <p/>
- * The node manager maintains statistics that gets send as part of
- * heartbeats.
- * <p/>
- * The container manager polls the node manager to learn the state of
- * datanodes  that it is interested in.
- * <p/>
- */
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[07/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js
deleted file mode 100644
index 1664873..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js
+++ /dev/null
@@ -1,5 +0,0 @@
-!function(){function n(n){return n&&(n.ownerDocument||n.document||n).documentElement}function t(n){return n&&(n.ownerDocument&&n.ownerDocument.defaultView||n.document&&n||n.defaultView)}function e(n,t){return t>n?-1:n>t?1:n>=t?0:NaN}function r(n){return null===n?NaN:+n}function i(n){return!isNaN(n)}function u(n){return{left:function(t,e,r,i){for(arguments.length<3&&(r=0),arguments.length<4&&(i=t.length);i>r;){var u=r+i>>>1;n(t[u],e)<0?r=u+1:i=u}return r},right:function(t,e,r,i){for(arguments.length<3&&(r=0),arguments.length<4&&(i=t.length);i>r;){var u=r+i>>>1;n(t[u],e)>0?i=u:r=u+1}return r}}}function o(n){return n.length}function a(n){for(var t=1;n*t%1;)t*=10;return t}function l(n,t){for(var e in t)Object.defineProperty(n.prototype,e,{value:t[e],enumerable:!1})}function c(){this._=Object.create(null)}function f(n){return(n+="")===bo||n[0]===_o?_o+n:n}function s(n){return(n+="")[0]===_o?n.slice(1):n}function h(n){return f(n)in this._}function p(n){return(n=f(n))in this._&&delete this
 ._[n]}function g(){var n=[];for(var t in this._)n.push(s(t));return n}function v(){var n=0;for(var t in this._)++n;return n}function d(){for(var n in this._)return!1;return!0}function y(){this._=Object.create(null)}function m(n){return n}function M(n,t,e){return function(){var r=e.apply(t,arguments);return r===t?n:r}}function x(n,t){if(t in n)return t;t=t.charAt(0).toUpperCase()+t.slice(1);for(var e=0,r=wo.length;r>e;++e){var i=wo[e]+t;if(i in n)return i}}function b(){}function _(){}function w(n){function t(){for(var t,r=e,i=-1,u=r.length;++i<u;)(t=r[i].on)&&t.apply(this,arguments);return n}var e=[],r=new c;return t.on=function(t,i){var u,o=r.get(t);return arguments.length<2?o&&o.on:(o&&(o.on=null,e=e.slice(0,u=e.indexOf(o)).concat(e.slice(u+1)),r.remove(t)),i&&e.push(r.set(t,{on:i})),n)},t}function S(){ao.event.preventDefault()}function k(){for(var n,t=ao.event;n=t.sourceEvent;)t=n;return t}function N(n){for(var t=new _,e=0,r=arguments.length;++e<r;)t[arguments[e]]=w(t);return t.of
 =function(e,r){return function(i){try{var u=i.sourceEvent=ao.event;i.target=n,ao.event=i,t[i.type].apply(e,r)}finally{ao.event=u}}},t}function E(n){return ko(n,Co),n}function A(n){return"function"==typeof n?n:function(){return No(n,this)}}function C(n){return"function"==typeof n?n:function(){return Eo(n,this)}}function z(n,t){function e(){this.removeAttribute(n)}function r(){this.removeAttributeNS(n.space,n.local)}function i(){this.setAttribute(n,t)}function u(){this.setAttributeNS(n.space,n.local,t)}function o(){var e=t.apply(this,arguments);null==e?this.removeAttribute(n):this.setAttribute(n,e)}function a(){var e=t.apply(this,arguments);null==e?this.removeAttributeNS(n.space,n.local):this.setAttributeNS(n.space,n.local,e)}return n=ao.ns.qualify(n),null==t?n.local?r:e:"function"==typeof t?n.local?a:o:n.local?u:i}function L(n){return n.trim().replace(/\s+/g," ")}function q(n){return new RegExp("(?:^|\\s+)"+ao.requote(n)+"(?:\\s+|$)","g")}function T(n){return(n+"").trim().split(/^|\s
 +/)}function R(n,t){function e(){for(var e=-1;++e<i;)n[e](this,t)}function r(){for(var e=-1,r=t.apply(this,arguments);++e<i;)n[e](this,r)}n=T(n).map(D);var i=n.length;return"function"==typeof t?r:e}function D(n){var t=q(n);return function(e,r){if(i=e.classList)return r?i.add(n):i.remove(n);var i=e.getAttribute("class")||"";r?(t.lastIndex=0,t.test(i)||e.setAttribute("class",L(i+" "+n))):e.setAttribute("class",L(i.replace(t," ")))}}function P(n,t,e){function r(){this.style.removeProperty(n)}function i(){this.style.setProperty(n,t,e)}function u(){var r=t.apply(this,arguments);null==r?this.style.removeProperty(n):this.style.setProperty(n,r,e)}return null==t?r:"function"==typeof t?u:i}function U(n,t){function e(){delete this[n]}function r(){this[n]=t}function i(){var e=t.apply(this,arguments);null==e?delete this[n]:this[n]=e}return null==t?e:"function"==typeof t?i:r}function j(n){function t(){var t=this.ownerDocument,e=this.namespaceURI;return e===zo&&t.documentElement.namespaceURI===zo?
 t.createElement(n):t.createElementNS(e,n)}function e(){return this.ownerDocument.createElementNS(n.space,n.local)}return"function"==typeof n?n:(n=ao.ns.qualify(n)).local?e:t}function F(){var n=this.parentNode;n&&n.removeChild(this)}function H(n){return{__data__:n}}function O(n){return function(){return Ao(this,n)}}function I(n){return arguments.length||(n=e),function(t,e){return t&&e?n(t.__data__,e.__data__):!t-!e}}function Y(n,t){for(var e=0,r=n.length;r>e;e++)for(var i,u=n[e],o=0,a=u.length;a>o;o++)(i=u[o])&&t(i,o,e);return n}function Z(n){return ko(n,qo),n}function V(n){var t,e;return function(r,i,u){var o,a=n[u].update,l=a.length;for(u!=e&&(e=u,t=0),i>=t&&(t=i+1);!(o=a[t])&&++t<l;);return o}}function X(n,t,e){function r(){var t=this[o];t&&(this.removeEventListener(n,t,t.$),delete this[o])}function i(){var i=l(t,co(arguments));r.call(this),this.addEventListener(n,this[o]=i,i.$=e),i._=t}function u(){var t,e=new RegExp("^__on([^.]+)"+ao.requote(n)+"$");for(var r in this)if(t=r.matc
 h(e)){var i=this[r];this.removeEventListener(t[1],i,i.$),delete this[r]}}var o="__on"+n,a=n.indexOf("."),l=$;a>0&&(n=n.slice(0,a));var c=To.get(n);return c&&(n=c,l=B),a?t?i:r:t?b:u}function $(n,t){return function(e){var r=ao.event;ao.event=e,t[0]=this.__data__;try{n.apply(this,t)}finally{ao.event=r}}}function B(n,t){var e=$(n,t);return function(n){var t=this,r=n.relatedTarget;r&&(r===t||8&r.compareDocumentPosition(t))||e.call(t,n)}}function W(e){var r=".dragsuppress-"+ ++Do,i="click"+r,u=ao.select(t(e)).on("touchmove"+r,S).on("dragstart"+r,S).on("selectstart"+r,S);if(null==Ro&&(Ro="onselectstart"in e?!1:x(e.style,"userSelect")),Ro){var o=n(e).style,a=o[Ro];o[Ro]="none"}return function(n){if(u.on(r,null),Ro&&(o[Ro]=a),n){var t=function(){u.on(i,null)};u.on(i,function(){S(),t()},!0),setTimeout(t,0)}}}function J(n,e){e.changedTouches&&(e=e.changedTouches[0]);var r=n.ownerSVGElement||n;if(r.createSVGPoint){var i=r.createSVGPoint();if(0>Po){var u=t(n);if(u.scrollX||u.scrollY){r=ao.select
 ("body").append("svg").style({position:"absolute",top:0,left:0,margin:0,padding:0,border:"none"},"important");var o=r[0][0].getScreenCTM();Po=!(o.f||o.e),r.remove()}}return Po?(i.x=e.pageX,i.y=e.pageY):(i.x=e.clientX,i.y=e.clientY),i=i.matrixTransform(n.getScreenCTM().inverse()),[i.x,i.y]}var a=n.getBoundingClientRect();return[e.clientX-a.left-n.clientLeft,e.clientY-a.top-n.clientTop]}function G(){return ao.event.changedTouches[0].identifier}function K(n){return n>0?1:0>n?-1:0}function Q(n,t,e){return(t[0]-n[0])*(e[1]-n[1])-(t[1]-n[1])*(e[0]-n[0])}function nn(n){return n>1?0:-1>n?Fo:Math.acos(n)}function tn(n){return n>1?Io:-1>n?-Io:Math.asin(n)}function en(n){return((n=Math.exp(n))-1/n)/2}function rn(n){return((n=Math.exp(n))+1/n)/2}function un(n){return((n=Math.exp(2*n))-1)/(n+1)}function on(n){return(n=Math.sin(n/2))*n}function an(){}function ln(n,t,e){return this instanceof ln?(this.h=+n,this.s=+t,void(this.l=+e)):arguments.length<2?n instanceof ln?new ln(n.h,n.s,n.l):_n(""+n,wn
 ,ln):new ln(n,t,e)}function cn(n,t,e){function r(n){return n>360?n-=360:0>n&&(n+=360),60>n?u+(o-u)*n/60:180>n?o:240>n?u+(o-u)*(240-n)/60:u}function i(n){return Math.round(255*r(n))}var u,o;return n=isNaN(n)?0:(n%=360)<0?n+360:n,t=isNaN(t)?0:0>t?0:t>1?1:t,e=0>e?0:e>1?1:e,o=.5>=e?e*(1+t):e+t-e*t,u=2*e-o,new mn(i(n+120),i(n),i(n-120))}function fn(n,t,e){return this instanceof fn?(this.h=+n,this.c=+t,void(this.l=+e)):arguments.length<2?n instanceof fn?new fn(n.h,n.c,n.l):n instanceof hn?gn(n.l,n.a,n.b):gn((n=Sn((n=ao.rgb(n)).r,n.g,n.b)).l,n.a,n.b):new fn(n,t,e)}function sn(n,t,e){return isNaN(n)&&(n=0),isNaN(t)&&(t=0),new hn(e,Math.cos(n*=Yo)*t,Math.sin(n)*t)}function hn(n,t,e){return this instanceof hn?(this.l=+n,this.a=+t,void(this.b=+e)):arguments.length<2?n instanceof hn?new hn(n.l,n.a,n.b):n instanceof fn?sn(n.h,n.c,n.l):Sn((n=mn(n)).r,n.g,n.b):new hn(n,t,e)}function pn(n,t,e){var r=(n+16)/116,i=r+t/500,u=r-e/200;return i=vn(i)*na,r=vn(r)*ta,u=vn(u)*ea,new mn(yn(3.2404542*i-1.53713
 85*r-.4985314*u),yn(-.969266*i+1.8760108*r+.041556*u),yn(.0556434*i-.2040259*r+1.0572252*u))}function gn(n,t,e){return n>0?new fn(Math.atan2(e,t)*Zo,Math.sqrt(t*t+e*e),n):new fn(NaN,NaN,n)}function vn(n){return n>.206893034?n*n*n:(n-4/29)/7.787037}function dn(n){return n>.008856?Math.pow(n,1/3):7.787037*n+4/29}function yn(n){return Math.round(255*(.00304>=n?12.92*n:1.055*Math.pow(n,1/2.4)-.055))}function mn(n,t,e){return this instanceof mn?(this.r=~~n,this.g=~~t,void(this.b=~~e)):arguments.length<2?n instanceof mn?new mn(n.r,n.g,n.b):_n(""+n,mn,cn):new mn(n,t,e)}function Mn(n){return new mn(n>>16,n>>8&255,255&n)}function xn(n){return Mn(n)+""}function bn(n){return 16>n?"0"+Math.max(0,n).toString(16):Math.min(255,n).toString(16)}function _n(n,t,e){var r,i,u,o=0,a=0,l=0;if(r=/([a-z]+)\((.*)\)/.exec(n=n.toLowerCase()))switch(i=r[2].split(","),r[1]){case"hsl":return e(parseFloat(i[0]),parseFloat(i[1])/100,parseFloat(i[2])/100);case"rgb":return t(Nn(i[0]),Nn(i[1]),Nn(i[2]))}return(u=ua.g
 et(n))?t(u.r,u.g,u.b):(null==n||"#"!==n.charAt(0)||isNaN(u=parseInt(n.slice(1),16))||(4===n.length?(o=(3840&u)>>4,o=o>>4|o,a=240&u,a=a>>4|a,l=15&u,l=l<<4|l):7===n.length&&(o=(16711680&u)>>16,a=(65280&u)>>8,l=255&u)),t(o,a,l))}function wn(n,t,e){var r,i,u=Math.min(n/=255,t/=255,e/=255),o=Math.max(n,t,e),a=o-u,l=(o+u)/2;return a?(i=.5>l?a/(o+u):a/(2-o-u),r=n==o?(t-e)/a+(e>t?6:0):t==o?(e-n)/a+2:(n-t)/a+4,r*=60):(r=NaN,i=l>0&&1>l?0:r),new ln(r,i,l)}function Sn(n,t,e){n=kn(n),t=kn(t),e=kn(e);var r=dn((.4124564*n+.3575761*t+.1804375*e)/na),i=dn((.2126729*n+.7151522*t+.072175*e)/ta),u=dn((.0193339*n+.119192*t+.9503041*e)/ea);return hn(116*i-16,500*(r-i),200*(i-u))}function kn(n){return(n/=255)<=.04045?n/12.92:Math.pow((n+.055)/1.055,2.4)}function Nn(n){var t=parseFloat(n);return"%"===n.charAt(n.length-1)?Math.round(2.55*t):t}function En(n){return"function"==typeof n?n:function(){return n}}function An(n){return function(t,e,r){return 2===arguments.length&&"function"==typeof e&&(r=e,e=null),
 Cn(t,e,n,r)}}function Cn(n,t,e,r){function i(){var n,t=l.status;if(!t&&Ln(l)||t>=200&&300>t||304===t){try{n=e.call(u,l)}catch(r){return void o.error.call(u,r)}o.load.call(u,n)}else o.error.call(u,l)}var u={},o=ao.dispatch("beforesend","progress","load","error"),a={},l=new XMLHttpRequest,c=null;return!this.XDomainRequest||"withCredentials"in l||!/^(http(s)?:)?\/\//.test(n)||(l=new XDomainRequest),"onload"in l?l.onload=l.onerror=i:l.onreadystatechange=function(){l.readyState>3&&i()},l.onprogress=function(n){var t=ao.event;ao.event=n;try{o.progress.call(u,l)}finally{ao.event=t}},u.header=function(n,t){return n=(n+"").toLowerCase(),arguments.length<2?a[n]:(null==t?delete a[n]:a[n]=t+"",u)},u.mimeType=function(n){return arguments.length?(t=null==n?null:n+"",u):t},u.responseType=function(n){return arguments.length?(c=n,u):c},u.response=function(n){return e=n,u},["get","post"].forEach(function(n){u[n]=function(){return u.send.apply(u,[n].concat(co(arguments)))}}),u.send=function(e,r,i){if(
 2===arguments.length&&"function"==typeof r&&(i=r,r=null),l.open(e,n,!0),null==t||"accept"in a||(a.accept=t+",*/*"),l.setRequestHeader)for(var f in a)l.setRequestHeader(f,a[f]);return null!=t&&l.overrideMimeType&&l.overrideMimeType(t),null!=c&&(l.responseType=c),null!=i&&u.on("error",i).on("load",function(n){i(null,n)}),o.beforesend.call(u,l),l.send(null==r?null:r),u},u.abort=function(){return l.abort(),u},ao.rebind(u,o,"on"),null==r?u:u.get(zn(r))}function zn(n){return 1===n.length?function(t,e){n(null==t?e:null)}:n}function Ln(n){var t=n.responseType;return t&&"text"!==t?n.response:n.responseText}function qn(n,t,e){var r=arguments.length;2>r&&(t=0),3>r&&(e=Date.now());var i=e+t,u={c:n,t:i,n:null};return aa?aa.n=u:oa=u,aa=u,la||(ca=clearTimeout(ca),la=1,fa(Tn)),u}function Tn(){var n=Rn(),t=Dn()-n;t>24?(isFinite(t)&&(clearTimeout(ca),ca=setTimeout(Tn,t)),la=0):(la=1,fa(Tn))}function Rn(){for(var n=Date.now(),t=oa;t;)n>=t.t&&t.c(n-t.t)&&(t.c=null),t=t.n;return n}function Dn(){for(var 
 n,t=oa,e=1/0;t;)t.c?(t.t<e&&(e=t.t),t=(n=t).n):t=n?n.n=t.n:oa=t.n;return aa=n,e}function Pn(n,t){return t-(n?Math.ceil(Math.log(n)/Math.LN10):1)}function Un(n,t){var e=Math.pow(10,3*xo(8-t));return{scale:t>8?function(n){return n/e}:function(n){return n*e},symbol:n}}function jn(n){var t=n.decimal,e=n.thousands,r=n.grouping,i=n.currency,u=r&&e?function(n,t){for(var i=n.length,u=[],o=0,a=r[0],l=0;i>0&&a>0&&(l+a+1>t&&(a=Math.max(1,t-l)),u.push(n.substring(i-=a,i+a)),!((l+=a+1)>t));)a=r[o=(o+1)%r.length];return u.reverse().join(e)}:m;return function(n){var e=ha.exec(n),r=e[1]||" ",o=e[2]||">",a=e[3]||"-",l=e[4]||"",c=e[5],f=+e[6],s=e[7],h=e[8],p=e[9],g=1,v="",d="",y=!1,m=!0;switch(h&&(h=+h.substring(1)),(c||"0"===r&&"="===o)&&(c=r="0",o="="),p){case"n":s=!0,p="g";break;case"%":g=100,d="%",p="f";break;case"p":g=100,d="%",p="r";break;case"b":case"o":case"x":case"X":"#"===l&&(v="0"+p.toLowerCase());case"c":m=!1;case"d":y=!0,h=0;break;case"s":g=-1,p="r"}"$"===l&&(v=i[0],d=i[1]),"r"!=p||h||(p
 ="g"),null!=h&&("g"==p?h=Math.max(1,Math.min(21,h)):"e"!=p&&"f"!=p||(h=Math.max(0,Math.min(20,h)))),p=pa.get(p)||Fn;var M=c&&s;return function(n){var e=d;if(y&&n%1)return"";var i=0>n||0===n&&0>1/n?(n=-n,"-"):"-"===a?"":a;if(0>g){var l=ao.formatPrefix(n,h);n=l.scale(n),e=l.symbol+d}else n*=g;n=p(n,h);var x,b,_=n.lastIndexOf(".");if(0>_){var w=m?n.lastIndexOf("e"):-1;0>w?(x=n,b=""):(x=n.substring(0,w),b=n.substring(w))}else x=n.substring(0,_),b=t+n.substring(_+1);!c&&s&&(x=u(x,1/0));var S=v.length+x.length+b.length+(M?0:i.length),k=f>S?new Array(S=f-S+1).join(r):"";return M&&(x=u(k+x,k.length?f-b.length:1/0)),i+=v,n=x+b,("<"===o?i+n+k:">"===o?k+i+n:"^"===o?k.substring(0,S>>=1)+i+n+k.substring(S):i+(M?n:k+n))+e}}}function Fn(n){return n+""}function Hn(){this._=new Date(arguments.length>1?Date.UTC.apply(this,arguments):arguments[0])}function On(n,t,e){function r(t){var e=n(t),r=u(e,1);return r-t>t-e?e:r}function i(e){return t(e=n(new va(e-1)),1),e}function u(n,e){return t(n=new va(+n),e
 ),n}function o(n,r,u){var o=i(n),a=[];if(u>1)for(;r>o;)e(o)%u||a.push(new Date(+o)),t(o,1);else for(;r>o;)a.push(new Date(+o)),t(o,1);return a}function a(n,t,e){try{va=Hn;var r=new Hn;return r._=n,o(r,t,e)}finally{va=Date}}n.floor=n,n.round=r,n.ceil=i,n.offset=u,n.range=o;var l=n.utc=In(n);return l.floor=l,l.round=In(r),l.ceil=In(i),l.offset=In(u),l.range=a,n}function In(n){return function(t,e){try{va=Hn;var r=new Hn;return r._=t,n(r,e)._}finally{va=Date}}}function Yn(n){function t(n){function t(t){for(var e,i,u,o=[],a=-1,l=0;++a<r;)37===n.charCodeAt(a)&&(o.push(n.slice(l,a)),null!=(i=ya[e=n.charAt(++a)])&&(e=n.charAt(++a)),(u=A[e])&&(e=u(t,null==i?"e"===e?" ":"0":i)),o.push(e),l=a+1);return o.push(n.slice(l,a)),o.join("")}var r=n.length;return t.parse=function(t){var r={y:1900,m:0,d:1,H:0,M:0,S:0,L:0,Z:null},i=e(r,n,t,0);if(i!=t.length)return null;"p"in r&&(r.H=r.H%12+12*r.p);var u=null!=r.Z&&va!==Hn,o=new(u?Hn:va);return"j"in r?o.setFullYear(r.y,0,r.j):"W"in r||"U"in r?("w"in r||(
 r.w="W"in r?1:0),o.setFullYear(r.y,0,1),o.setFullYear(r.y,0,"W"in r?(r.w+6)%7+7*r.W-(o.getDay()+5)%7:r.w+7*r.U-(o.getDay()+6)%7)):o.setFullYear(r.y,r.m,r.d),o.setHours(r.H+(r.Z/100|0),r.M+r.Z%100,r.S,r.L),u?o._:o},t.toString=function(){return n},t}function e(n,t,e,r){for(var i,u,o,a=0,l=t.length,c=e.length;l>a;){if(r>=c)return-1;if(i=t.charCodeAt(a++),37===i){if(o=t.charAt(a++),u=C[o in ya?t.charAt(a++):o],!u||(r=u(n,e,r))<0)return-1}else if(i!=e.charCodeAt(r++))return-1}return r}function r(n,t,e){_.lastIndex=0;var r=_.exec(t.slice(e));return r?(n.w=w.get(r[0].toLowerCase()),e+r[0].length):-1}function i(n,t,e){x.lastIndex=0;var r=x.exec(t.slice(e));return r?(n.w=b.get(r[0].toLowerCase()),e+r[0].length):-1}function u(n,t,e){N.lastIndex=0;var r=N.exec(t.slice(e));return r?(n.m=E.get(r[0].toLowerCase()),e+r[0].length):-1}function o(n,t,e){S.lastIndex=0;var r=S.exec(t.slice(e));return r?(n.m=k.get(r[0].toLowerCase()),e+r[0].length):-1}function a(n,t,r){return e(n,A.c.toString(),t,r)}fun
 ction l(n,t,r){return e(n,A.x.toString(),t,r)}function c(n,t,r){return e(n,A.X.toString(),t,r)}function f(n,t,e){var r=M.get(t.slice(e,e+=2).toLowerCase());return null==r?-1:(n.p=r,e)}var s=n.dateTime,h=n.date,p=n.time,g=n.periods,v=n.days,d=n.shortDays,y=n.months,m=n.shortMonths;t.utc=function(n){function e(n){try{va=Hn;var t=new va;return t._=n,r(t)}finally{va=Date}}var r=t(n);return e.parse=function(n){try{va=Hn;var t=r.parse(n);return t&&t._}finally{va=Date}},e.toString=r.toString,e},t.multi=t.utc.multi=ct;var M=ao.map(),x=Vn(v),b=Xn(v),_=Vn(d),w=Xn(d),S=Vn(y),k=Xn(y),N=Vn(m),E=Xn(m);g.forEach(function(n,t){M.set(n.toLowerCase(),t)});var A={a:function(n){return d[n.getDay()]},A:function(n){return v[n.getDay()]},b:function(n){return m[n.getMonth()]},B:function(n){return y[n.getMonth()]},c:t(s),d:function(n,t){return Zn(n.getDate(),t,2)},e:function(n,t){return Zn(n.getDate(),t,2)},H:function(n,t){return Zn(n.getHours(),t,2)},I:function(n,t){return Zn(n.getHours()%12||12,t,2)},j:fu
 nction(n,t){return Zn(1+ga.dayOfYear(n),t,3)},L:function(n,t){return Zn(n.getMilliseconds(),t,3)},m:function(n,t){return Zn(n.getMonth()+1,t,2)},M:function(n,t){return Zn(n.getMinutes(),t,2)},p:function(n){return g[+(n.getHours()>=12)]},S:function(n,t){return Zn(n.getSeconds(),t,2)},U:function(n,t){return Zn(ga.sundayOfYear(n),t,2)},w:function(n){return n.getDay()},W:function(n,t){return Zn(ga.mondayOfYear(n),t,2)},x:t(h),X:t(p),y:function(n,t){return Zn(n.getFullYear()%100,t,2)},Y:function(n,t){return Zn(n.getFullYear()%1e4,t,4)},Z:at,"%":function(){return"%"}},C={a:r,A:i,b:u,B:o,c:a,d:tt,e:tt,H:rt,I:rt,j:et,L:ot,m:nt,M:it,p:f,S:ut,U:Bn,w:$n,W:Wn,x:l,X:c,y:Gn,Y:Jn,Z:Kn,"%":lt};return t}function Zn(n,t,e){var r=0>n?"-":"",i=(r?-n:n)+"",u=i.length;return r+(e>u?new Array(e-u+1).join(t)+i:i)}function Vn(n){return new RegExp("^(?:"+n.map(ao.requote).join("|")+")","i")}function Xn(n){for(var t=new c,e=-1,r=n.length;++e<r;)t.set(n[e].toLowerCase(),e);return t}function $n(n,t,e){ma.lastIn
 dex=0;var r=ma.exec(t.slice(e,e+1));return r?(n.w=+r[0],e+r[0].length):-1}function Bn(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e));return r?(n.U=+r[0],e+r[0].length):-1}function Wn(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e));return r?(n.W=+r[0],e+r[0].length):-1}function Jn(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+4));return r?(n.y=+r[0],e+r[0].length):-1}function Gn(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.y=Qn(+r[0]),e+r[0].length):-1}function Kn(n,t,e){return/^[+-]\d{4}$/.test(t=t.slice(e,e+5))?(n.Z=-t,e+5):-1}function Qn(n){return n+(n>68?1900:2e3)}function nt(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.m=r[0]-1,e+r[0].length):-1}function tt(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.d=+r[0],e+r[0].length):-1}function et(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+3));return r?(n.j=+r[0],e+r[0].length):-1}function rt(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.H=+r[0],e+r[0].len
 gth):-1}function it(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.M=+r[0],e+r[0].length):-1}function ut(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.S=+r[0],e+r[0].length):-1}function ot(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+3));return r?(n.L=+r[0],e+r[0].length):-1}function at(n){var t=n.getTimezoneOffset(),e=t>0?"-":"+",r=xo(t)/60|0,i=xo(t)%60;return e+Zn(r,"0",2)+Zn(i,"0",2)}function lt(n,t,e){Ma.lastIndex=0;var r=Ma.exec(t.slice(e,e+1));return r?e+r[0].length:-1}function ct(n){for(var t=n.length,e=-1;++e<t;)n[e][0]=this(n[e][0]);return function(t){for(var e=0,r=n[e];!r[1](t);)r=n[++e];return r[0](t)}}function ft(){}function st(n,t,e){var r=e.s=n+t,i=r-n,u=r-i;e.t=n-u+(t-i)}function ht(n,t){n&&wa.hasOwnProperty(n.type)&&wa[n.type](n,t)}function pt(n,t,e){var r,i=-1,u=n.length-e;for(t.lineStart();++i<u;)r=n[i],t.point(r[0],r[1],r[2]);t.lineEnd()}function gt(n,t){var e=-1,r=n.length;for(t.polygonStart();++e<r;)pt(n[e],t,1);t.polygonEnd
 ()}function vt(){function n(n,t){n*=Yo,t=t*Yo/2+Fo/4;var e=n-r,o=e>=0?1:-1,a=o*e,l=Math.cos(t),c=Math.sin(t),f=u*c,s=i*l+f*Math.cos(a),h=f*o*Math.sin(a);ka.add(Math.atan2(h,s)),r=n,i=l,u=c}var t,e,r,i,u;Na.point=function(o,a){Na.point=n,r=(t=o)*Yo,i=Math.cos(a=(e=a)*Yo/2+Fo/4),u=Math.sin(a)},Na.lineEnd=function(){n(t,e)}}function dt(n){var t=n[0],e=n[1],r=Math.cos(e);return[r*Math.cos(t),r*Math.sin(t),Math.sin(e)]}function yt(n,t){return n[0]*t[0]+n[1]*t[1]+n[2]*t[2]}function mt(n,t){return[n[1]*t[2]-n[2]*t[1],n[2]*t[0]-n[0]*t[2],n[0]*t[1]-n[1]*t[0]]}function Mt(n,t){n[0]+=t[0],n[1]+=t[1],n[2]+=t[2]}function xt(n,t){return[n[0]*t,n[1]*t,n[2]*t]}function bt(n){var t=Math.sqrt(n[0]*n[0]+n[1]*n[1]+n[2]*n[2]);n[0]/=t,n[1]/=t,n[2]/=t}function _t(n){return[Math.atan2(n[1],n[0]),tn(n[2])]}function wt(n,t){return xo(n[0]-t[0])<Uo&&xo(n[1]-t[1])<Uo}function St(n,t){n*=Yo;var e=Math.cos(t*=Yo);kt(e*Math.cos(n),e*Math.sin(n),Math.sin(t))}function kt(n,t,e){++Ea,Ca+=(n-Ca)/Ea,za+=(t-za)/Ea,La+=
 (e-La)/Ea}function Nt(){function n(n,i){n*=Yo;var u=Math.cos(i*=Yo),o=u*Math.cos(n),a=u*Math.sin(n),l=Math.sin(i),c=Math.atan2(Math.sqrt((c=e*l-r*a)*c+(c=r*o-t*l)*c+(c=t*a-e*o)*c),t*o+e*a+r*l);Aa+=c,qa+=c*(t+(t=o)),Ta+=c*(e+(e=a)),Ra+=c*(r+(r=l)),kt(t,e,r)}var t,e,r;ja.point=function(i,u){i*=Yo;var o=Math.cos(u*=Yo);t=o*Math.cos(i),e=o*Math.sin(i),r=Math.sin(u),ja.point=n,kt(t,e,r)}}function Et(){ja.point=St}function At(){function n(n,t){n*=Yo;var e=Math.cos(t*=Yo),o=e*Math.cos(n),a=e*Math.sin(n),l=Math.sin(t),c=i*l-u*a,f=u*o-r*l,s=r*a-i*o,h=Math.sqrt(c*c+f*f+s*s),p=r*o+i*a+u*l,g=h&&-nn(p)/h,v=Math.atan2(h,p);Da+=g*c,Pa+=g*f,Ua+=g*s,Aa+=v,qa+=v*(r+(r=o)),Ta+=v*(i+(i=a)),Ra+=v*(u+(u=l)),kt(r,i,u)}var t,e,r,i,u;ja.point=function(o,a){t=o,e=a,ja.point=n,o*=Yo;var l=Math.cos(a*=Yo);r=l*Math.cos(o),i=l*Math.sin(o),u=Math.sin(a),kt(r,i,u)},ja.lineEnd=function(){n(t,e),ja.lineEnd=Et,ja.point=St}}function Ct(n,t){function e(e,r){return e=n(e,r),t(e[0],e[1])}return n.invert&&t.invert&&(e.inv
 ert=function(e,r){return e=t.invert(e,r),e&&n.invert(e[0],e[1])}),e}function zt(){return!0}function Lt(n,t,e,r,i){var u=[],o=[];if(n.forEach(function(n){if(!((t=n.length-1)<=0)){var t,e=n[0],r=n[t];if(wt(e,r)){i.lineStart();for(var a=0;t>a;++a)i.point((e=n[a])[0],e[1]);return void i.lineEnd()}var l=new Tt(e,n,null,!0),c=new Tt(e,null,l,!1);l.o=c,u.push(l),o.push(c),l=new Tt(r,n,null,!1),c=new Tt(r,null,l,!0),l.o=c,u.push(l),o.push(c)}}),o.sort(t),qt(u),qt(o),u.length){for(var a=0,l=e,c=o.length;c>a;++a)o[a].e=l=!l;for(var f,s,h=u[0];;){for(var p=h,g=!0;p.v;)if((p=p.n)===h)return;f=p.z,i.lineStart();do{if(p.v=p.o.v=!0,p.e){if(g)for(var a=0,c=f.length;c>a;++a)i.point((s=f[a])[0],s[1]);else r(p.x,p.n.x,1,i);p=p.n}else{if(g){f=p.p.z;for(var a=f.length-1;a>=0;--a)i.point((s=f[a])[0],s[1])}else r(p.x,p.p.x,-1,i);p=p.p}p=p.o,f=p.z,g=!g}while(!p.v);i.lineEnd()}}}function qt(n){if(t=n.length){for(var t,e,r=0,i=n[0];++r<t;)i.n=e=n[r],e.p=i,i=e;i.n=e=n[0],e.p=i}}function Tt(n,t,e,r){this.x=n,t
 his.z=t,this.o=e,this.e=r,this.v=!1,this.n=this.p=null}function Rt(n,t,e,r){return function(i,u){function o(t,e){var r=i(t,e);n(t=r[0],e=r[1])&&u.point(t,e)}function a(n,t){var e=i(n,t);d.point(e[0],e[1])}function l(){m.point=a,d.lineStart()}function c(){m.point=o,d.lineEnd()}function f(n,t){v.push([n,t]);var e=i(n,t);x.point(e[0],e[1])}function s(){x.lineStart(),v=[]}function h(){f(v[0][0],v[0][1]),x.lineEnd();var n,t=x.clean(),e=M.buffer(),r=e.length;if(v.pop(),g.push(v),v=null,r)if(1&t){n=e[0];var i,r=n.length-1,o=-1;if(r>0){for(b||(u.polygonStart(),b=!0),u.lineStart();++o<r;)u.point((i=n[o])[0],i[1]);u.lineEnd()}}else r>1&&2&t&&e.push(e.pop().concat(e.shift())),p.push(e.filter(Dt))}var p,g,v,d=t(u),y=i.invert(r[0],r[1]),m={point:o,lineStart:l,lineEnd:c,polygonStart:function(){m.point=f,m.lineStart=s,m.lineEnd=h,p=[],g=[]},polygonEnd:function(){m.point=o,m.lineStart=l,m.lineEnd=c,p=ao.merge(p);var n=Ot(y,g);p.length?(b||(u.polygonStart(),b=!0),Lt(p,Ut,n,e,u)):n&&(b||(u.polygonSta
 rt(),b=!0),u.lineStart(),e(null,null,1,u),u.lineEnd()),b&&(u.polygonEnd(),b=!1),p=g=null},sphere:function(){u.polygonStart(),u.lineStart(),e(null,null,1,u),u.lineEnd(),u.polygonEnd()}},M=Pt(),x=t(M),b=!1;return m}}function Dt(n){return n.length>1}function Pt(){var n,t=[];return{lineStart:function(){t.push(n=[])},point:function(t,e){n.push([t,e])},lineEnd:b,buffer:function(){var e=t;return t=[],n=null,e},rejoin:function(){t.length>1&&t.push(t.pop().concat(t.shift()))}}}function Ut(n,t){return((n=n.x)[0]<0?n[1]-Io-Uo:Io-n[1])-((t=t.x)[0]<0?t[1]-Io-Uo:Io-t[1])}function jt(n){var t,e=NaN,r=NaN,i=NaN;return{lineStart:function(){n.lineStart(),t=1},point:function(u,o){var a=u>0?Fo:-Fo,l=xo(u-e);xo(l-Fo)<Uo?(n.point(e,r=(r+o)/2>0?Io:-Io),n.point(i,r),n.lineEnd(),n.lineStart(),n.point(a,r),n.point(u,r),t=0):i!==a&&l>=Fo&&(xo(e-i)<Uo&&(e-=i*Uo),xo(u-a)<Uo&&(u-=a*Uo),r=Ft(e,r,u,o),n.point(i,r),n.lineEnd(),n.lineStart(),n.point(a,r),t=0),n.point(e=u,r=o),i=a},lineEnd:function(){n.lineEnd(),e=r=
 NaN},clean:function(){return 2-t}}}function Ft(n,t,e,r){var i,u,o=Math.sin(n-e);return xo(o)>Uo?Math.atan((Math.sin(t)*(u=Math.cos(r))*Math.sin(e)-Math.sin(r)*(i=Math.cos(t))*Math.sin(n))/(i*u*o)):(t+r)/2}function Ht(n,t,e,r){var i;if(null==n)i=e*Io,r.point(-Fo,i),r.point(0,i),r.point(Fo,i),r.point(Fo,0),r.point(Fo,-i),r.point(0,-i),r.point(-Fo,-i),r.point(-Fo,0),r.point(-Fo,i);else if(xo(n[0]-t[0])>Uo){var u=n[0]<t[0]?Fo:-Fo;i=e*u/2,r.point(-u,i),r.point(0,i),r.point(u,i)}else r.point(t[0],t[1])}function Ot(n,t){var e=n[0],r=n[1],i=[Math.sin(e),-Math.cos(e),0],u=0,o=0;ka.reset();for(var a=0,l=t.length;l>a;++a){var c=t[a],f=c.length;if(f)for(var s=c[0],h=s[0],p=s[1]/2+Fo/4,g=Math.sin(p),v=Math.cos(p),d=1;;){d===f&&(d=0),n=c[d];var y=n[0],m=n[1]/2+Fo/4,M=Math.sin(m),x=Math.cos(m),b=y-h,_=b>=0?1:-1,w=_*b,S=w>Fo,k=g*M;if(ka.add(Math.atan2(k*_*Math.sin(w),v*x+k*Math.cos(w))),u+=S?b+_*Ho:b,S^h>=e^y>=e){var N=mt(dt(s),dt(n));bt(N);var E=mt(i,N);bt(E);var A=(S^b>=0?-1:1)*tn(E[2]);(r>A||r==
 =A&&(N[0]||N[1]))&&(o+=S^b>=0?1:-1)}if(!d++)break;h=y,g=M,v=x,s=n}}return(-Uo>u||Uo>u&&-Uo>ka)^1&o}function It(n){function t(n,t){return Math.cos(n)*Math.cos(t)>u}function e(n){var e,u,l,c,f;return{lineStart:function(){c=l=!1,f=1},point:function(s,h){var p,g=[s,h],v=t(s,h),d=o?v?0:i(s,h):v?i(s+(0>s?Fo:-Fo),h):0;if(!e&&(c=l=v)&&n.lineStart(),v!==l&&(p=r(e,g),(wt(e,p)||wt(g,p))&&(g[0]+=Uo,g[1]+=Uo,v=t(g[0],g[1]))),v!==l)f=0,v?(n.lineStart(),p=r(g,e),n.point(p[0],p[1])):(p=r(e,g),n.point(p[0],p[1]),n.lineEnd()),e=p;else if(a&&e&&o^v){var y;d&u||!(y=r(g,e,!0))||(f=0,o?(n.lineStart(),n.point(y[0][0],y[0][1]),n.point(y[1][0],y[1][1]),n.lineEnd()):(n.point(y[1][0],y[1][1]),n.lineEnd(),n.lineStart(),n.point(y[0][0],y[0][1])))}!v||e&&wt(e,g)||n.point(g[0],g[1]),e=g,l=v,u=d},lineEnd:function(){l&&n.lineEnd(),e=null},clean:function(){return f|(c&&l)<<1}}}function r(n,t,e){var r=dt(n),i=dt(t),o=[1,0,0],a=mt(r,i),l=yt(a,a),c=a[0],f=l-c*c;if(!f)return!e&&n;var s=u*l/f,h=-u*c/f,p=mt(o,a),g=xt(o,s)
 ,v=xt(a,h);Mt(g,v);var d=p,y=yt(g,d),m=yt(d,d),M=y*y-m*(yt(g,g)-1);if(!(0>M)){var x=Math.sqrt(M),b=xt(d,(-y-x)/m);if(Mt(b,g),b=_t(b),!e)return b;var _,w=n[0],S=t[0],k=n[1],N=t[1];w>S&&(_=w,w=S,S=_);var E=S-w,A=xo(E-Fo)<Uo,C=A||Uo>E;if(!A&&k>N&&(_=k,k=N,N=_),C?A?k+N>0^b[1]<(xo(b[0]-w)<Uo?k:N):k<=b[1]&&b[1]<=N:E>Fo^(w<=b[0]&&b[0]<=S)){var z=xt(d,(-y+x)/m);return Mt(z,g),[b,_t(z)]}}}function i(t,e){var r=o?n:Fo-n,i=0;return-r>t?i|=1:t>r&&(i|=2),-r>e?i|=4:e>r&&(i|=8),i}var u=Math.cos(n),o=u>0,a=xo(u)>Uo,l=ve(n,6*Yo);return Rt(t,e,l,o?[0,-n]:[-Fo,n-Fo])}function Yt(n,t,e,r){return function(i){var u,o=i.a,a=i.b,l=o.x,c=o.y,f=a.x,s=a.y,h=0,p=1,g=f-l,v=s-c;if(u=n-l,g||!(u>0)){if(u/=g,0>g){if(h>u)return;p>u&&(p=u)}else if(g>0){if(u>p)return;u>h&&(h=u)}if(u=e-l,g||!(0>u)){if(u/=g,0>g){if(u>p)return;u>h&&(h=u)}else if(g>0){if(h>u)return;p>u&&(p=u)}if(u=t-c,v||!(u>0)){if(u/=v,0>v){if(h>u)return;p>u&&(p=u)}else if(v>0){if(u>p)return;u>h&&(h=u)}if(u=r-c,v||!(0>u)){if(u/=v,0>v){if(u>p)return;u>h&&
 (h=u)}else if(v>0){if(h>u)return;p>u&&(p=u)}return h>0&&(i.a={x:l+h*g,y:c+h*v}),1>p&&(i.b={x:l+p*g,y:c+p*v}),i}}}}}}function Zt(n,t,e,r){function i(r,i){return xo(r[0]-n)<Uo?i>0?0:3:xo(r[0]-e)<Uo?i>0?2:1:xo(r[1]-t)<Uo?i>0?1:0:i>0?3:2}function u(n,t){return o(n.x,t.x)}function o(n,t){var e=i(n,1),r=i(t,1);return e!==r?e-r:0===e?t[1]-n[1]:1===e?n[0]-t[0]:2===e?n[1]-t[1]:t[0]-n[0]}return function(a){function l(n){for(var t=0,e=d.length,r=n[1],i=0;e>i;++i)for(var u,o=1,a=d[i],l=a.length,c=a[0];l>o;++o)u=a[o],c[1]<=r?u[1]>r&&Q(c,u,n)>0&&++t:u[1]<=r&&Q(c,u,n)<0&&--t,c=u;return 0!==t}function c(u,a,l,c){var f=0,s=0;if(null==u||(f=i(u,l))!==(s=i(a,l))||o(u,a)<0^l>0){do c.point(0===f||3===f?n:e,f>1?r:t);while((f=(f+l+4)%4)!==s)}else c.point(a[0],a[1])}function f(i,u){return i>=n&&e>=i&&u>=t&&r>=u}function s(n,t){f(n,t)&&a.point(n,t)}function h(){C.point=g,d&&d.push(y=[]),S=!0,w=!1,b=_=NaN}function p(){v&&(g(m,M),x&&w&&E.rejoin(),v.push(E.buffer())),C.point=s,w&&a.lineEnd()}function g(n,t){n=
 Math.max(-Ha,Math.min(Ha,n)),t=Math.max(-Ha,Math.min(Ha,t));var e=f(n,t);if(d&&y.push([n,t]),S)m=n,M=t,x=e,S=!1,e&&(a.lineStart(),a.point(n,t));else if(e&&w)a.point(n,t);else{var r={a:{x:b,y:_},b:{x:n,y:t}};A(r)?(w||(a.lineStart(),a.point(r.a.x,r.a.y)),a.point(r.b.x,r.b.y),e||a.lineEnd(),k=!1):e&&(a.lineStart(),a.point(n,t),k=!1)}b=n,_=t,w=e}var v,d,y,m,M,x,b,_,w,S,k,N=a,E=Pt(),A=Yt(n,t,e,r),C={point:s,lineStart:h,lineEnd:p,polygonStart:function(){a=E,v=[],d=[],k=!0},polygonEnd:function(){a=N,v=ao.merge(v);var t=l([n,r]),e=k&&t,i=v.length;(e||i)&&(a.polygonStart(),e&&(a.lineStart(),c(null,null,1,a),a.lineEnd()),i&&Lt(v,u,t,c,a),a.polygonEnd()),v=d=y=null}};return C}}function Vt(n){var t=0,e=Fo/3,r=ae(n),i=r(t,e);return i.parallels=function(n){return arguments.length?r(t=n[0]*Fo/180,e=n[1]*Fo/180):[t/Fo*180,e/Fo*180]},i}function Xt(n,t){function e(n,t){var e=Math.sqrt(u-2*i*Math.sin(t))/i;return[e*Math.sin(n*=i),o-e*Math.cos(n)]}var r=Math.sin(n),i=(r+Math.sin(t))/2,u=1+r*(2*i-r),o=M
 ath.sqrt(u)/i;return e.invert=function(n,t){var e=o-t;return[Math.atan2(n,e)/i,tn((u-(n*n+e*e)*i*i)/(2*i))]},e}function $t(){function n(n,t){Ia+=i*n-r*t,r=n,i=t}var t,e,r,i;$a.point=function(u,o){$a.point=n,t=r=u,e=i=o},$a.lineEnd=function(){n(t,e)}}function Bt(n,t){Ya>n&&(Ya=n),n>Va&&(Va=n),Za>t&&(Za=t),t>Xa&&(Xa=t)}function Wt(){function n(n,t){o.push("M",n,",",t,u)}function t(n,t){o.push("M",n,",",t),a.point=e}function e(n,t){o.push("L",n,",",t)}function r(){a.point=n}function i(){o.push("Z")}var u=Jt(4.5),o=[],a={point:n,lineStart:function(){a.point=t},lineEnd:r,polygonStart:function(){a.lineEnd=i},polygonEnd:function(){a.lineEnd=r,a.point=n},pointRadius:function(n){return u=Jt(n),a},result:function(){if(o.length){var n=o.join("");return o=[],n}}};return a}function Jt(n){return"m0,"+n+"a"+n+","+n+" 0 1,1 0,"+-2*n+"a"+n+","+n+" 0 1,1 0,"+2*n+"z"}function Gt(n,t){Ca+=n,za+=t,++La}function Kt(){function n(n,r){var i=n-t,u=r-e,o=Math.sqrt(i*i+u*u);qa+=o*(t+n)/2,Ta+=o*(e+r)/2,Ra+=o,G
 t(t=n,e=r)}var t,e;Wa.point=function(r,i){Wa.point=n,Gt(t=r,e=i)}}function Qt(){Wa.point=Gt}function ne(){function n(n,t){var e=n-r,u=t-i,o=Math.sqrt(e*e+u*u);qa+=o*(r+n)/2,Ta+=o*(i+t)/2,Ra+=o,o=i*n-r*t,Da+=o*(r+n),Pa+=o*(i+t),Ua+=3*o,Gt(r=n,i=t)}var t,e,r,i;Wa.point=function(u,o){Wa.point=n,Gt(t=r=u,e=i=o)},Wa.lineEnd=function(){n(t,e)}}function te(n){function t(t,e){n.moveTo(t+o,e),n.arc(t,e,o,0,Ho)}function e(t,e){n.moveTo(t,e),a.point=r}function r(t,e){n.lineTo(t,e)}function i(){a.point=t}function u(){n.closePath()}var o=4.5,a={point:t,lineStart:function(){a.point=e},lineEnd:i,polygonStart:function(){a.lineEnd=u},polygonEnd:function(){a.lineEnd=i,a.point=t},pointRadius:function(n){return o=n,a},result:b};return a}function ee(n){function t(n){return(a?r:e)(n)}function e(t){return ue(t,function(e,r){e=n(e,r),t.point(e[0],e[1])})}function r(t){function e(e,r){e=n(e,r),t.point(e[0],e[1])}function r(){M=NaN,S.point=u,t.lineStart()}function u(e,r){var u=dt([e,r]),o=n(e,r);i(M,x,m,b,_,
 w,M=o[0],x=o[1],m=e,b=u[0],_=u[1],w=u[2],a,t),t.point(M,x)}function o(){S.point=e,t.lineEnd()}function l(){
-r(),S.point=c,S.lineEnd=f}function c(n,t){u(s=n,h=t),p=M,g=x,v=b,d=_,y=w,S.point=u}function f(){i(M,x,m,b,_,w,p,g,s,v,d,y,a,t),S.lineEnd=o,o()}var s,h,p,g,v,d,y,m,M,x,b,_,w,S={point:e,lineStart:r,lineEnd:o,polygonStart:function(){t.polygonStart(),S.lineStart=l},polygonEnd:function(){t.polygonEnd(),S.lineStart=r}};return S}function i(t,e,r,a,l,c,f,s,h,p,g,v,d,y){var m=f-t,M=s-e,x=m*m+M*M;if(x>4*u&&d--){var b=a+p,_=l+g,w=c+v,S=Math.sqrt(b*b+_*_+w*w),k=Math.asin(w/=S),N=xo(xo(w)-1)<Uo||xo(r-h)<Uo?(r+h)/2:Math.atan2(_,b),E=n(N,k),A=E[0],C=E[1],z=A-t,L=C-e,q=M*z-m*L;(q*q/x>u||xo((m*z+M*L)/x-.5)>.3||o>a*p+l*g+c*v)&&(i(t,e,r,a,l,c,A,C,N,b/=S,_/=S,w,d,y),y.point(A,C),i(A,C,N,b,_,w,f,s,h,p,g,v,d,y))}}var u=.5,o=Math.cos(30*Yo),a=16;return t.precision=function(n){return arguments.length?(a=(u=n*n)>0&&16,t):Math.sqrt(u)},t}function re(n){var t=ee(function(t,e){return n([t*Zo,e*Zo])});return function(n){return le(t(n))}}function ie(n){this.stream=n}function ue(n,t){return{point:t,sphere:functio
 n(){n.sphere()},lineStart:function(){n.lineStart()},lineEnd:function(){n.lineEnd()},polygonStart:function(){n.polygonStart()},polygonEnd:function(){n.polygonEnd()}}}function oe(n){return ae(function(){return n})()}function ae(n){function t(n){return n=a(n[0]*Yo,n[1]*Yo),[n[0]*h+l,c-n[1]*h]}function e(n){return n=a.invert((n[0]-l)/h,(c-n[1])/h),n&&[n[0]*Zo,n[1]*Zo]}function r(){a=Ct(o=se(y,M,x),u);var n=u(v,d);return l=p-n[0]*h,c=g+n[1]*h,i()}function i(){return f&&(f.valid=!1,f=null),t}var u,o,a,l,c,f,s=ee(function(n,t){return n=u(n,t),[n[0]*h+l,c-n[1]*h]}),h=150,p=480,g=250,v=0,d=0,y=0,M=0,x=0,b=Fa,_=m,w=null,S=null;return t.stream=function(n){return f&&(f.valid=!1),f=le(b(o,s(_(n)))),f.valid=!0,f},t.clipAngle=function(n){return arguments.length?(b=null==n?(w=n,Fa):It((w=+n)*Yo),i()):w},t.clipExtent=function(n){return arguments.length?(S=n,_=n?Zt(n[0][0],n[0][1],n[1][0],n[1][1]):m,i()):S},t.scale=function(n){return arguments.length?(h=+n,r()):h},t.translate=function(n){return argum
 ents.length?(p=+n[0],g=+n[1],r()):[p,g]},t.center=function(n){return arguments.length?(v=n[0]%360*Yo,d=n[1]%360*Yo,r()):[v*Zo,d*Zo]},t.rotate=function(n){return arguments.length?(y=n[0]%360*Yo,M=n[1]%360*Yo,x=n.length>2?n[2]%360*Yo:0,r()):[y*Zo,M*Zo,x*Zo]},ao.rebind(t,s,"precision"),function(){return u=n.apply(this,arguments),t.invert=u.invert&&e,r()}}function le(n){return ue(n,function(t,e){n.point(t*Yo,e*Yo)})}function ce(n,t){return[n,t]}function fe(n,t){return[n>Fo?n-Ho:-Fo>n?n+Ho:n,t]}function se(n,t,e){return n?t||e?Ct(pe(n),ge(t,e)):pe(n):t||e?ge(t,e):fe}function he(n){return function(t,e){return t+=n,[t>Fo?t-Ho:-Fo>t?t+Ho:t,e]}}function pe(n){var t=he(n);return t.invert=he(-n),t}function ge(n,t){function e(n,t){var e=Math.cos(t),a=Math.cos(n)*e,l=Math.sin(n)*e,c=Math.sin(t),f=c*r+a*i;return[Math.atan2(l*u-f*o,a*r-c*i),tn(f*u+l*o)]}var r=Math.cos(n),i=Math.sin(n),u=Math.cos(t),o=Math.sin(t);return e.invert=function(n,t){var e=Math.cos(t),a=Math.cos(n)*e,l=Math.sin(n)*e,c=Math
 .sin(t),f=c*u-l*o;return[Math.atan2(l*u+c*o,a*r+f*i),tn(f*r-a*i)]},e}function ve(n,t){var e=Math.cos(n),r=Math.sin(n);return function(i,u,o,a){var l=o*t;null!=i?(i=de(e,i),u=de(e,u),(o>0?u>i:i>u)&&(i+=o*Ho)):(i=n+o*Ho,u=n-.5*l);for(var c,f=i;o>0?f>u:u>f;f-=l)a.point((c=_t([e,-r*Math.cos(f),-r*Math.sin(f)]))[0],c[1])}}function de(n,t){var e=dt(t);e[0]-=n,bt(e);var r=nn(-e[1]);return((-e[2]<0?-r:r)+2*Math.PI-Uo)%(2*Math.PI)}function ye(n,t,e){var r=ao.range(n,t-Uo,e).concat(t);return function(n){return r.map(function(t){return[n,t]})}}function me(n,t,e){var r=ao.range(n,t-Uo,e).concat(t);return function(n){return r.map(function(t){return[t,n]})}}function Me(n){return n.source}function xe(n){return n.target}function be(n,t,e,r){var i=Math.cos(t),u=Math.sin(t),o=Math.cos(r),a=Math.sin(r),l=i*Math.cos(n),c=i*Math.sin(n),f=o*Math.cos(e),s=o*Math.sin(e),h=2*Math.asin(Math.sqrt(on(r-t)+i*o*on(e-n))),p=1/Math.sin(h),g=h?function(n){var t=Math.sin(n*=h)*p,e=Math.sin(h-n)*p,r=e*l+t*f,i=e*c+t*s
 ,o=e*u+t*a;return[Math.atan2(i,r)*Zo,Math.atan2(o,Math.sqrt(r*r+i*i))*Zo]}:function(){return[n*Zo,t*Zo]};return g.distance=h,g}function _e(){function n(n,i){var u=Math.sin(i*=Yo),o=Math.cos(i),a=xo((n*=Yo)-t),l=Math.cos(a);Ja+=Math.atan2(Math.sqrt((a=o*Math.sin(a))*a+(a=r*u-e*o*l)*a),e*u+r*o*l),t=n,e=u,r=o}var t,e,r;Ga.point=function(i,u){t=i*Yo,e=Math.sin(u*=Yo),r=Math.cos(u),Ga.point=n},Ga.lineEnd=function(){Ga.point=Ga.lineEnd=b}}function we(n,t){function e(t,e){var r=Math.cos(t),i=Math.cos(e),u=n(r*i);return[u*i*Math.sin(t),u*Math.sin(e)]}return e.invert=function(n,e){var r=Math.sqrt(n*n+e*e),i=t(r),u=Math.sin(i),o=Math.cos(i);return[Math.atan2(n*u,r*o),Math.asin(r&&e*u/r)]},e}function Se(n,t){function e(n,t){o>0?-Io+Uo>t&&(t=-Io+Uo):t>Io-Uo&&(t=Io-Uo);var e=o/Math.pow(i(t),u);return[e*Math.sin(u*n),o-e*Math.cos(u*n)]}var r=Math.cos(n),i=function(n){return Math.tan(Fo/4+n/2)},u=n===t?Math.sin(n):Math.log(r/Math.cos(t))/Math.log(i(t)/i(n)),o=r*Math.pow(i(n),u)/u;return u?(e.inver
 t=function(n,t){var e=o-t,r=K(u)*Math.sqrt(n*n+e*e);return[Math.atan2(n,e)/u,2*Math.atan(Math.pow(o/r,1/u))-Io]},e):Ne}function ke(n,t){function e(n,t){var e=u-t;return[e*Math.sin(i*n),u-e*Math.cos(i*n)]}var r=Math.cos(n),i=n===t?Math.sin(n):(r-Math.cos(t))/(t-n),u=r/i+n;return xo(i)<Uo?ce:(e.invert=function(n,t){var e=u-t;return[Math.atan2(n,e)/i,u-K(i)*Math.sqrt(n*n+e*e)]},e)}function Ne(n,t){return[n,Math.log(Math.tan(Fo/4+t/2))]}function Ee(n){var t,e=oe(n),r=e.scale,i=e.translate,u=e.clipExtent;return e.scale=function(){var n=r.apply(e,arguments);return n===e?t?e.clipExtent(null):e:n},e.translate=function(){var n=i.apply(e,arguments);return n===e?t?e.clipExtent(null):e:n},e.clipExtent=function(n){var o=u.apply(e,arguments);if(o===e){if(t=null==n){var a=Fo*r(),l=i();u([[l[0]-a,l[1]-a],[l[0]+a,l[1]+a]])}}else t&&(o=null);return o},e.clipExtent(null)}function Ae(n,t){return[Math.log(Math.tan(Fo/4+t/2)),-n]}function Ce(n){return n[0]}function ze(n){return n[1]}function Le(n){for(va
 r t=n.length,e=[0,1],r=2,i=2;t>i;i++){for(;r>1&&Q(n[e[r-2]],n[e[r-1]],n[i])<=0;)--r;e[r++]=i}return e.slice(0,r)}function qe(n,t){return n[0]-t[0]||n[1]-t[1]}function Te(n,t,e){return(e[0]-t[0])*(n[1]-t[1])<(e[1]-t[1])*(n[0]-t[0])}function Re(n,t,e,r){var i=n[0],u=e[0],o=t[0]-i,a=r[0]-u,l=n[1],c=e[1],f=t[1]-l,s=r[1]-c,h=(a*(l-c)-s*(i-u))/(s*o-a*f);return[i+h*o,l+h*f]}function De(n){var t=n[0],e=n[n.length-1];return!(t[0]-e[0]||t[1]-e[1])}function Pe(){rr(this),this.edge=this.site=this.circle=null}function Ue(n){var t=cl.pop()||new Pe;return t.site=n,t}function je(n){Be(n),ol.remove(n),cl.push(n),rr(n)}function Fe(n){var t=n.circle,e=t.x,r=t.cy,i={x:e,y:r},u=n.P,o=n.N,a=[n];je(n);for(var l=u;l.circle&&xo(e-l.circle.x)<Uo&&xo(r-l.circle.cy)<Uo;)u=l.P,a.unshift(l),je(l),l=u;a.unshift(l),Be(l);for(var c=o;c.circle&&xo(e-c.circle.x)<Uo&&xo(r-c.circle.cy)<Uo;)o=c.N,a.push(c),je(c),c=o;a.push(c),Be(c);var f,s=a.length;for(f=1;s>f;++f)c=a[f],l=a[f-1],nr(c.edge,l.site,c.site,i);l=a[0],c=a[s-
 1],c.edge=Ke(l.site,c.site,null,i),$e(l),$e(c)}function He(n){for(var t,e,r,i,u=n.x,o=n.y,a=ol._;a;)if(r=Oe(a,o)-u,r>Uo)a=a.L;else{if(i=u-Ie(a,o),!(i>Uo)){r>-Uo?(t=a.P,e=a):i>-Uo?(t=a,e=a.N):t=e=a;break}if(!a.R){t=a;break}a=a.R}var l=Ue(n);if(ol.insert(t,l),t||e){if(t===e)return Be(t),e=Ue(t.site),ol.insert(l,e),l.edge=e.edge=Ke(t.site,l.site),$e(t),void $e(e);if(!e)return void(l.edge=Ke(t.site,l.site));Be(t),Be(e);var c=t.site,f=c.x,s=c.y,h=n.x-f,p=n.y-s,g=e.site,v=g.x-f,d=g.y-s,y=2*(h*d-p*v),m=h*h+p*p,M=v*v+d*d,x={x:(d*m-p*M)/y+f,y:(h*M-v*m)/y+s};nr(e.edge,c,g,x),l.edge=Ke(c,n,null,x),e.edge=Ke(n,g,null,x),$e(t),$e(e)}}function Oe(n,t){var e=n.site,r=e.x,i=e.y,u=i-t;if(!u)return r;var o=n.P;if(!o)return-(1/0);e=o.site;var a=e.x,l=e.y,c=l-t;if(!c)return a;var f=a-r,s=1/u-1/c,h=f/c;return s?(-h+Math.sqrt(h*h-2*s*(f*f/(-2*c)-l+c/2+i-u/2)))/s+r:(r+a)/2}function Ie(n,t){var e=n.N;if(e)return Oe(e,t);var r=n.site;return r.y===t?r.x:1/0}function Ye(n){this.site=n,this.edges=[]}function Z
 e(n){for(var t,e,r,i,u,o,a,l,c,f,s=n[0][0],h=n[1][0],p=n[0][1],g=n[1][1],v=ul,d=v.length;d--;)if(u=v[d],u&&u.prepare())for(a=u.edges,l=a.length,o=0;l>o;)f=a[o].end(),r=f.x,i=f.y,c=a[++o%l].start(),t=c.x,e=c.y,(xo(r-t)>Uo||xo(i-e)>Uo)&&(a.splice(o,0,new tr(Qe(u.site,f,xo(r-s)<Uo&&g-i>Uo?{x:s,y:xo(t-s)<Uo?e:g}:xo(i-g)<Uo&&h-r>Uo?{x:xo(e-g)<Uo?t:h,y:g}:xo(r-h)<Uo&&i-p>Uo?{x:h,y:xo(t-h)<Uo?e:p}:xo(i-p)<Uo&&r-s>Uo?{x:xo(e-p)<Uo?t:s,y:p}:null),u.site,null)),++l)}function Ve(n,t){return t.angle-n.angle}function Xe(){rr(this),this.x=this.y=this.arc=this.site=this.cy=null}function $e(n){var t=n.P,e=n.N;if(t&&e){var r=t.site,i=n.site,u=e.site;if(r!==u){var o=i.x,a=i.y,l=r.x-o,c=r.y-a,f=u.x-o,s=u.y-a,h=2*(l*s-c*f);if(!(h>=-jo)){var p=l*l+c*c,g=f*f+s*s,v=(s*p-c*g)/h,d=(l*g-f*p)/h,s=d+a,y=fl.pop()||new Xe;y.arc=n,y.site=i,y.x=v+o,y.y=s+Math.sqrt(v*v+d*d),y.cy=s,n.circle=y;for(var m=null,M=ll._;M;)if(y.y<M.y||y.y===M.y&&y.x<=M.x){if(!M.L){m=M.P;break}M=M.L}else{if(!M.R){m=M;break}M=M.R}ll.insert(
 m,y),m||(al=y)}}}}function Be(n){var t=n.circle;t&&(t.P||(al=t.N),ll.remove(t),fl.push(t),rr(t),n.circle=null)}function We(n){for(var t,e=il,r=Yt(n[0][0],n[0][1],n[1][0],n[1][1]),i=e.length;i--;)t=e[i],(!Je(t,n)||!r(t)||xo(t.a.x-t.b.x)<Uo&&xo(t.a.y-t.b.y)<Uo)&&(t.a=t.b=null,e.splice(i,1))}function Je(n,t){var e=n.b;if(e)return!0;var r,i,u=n.a,o=t[0][0],a=t[1][0],l=t[0][1],c=t[1][1],f=n.l,s=n.r,h=f.x,p=f.y,g=s.x,v=s.y,d=(h+g)/2,y=(p+v)/2;if(v===p){if(o>d||d>=a)return;if(h>g){if(u){if(u.y>=c)return}else u={x:d,y:l};e={x:d,y:c}}else{if(u){if(u.y<l)return}else u={x:d,y:c};e={x:d,y:l}}}else if(r=(h-g)/(v-p),i=y-r*d,-1>r||r>1)if(h>g){if(u){if(u.y>=c)return}else u={x:(l-i)/r,y:l};e={x:(c-i)/r,y:c}}else{if(u){if(u.y<l)return}else u={x:(c-i)/r,y:c};e={x:(l-i)/r,y:l}}else if(v>p){if(u){if(u.x>=a)return}else u={x:o,y:r*o+i};e={x:a,y:r*a+i}}else{if(u){if(u.x<o)return}else u={x:a,y:r*a+i};e={x:o,y:r*o+i}}return n.a=u,n.b=e,!0}function Ge(n,t){this.l=n,this.r=t,this.a=this.b=null}function Ke(n,t,
 e,r){var i=new Ge(n,t);return il.push(i),e&&nr(i,n,t,e),r&&nr(i,t,n,r),ul[n.i].edges.push(new tr(i,n,t)),ul[t.i].edges.push(new tr(i,t,n)),i}function Qe(n,t,e){var r=new Ge(n,null);return r.a=t,r.b=e,il.push(r),r}function nr(n,t,e,r){n.a||n.b?n.l===e?n.b=r:n.a=r:(n.a=r,n.l=t,n.r=e)}function tr(n,t,e){var r=n.a,i=n.b;this.edge=n,this.site=t,this.angle=e?Math.atan2(e.y-t.y,e.x-t.x):n.l===t?Math.atan2(i.x-r.x,r.y-i.y):Math.atan2(r.x-i.x,i.y-r.y)}function er(){this._=null}function rr(n){n.U=n.C=n.L=n.R=n.P=n.N=null}function ir(n,t){var e=t,r=t.R,i=e.U;i?i.L===e?i.L=r:i.R=r:n._=r,r.U=i,e.U=r,e.R=r.L,e.R&&(e.R.U=e),r.L=e}function ur(n,t){var e=t,r=t.L,i=e.U;i?i.L===e?i.L=r:i.R=r:n._=r,r.U=i,e.U=r,e.L=r.R,e.L&&(e.L.U=e),r.R=e}function or(n){for(;n.L;)n=n.L;return n}function ar(n,t){var e,r,i,u=n.sort(lr).pop();for(il=[],ul=new Array(n.length),ol=new er,ll=new er;;)if(i=al,u&&(!i||u.y<i.y||u.y===i.y&&u.x<i.x))u.x===e&&u.y===r||(ul[u.i]=new Ye(u),He(u),e=u.x,r=u.y),u=n.pop();else{if(!i)break
 ;Fe(i.arc)}t&&(We(t),Ze(t));var o={cells:ul,edges:il};return ol=ll=il=ul=null,o}function lr(n,t){return t.y-n.y||t.x-n.x}function cr(n,t,e){return(n.x-e.x)*(t.y-n.y)-(n.x-t.x)*(e.y-n.y)}function fr(n){return n.x}function sr(n){return n.y}function hr(){return{leaf:!0,nodes:[],point:null,x:null,y:null}}function pr(n,t,e,r,i,u){if(!n(t,e,r,i,u)){var o=.5*(e+i),a=.5*(r+u),l=t.nodes;l[0]&&pr(n,l[0],e,r,o,a),l[1]&&pr(n,l[1],o,r,i,a),l[2]&&pr(n,l[2],e,a,o,u),l[3]&&pr(n,l[3],o,a,i,u)}}function gr(n,t,e,r,i,u,o){var a,l=1/0;return function c(n,f,s,h,p){if(!(f>u||s>o||r>h||i>p)){if(g=n.point){var g,v=t-n.x,d=e-n.y,y=v*v+d*d;if(l>y){var m=Math.sqrt(l=y);r=t-m,i=e-m,u=t+m,o=e+m,a=g}}for(var M=n.nodes,x=.5*(f+h),b=.5*(s+p),_=t>=x,w=e>=b,S=w<<1|_,k=S+4;k>S;++S)if(n=M[3&S])switch(3&S){case 0:c(n,f,s,x,b);break;case 1:c(n,x,s,h,b);break;case 2:c(n,f,b,x,p);break;case 3:c(n,x,b,h,p)}}}(n,r,i,u,o),a}function vr(n,t){n=ao.rgb(n),t=ao.rgb(t);var e=n.r,r=n.g,i=n.b,u=t.r-e,o=t.g-r,a=t.b-i;return function
 (n){return"#"+bn(Math.round(e+u*n))+bn(Math.round(r+o*n))+bn(Math.round(i+a*n))}}function dr(n,t){var e,r={},i={};for(e in n)e in t?r[e]=Mr(n[e],t[e]):i[e]=n[e];for(e in t)e in n||(i[e]=t[e]);return function(n){for(e in r)i[e]=r[e](n);return i}}function yr(n,t){return n=+n,t=+t,function(e){return n*(1-e)+t*e}}function mr(n,t){var e,r,i,u=hl.lastIndex=pl.lastIndex=0,o=-1,a=[],l=[];for(n+="",t+="";(e=hl.exec(n))&&(r=pl.exec(t));)(i=r.index)>u&&(i=t.slice(u,i),a[o]?a[o]+=i:a[++o]=i),(e=e[0])===(r=r[0])?a[o]?a[o]+=r:a[++o]=r:(a[++o]=null,l.push({i:o,x:yr(e,r)})),u=pl.lastIndex;return u<t.length&&(i=t.slice(u),a[o]?a[o]+=i:a[++o]=i),a.length<2?l[0]?(t=l[0].x,function(n){return t(n)+""}):function(){return t}:(t=l.length,function(n){for(var e,r=0;t>r;++r)a[(e=l[r]).i]=e.x(n);return a.join("")})}function Mr(n,t){for(var e,r=ao.interpolators.length;--r>=0&&!(e=ao.interpolators[r](n,t)););return e}function xr(n,t){var e,r=[],i=[],u=n.length,o=t.length,a=Math.min(n.length,t.length);for(e=0;a>e
 ;++e)r.push(Mr(n[e],t[e]));for(;u>e;++e)i[e]=n[e];for(;o>e;++e)i[e]=t[e];return function(n){for(e=0;a>e;++e)i[e]=r[e](n);return i}}function br(n){return function(t){return 0>=t?0:t>=1?1:n(t)}}function _r(n){return function(t){return 1-n(1-t)}}function wr(n){return function(t){return.5*(.5>t?n(2*t):2-n(2-2*t))}}function Sr(n){return n*n}function kr(n){return n*n*n}function Nr(n){if(0>=n)return 0;if(n>=1)return 1;var t=n*n,e=t*n;return 4*(.5>n?e:3*(n-t)+e-.75)}function Er(n){return function(t){return Math.pow(t,n)}}function Ar(n){return 1-Math.cos(n*Io)}function Cr(n){return Math.pow(2,10*(n-1))}function zr(n){return 1-Math.sqrt(1-n*n)}function Lr(n,t){var e;return arguments.length<2&&(t=.45),arguments.length?e=t/Ho*Math.asin(1/n):(n=1,e=t/4),function(r){return 1+n*Math.pow(2,-10*r)*Math.sin((r-e)*Ho/t)}}function qr(n){return n||(n=1.70158),function(t){return t*t*((n+1)*t-n)}}function Tr(n){return 1/2.75>n?7.5625*n*n:2/2.75>n?7.5625*(n-=1.5/2.75)*n+.75:2.5/2.75>n?7.5625*(n-=2.25/2.75)
 *n+.9375:7.5625*(n-=2.625/2.75)*n+.984375}function Rr(n,t){n=ao.hcl(n),t=ao.hcl(t);var e=n.h,r=n.c,i=n.l,u=t.h-e,o=t.c-r,a=t.l-i;return isNaN(o)&&(o=0,r=isNaN(r)?t.c:r),isNaN(u)?(u=0,e=isNaN(e)?t.h:e):u>180?u-=360:-180>u&&(u+=360),function(n){return sn(e+u*n,r+o*n,i+a*n)+""}}function Dr(n,t){n=ao.hsl(n),t=ao.hsl(t);var e=n.h,r=n.s,i=n.l,u=t.h-e,o=t.s-r,a=t.l-i;return isNaN(o)&&(o=0,r=isNaN(r)?t.s:r),isNaN(u)?(u=0,e=isNaN(e)?t.h:e):u>180?u-=360:-180>u&&(u+=360),function(n){return cn(e+u*n,r+o*n,i+a*n)+""}}function Pr(n,t){n=ao.lab(n),t=ao.lab(t);var e=n.l,r=n.a,i=n.b,u=t.l-e,o=t.a-r,a=t.b-i;return function(n){return pn(e+u*n,r+o*n,i+a*n)+""}}function Ur(n,t){return t-=n,function(e){return Math.round(n+t*e)}}function jr(n){var t=[n.a,n.b],e=[n.c,n.d],r=Hr(t),i=Fr(t,e),u=Hr(Or(e,t,-i))||0;t[0]*e[1]<e[0]*t[1]&&(t[0]*=-1,t[1]*=-1,r*=-1,i*=-1),this.rotate=(r?Math.atan2(t[1],t[0]):Math.atan2(-e[0],e[1]))*Zo,this.translate=[n.e,n.f],this.scale=[r,u],this.skew=u?Math.atan2(i,u)*Zo:0}function
  Fr(n,t){return n[0]*t[0]+n[1]*t[1]}function Hr(n){var t=Math.sqrt(Fr(n,n));return t&&(n[0]/=t,n[1]/=t),t}function Or(n,t,e){return n[0]+=e*t[0],n[1]+=e*t[1],n}function Ir(n){return n.length?n.pop()+",":""}function Yr(n,t,e,r){if(n[0]!==t[0]||n[1]!==t[1]){var i=e.push("translate(",null,",",null,")");r.push({i:i-4,x:yr(n[0],t[0])},{i:i-2,x:yr(n[1],t[1])})}else(t[0]||t[1])&&e.push("translate("+t+")")}function Zr(n,t,e,r){n!==t?(n-t>180?t+=360:t-n>180&&(n+=360),r.push({i:e.push(Ir(e)+"rotate(",null,")")-2,x:yr(n,t)})):t&&e.push(Ir(e)+"rotate("+t+")")}function Vr(n,t,e,r){n!==t?r.push({i:e.push(Ir(e)+"skewX(",null,")")-2,x:yr(n,t)}):t&&e.push(Ir(e)+"skewX("+t+")")}function Xr(n,t,e,r){if(n[0]!==t[0]||n[1]!==t[1]){var i=e.push(Ir(e)+"scale(",null,",",null,")");r.push({i:i-4,x:yr(n[0],t[0])},{i:i-2,x:yr(n[1],t[1])})}else 1===t[0]&&1===t[1]||e.push(Ir(e)+"scale("+t+")")}function $r(n,t){var e=[],r=[];return n=ao.transform(n),t=ao.transform(t),Yr(n.translate,t.translate,e,r),Zr(n.rotate,t.r
 otate,e,r),Vr(n.skew,t.skew,e,r),Xr(n.scale,t.scale,e,r),n=t=null,function(n){for(var t,i=-1,u=r.length;++i<u;)e[(t=r[i]).i]=t.x(n);return e.join("")}}function Br(n,t){return t=(t-=n=+n)||1/t,function(e){return(e-n)/t}}function Wr(n,t){return t=(t-=n=+n)||1/t,function(e){return Math.max(0,Math.min(1,(e-n)/t))}}function Jr(n){for(var t=n.source,e=n.target,r=Kr(t,e),i=[t];t!==r;)t=t.parent,i.push(t);for(var u=i.length;e!==r;)i.splice(u,0,e),e=e.parent;return i}function Gr(n){for(var t=[],e=n.parent;null!=e;)t.push(n),n=e,e=e.parent;return t.push(n),t}function Kr(n,t){if(n===t)return n;for(var e=Gr(n),r=Gr(t),i=e.pop(),u=r.pop(),o=null;i===u;)o=i,i=e.pop(),u=r.pop();return o}function Qr(n){n.fixed|=2}function ni(n){n.fixed&=-7}function ti(n){n.fixed|=4,n.px=n.x,n.py=n.y}function ei(n){n.fixed&=-5}function ri(n,t,e){var r=0,i=0;if(n.charge=0,!n.leaf)for(var u,o=n.nodes,a=o.length,l=-1;++l<a;)u=o[l],null!=u&&(ri(u,t,e),n.charge+=u.charge,r+=u.charge*u.cx,i+=u.charge*u.cy);if(n.point){n.l
 eaf||(n.point.x+=Math.random()-.5,n.point.y+=Math.random()-.5);var c=t*e[n.point.index];n.charge+=n.pointCharge=c,r+=c*n.point.x,i+=c*n.point.y}n.cx=r/n.charge,n.cy=i/n.charge}function ii(n,t){return ao.rebind(n,t,"sort","children","value"),n.nodes=n,n.links=fi,n}function ui(n,t){for(var e=[n];null!=(n=e.pop());)if(t(n),(i=n.children)&&(r=i.length))for(var r,i;--r>=0;)e.push(i[r])}function oi(n,t){for(var e=[n],r=[];null!=(n=e.pop());)if(r.push(n),(u=n.children)&&(i=u.length))for(var i,u,o=-1;++o<i;)e.push(u[o]);for(;null!=(n=r.pop());)t(n)}function ai(n){return n.children}function li(n){return n.value}function ci(n,t){return t.value-n.value}function fi(n){return ao.merge(n.map(function(n){return(n.children||[]).map(function(t){return{source:n,target:t}})}))}function si(n){return n.x}function hi(n){return n.y}function pi(n,t,e){n.y0=t,n.y=e}function gi(n){return ao.range(n.length)}function vi(n){for(var t=-1,e=n[0].length,r=[];++t<e;)r[t]=0;return r}function di(n){for(var t,e=1,r=0,
 i=n[0][1],u=n.length;u>e;++e)(t=n[e][1])>i&&(r=e,i=t);return r}function yi(n){return n.reduce(mi,0)}function mi(n,t){return n+t[1]}function Mi(n,t){return xi(n,Math.ceil(Math.log(t.length)/Math.LN2+1))}function xi(n,t){for(var e=-1,r=+n[0],i=(n[1]-r)/t,u=[];++e<=t;)u[e]=i*e+r;return u}function bi(n){return[ao.min(n),ao.max(n)]}function _i(n,t){return n.value-t.value}function wi(n,t){var e=n._pack_next;n._pack_next=t,t._pack_prev=n,t._pack_next=e,e._pack_prev=t}function Si(n,t){n._pack_next=t,t._pack_prev=n}function ki(n,t){var e=t.x-n.x,r=t.y-n.y,i=n.r+t.r;return.999*i*i>e*e+r*r}function Ni(n){function t(n){f=Math.min(n.x-n.r,f),s=Math.max(n.x+n.r,s),h=Math.min(n.y-n.r,h),p=Math.max(n.y+n.r,p)}if((e=n.children)&&(c=e.length)){var e,r,i,u,o,a,l,c,f=1/0,s=-(1/0),h=1/0,p=-(1/0);if(e.forEach(Ei),r=e[0],r.x=-r.r,r.y=0,t(r),c>1&&(i=e[1],i.x=i.r,i.y=0,t(i),c>2))for(u=e[2],zi(r,i,u),t(u),wi(r,u),r._pack_prev=u,wi(u,i),i=r._pack_next,o=3;c>o;o++){zi(r,i,u=e[o]);var g=0,v=1,d=1;for(a=i._pack_
 next;a!==i;a=a._pack_next,v++)if(ki(a,u)){g=1;break}if(1==g)for(l=r._pack_prev;l!==a._pack_prev&&!ki(l,u);l=l._pack_prev,d++);g?(d>v||v==d&&i.r<r.r?Si(r,i=a):Si(r=l,i),o--):(wi(r,u),i=u,t(u))}var y=(f+s)/2,m=(h+p)/2,M=0;for(o=0;c>o;o++)u=e[o],u.x-=y,u.y-=m,M=Math.max(M,u.r+Math.sqrt(u.x*u.x+u.y*u.y));n.r=M,e.forEach(Ai)}}function Ei(n){n._pack_next=n._pack_prev=n}function Ai(n){delete n._pack_next,delete n._pack_prev}function Ci(n,t,e,r){var i=n.children;if(n.x=t+=r*n.x,n.y=e+=r*n.y,n.r*=r,i)for(var u=-1,o=i.length;++u<o;)Ci(i[u],t,e,r)}function zi(n,t,e){var r=n.r+e.r,i=t.x-n.x,u=t.y-n.y;if(r&&(i||u)){var o=t.r+e.r,a=i*i+u*u;o*=o,r*=r;var l=.5+(r-o)/(2*a),c=Math.sqrt(Math.max(0,2*o*(r+a)-(r-=a)*r-o*o))/(2*a);e.x=n.x+l*i+c*u,e.y=n.y+l*u-c*i}else e.x=n.x+r,e.y=n.y}function Li(n,t){return n.parent==t.parent?1:2}function qi(n){var t=n.children;return t.length?t[0]:n.t}function Ti(n){var t,e=n.children;return(t=e.length)?e[t-1]:n.t}function Ri(n,t,e){var r=e/(t.i-n.i);t.c-=r,t.s+=e,n.c+
 =r,t.z+=e,t.m+=e}function Di(n){for(var t,e=0,r=0,i=n.children,u=i.length;--u>=0;)t=i[u],t.z+=e,t.m+=e,e+=t.s+(r+=t.c)}function Pi(n,t,e){return n.a.parent===t.parent?n.a:e}function Ui(n){return 1+ao.max(n,function(n){return n.y})}function ji(n){return n.reduce(function(n,t){return n+t.x},0)/n.length}function Fi(n){var t=n.children;return t&&t.length?Fi(t[0]):n}function Hi(n){var t,e=n.children;return e&&(t=e.length)?Hi(e[t-1]):n}function Oi(n){return{x:n.x,y:n.y,dx:n.dx,dy:n.dy}}function Ii(n,t){var e=n.x+t[3],r=n.y+t[0],i=n.dx-t[1]-t[3],u=n.dy-t[0]-t[2];return 0>i&&(e+=i/2,i=0),0>u&&(r+=u/2,u=0),{x:e,y:r,dx:i,dy:u}}function Yi(n){var t=n[0],e=n[n.length-1];return e>t?[t,e]:[e,t]}function Zi(n){return n.rangeExtent?n.rangeExtent():Yi(n.range())}function Vi(n,t,e,r){var i=e(n[0],n[1]),u=r(t[0],t[1]);return function(n){return u(i(n))}}function Xi(n,t){var e,r=0,i=n.length-1,u=n[r],o=n[i];return u>o&&(e=r,r=i,i=e,e=u,u=o,o=e),n[r]=t.floor(u),n[i]=t.ceil(o),n}function $i(n){return n?{f
 loor:function(t){return Math.floor(t/n)*n},ceil:function(t){return Math.ceil(t/n)*n}}:Sl}function Bi(n,t,e,r){var i=[],u=[],o=0,a=Math.min(n.length,t.length)-1;for(n[a]<n[0]&&(n=n.slice().reverse(),t=t.slice().reverse());++o<=a;)i.push(e(n[o-1],n[o])),u.push(r(t[o-1],t[o]));return function(t){var e=ao.bisect(n,t,1,a)-1;return u[e](i[e](t))}}function Wi(n,t,e,r){function i(){var i=Math.min(n.length,t.length)>2?Bi:Vi,l=r?Wr:Br;return o=i(n,t,l,e),a=i(t,n,l,Mr),u}function u(n){return o(n)}var o,a;return u.invert=function(n){return a(n)},u.domain=function(t){return arguments.length?(n=t.map(Number),i()):n},u.range=function(n){return arguments.length?(t=n,i()):t},u.rangeRound=function(n){return u.range(n).interpolate(Ur)},u.clamp=function(n){return arguments.length?(r=n,i()):r},u.interpolate=function(n){return arguments.length?(e=n,i()):e},u.ticks=function(t){return Qi(n,t)},u.tickFormat=function(t,e){return nu(n,t,e)},u.nice=function(t){return Gi(n,t),i()},u.copy=function(){return Wi(n,
 t,e,r)},i()}function Ji(n,t){return ao.rebind(n,t,"range","rangeRound","interpolate","clamp")}function Gi(n,t){return Xi(n,$i(Ki(n,t)[2])),Xi(n,$i(Ki(n,t)[2])),n}function Ki(n,t){null==t&&(t=10);var e=Yi(n),r=e[1]-e[0],i=Math.pow(10,Math.floor(Math.log(r/t)/Math.LN10)),u=t/r*i;return.15>=u?i*=10:.35>=u?i*=5:.75>=u&&(i*=2),e[0]=Math.ceil(e[0]/i)*i,e[1]=Math.floor(e[1]/i)*i+.5*i,e[2]=i,e}function Qi(n,t){return ao.range.apply(ao,Ki(n,t))}function nu(n,t,e){var r=Ki(n,t);if(e){var i=ha.exec(e);if(i.shift(),"s"===i[8]){var u=ao.formatPrefix(Math.max(xo(r[0]),xo(r[1])));return i[7]||(i[7]="."+tu(u.scale(r[2]))),i[8]="f",e=ao.format(i.join("")),function(n){return e(u.scale(n))+u.symbol}}i[7]||(i[7]="."+eu(i[8],r)),e=i.join("")}else e=",."+tu(r[2])+"f";return ao.format(e)}function tu(n){return-Math.floor(Math.log(n)/Math.LN10+.01)}function eu(n,t){var e=tu(t[2]);return n in kl?Math.abs(e-tu(Math.max(xo(t[0]),xo(t[1]))))+ +("e"!==n):e-2*("%"===n)}function ru(n,t,e,r){function i(n){return(e?
 Math.log(0>n?0:n):-Math.log(n>0?0:-n))/Math.log(t)}function u(n){return e?Math.pow(t,n):-Math.pow(t,-n)}function o(t){return n(i(t))}return o.invert=function(t){return u(n.invert(t))},o.domain=function(t){return arguments.length?(e=t[0]>=0,n.domain((r=t.map(Number)).map(i)),o):r},o.base=function(e){return arguments.length?(t=+e,n.domain(r.map(i)),o):t},o.nice=function(){var t=Xi(r.map(i),e?Math:El);return n.domain(t),r=t.map(u),o},o.ticks=function(){var n=Yi(r),o=[],a=n[0],l=n[1],c=Math.floor(i(a)),f=Math.ceil(i(l)),s=t%1?2:t;if(isFinite(f-c)){if(e){for(;f>c;c++)for(var h=1;s>h;h++)o.push(u(c)*h);o.push(u(c))}else for(o.push(u(c));c++<f;)for(var h=s-1;h>0;h--)o.push(u(c)*h);for(c=0;o[c]<a;c++);for(f=o.length;o[f-1]>l;f--);o=o.slice(c,f)}return o},o.tickFormat=function(n,e){if(!arguments.length)return Nl;arguments.length<2?e=Nl:"function"!=typeof e&&(e=ao.format(e));var r=Math.max(1,t*n/o.ticks().length);return function(n){var o=n/u(Math.round(i(n)));return t-.5>o*t&&(o*=t),r>=o?e(n)
 :""}},o.copy=function(){return ru(n.copy(),t,e,r)},Ji(o,n)}function iu(n,t,e){function r(t){return n(i(t))}var i=uu(t),u=uu(1/t);return r.invert=function(t){return u(n.invert(t))},r.domain=function(t){return arguments.length?(n.domain((e=t.map(Number)).map(i)),r):e},r.ticks=function(n){return Qi(e,n)},r.tickFormat=function(n,t){return nu(e,n,t)},r.nice=function(n){return r.domain(Gi(e,n))},r.exponent=function(o){return arguments.length?(i=uu(t=o),u=uu(1/t),n.domain(e.map(i)),r):t},r.copy=function(){return iu(n.copy(),t,e)},Ji(r,n)}function uu(n){return function(t){return 0>t?-Math.pow(-t,n):Math.pow(t,n)}}function ou(n,t){function e(e){return u[((i.get(e)||("range"===t.t?i.set(e,n.push(e)):NaN))-1)%u.length]}function r(t,e){return ao.range(n.length).map(function(n){return t+e*n})}var i,u,o;return e.domain=function(r){if(!arguments.length)return n;n=[],i=new c;for(var u,o=-1,a=r.length;++o<a;)i.has(u=r[o])||i.set(u,n.push(u));return e[t.t].apply(e,t.a)},e.range=function(n){return arg
 uments.length?(u=n,o=0,t={t:"range",a:arguments},e):u},e.rangePoints=function(i,a){arguments.length<2&&(a=0);var l=i[0],c=i[1],f=n.length<2?(l=(l+c)/2,0):(c-l)/(n.length-1+a);return u=r(l+f*a/2,f),o=0,t={t:"rangePoints",a:arguments},e},e.rangeRoundPoints=function(i,a){arguments.length<2&&(a=0);var l=i[0],c=i[1],f=n.length<2?(l=c=Math.round((l+c)/2),0):(c-l)/(n.length-1+a)|0;return u=r(l+Math.round(f*a/2+(c-l-(n.length-1+a)*f)/2),f),o=0,t={t:"rangeRoundPoints",a:arguments},e},e.rangeBands=function(i,a,l){arguments.length<2&&(a=0),arguments.length<3&&(l=a);var c=i[1]<i[0],f=i[c-0],s=i[1-c],h=(s-f)/(n.length-a+2*l);return u=r(f+h*l,h),c&&u.reverse(),o=h*(1-a),t={t:"rangeBands",a:arguments},e},e.rangeRoundBands=function(i,a,l){arguments.length<2&&(a=0),arguments.length<3&&(l=a);var c=i[1]<i[0],f=i[c-0],s=i[1-c],h=Math.floor((s-f)/(n.length-a+2*l));return u=r(f+Math.round((s-f-(n.length-a)*h)/2),h),c&&u.reverse(),o=Math.round(h*(1-a)),t={t:"rangeRoundBands",a:arguments},e},e.rangeBand=fu
 nction(){return o},e.rangeExtent=function(){return Yi(t.a[0])},e.copy=function(){return ou(n,t)},e.domain(n)}function au(n,t){function u(){var e=0,r=t.length;for(a=[];++e<r;)a[e-1]=ao.quantile(n,e/r);return o}function o(n){return isNaN(n=+n)?void 0:t[ao.bisect(a,n)]}var a;return o.domain=function(t){return arguments.length?(n=t.map(r).filter(i).sort(e),u()):n},o.range=function(n){return arguments.length?(t=n,u()):t},o.quantiles=function(){return a},o.invertExtent=function(e){return e=t.indexOf(e),0>e?[NaN,NaN]:[e>0?a[e-1]:n[0],e<a.length?a[e]:n[n.length-1]]},o.copy=function(){return au(n,t)},u()}function lu(n,t,e){function r(t){return e[Math.max(0,Math.min(o,Math.floor(u*(t-n))))]}function i(){return u=e.length/(t-n),o=e.length-1,r}var u,o;return r.domain=function(e){return arguments.length?(n=+e[0],t=+e[e.length-1],i()):[n,t]},r.range=function(n){return arguments.length?(e=n,i()):e},r.invertExtent=function(t){return t=e.indexOf(t),t=0>t?NaN:t/u+n,[t,t+1/u]},r.copy=function(){return
  lu(n,t,e)},i()}function cu(n,t){function e(e){return e>=e?t[ao.bisect(n,e)]:void 0}return e.domain=function(t){return arguments.length?(n=t,e):n},e.range=function(n){return arguments.length?(t=n,e):t},e.invertExtent=function(e){return e=t.indexOf(e),[n[e-1],n[e]]},e.copy=function(){return cu(n,t)},e}function fu(n){function t(n){return+n}return t.invert=t,t.domain=t.range=function(e){return arguments.length?(n=e.map(t),t):n},t.ticks=function(t){return Qi(n,t)},t.tickFormat=function(t,e){return nu(n,t,e)},t.copy=function(){return fu(n)},t}function su(){return 0}function hu(n){return n.innerRadius}function pu(n){return n.outerRadius}function gu(n){return n.startAngle}function vu(n){return n.endAngle}function du(n){return n&&n.padAngle}function yu(n,t,e,r){return(n-e)*t-(t-r)*n>0?0:1}function mu(n,t,e,r,i){var u=n[0]-t[0],o=n[1]-t[1],a=(i?r:-r)/Math.sqrt(u*u+o*o),l=a*o,c=-a*u,f=n[0]+l,s=n[1]+c,h=t[0]+l,p=t[1]+c,g=(f+h)/2,v=(s+p)/2,d=h-f,y=p-s,m=d*d+y*y,M=e-r,x=f*p-h*s,b=(0>y?-1:1)*Math
 .sqrt(Math.max(0,M*M*m-x*x)),_=(x*y-d*b)/m,w=(-x*d-y*b)/m,S=(x*y+d*b)/m,k=(-x*d+y*b)/m,N=_-g,E=w-v,A=S-g,C=k-v;return N*N+E*E>A*A+C*C&&(_=S,w=k),[[_-l,w-c],[_*e/M,w*e/M]]}function Mu(n){function t(t){function o(){c.push("M",u(n(f),a))}for(var l,c=[],f=[],s=-1,h=t.length,p=En(e),g=En(r);++s<h;)i.call(this,l=t[s],s)?f.push([+p.call(this,l,s),+g.call(this,l,s)]):f.length&&(o(),f=[]);return f.length&&o(),c.length?c.join(""):null}var e=Ce,r=ze,i=zt,u=xu,o=u.key,a=.7;return t.x=function(n){return arguments.length?(e=n,t):e},t.y=function(n){return arguments.length?(r=n,t):r},t.defined=function(n){return arguments.length?(i=n,t):i},t.interpolate=function(n){return arguments.length?(o="function"==typeof n?u=n:(u=Tl.get(n)||xu).key,t):o},t.tension=function(n){return arguments.length?(a=n,t):a},t}function xu(n){return n.length>1?n.join("L"):n+"Z"}function bu(n){return n.join("L")+"Z"}function _u(n){for(var t=0,e=n.length,r=n[0],i=[r[0],",",r[1]];++t<e;)i.push("H",(r[0]+(r=n[t])[0])/2,"V",r[1])
 ;return e>1&&i.push("H",r[0]),i.join("")}function wu(n){for(var t=0,e=n.length,r=n[0],i=[r[0],",",r[1]];++t<e;)i.push("V",(r=n[t])[1],"H",r[0]);return i.join("")}function Su(n){for(var t=0,e=n.length,r=n[0],i=[r[0],",",r[1]];++t<e;)i.push("H",(r=n[t])[0],"V",r[1]);return i.join("")}function ku(n,t){return n.length<4?xu(n):n[1]+Au(n.slice(1,-1),Cu(n,t))}function Nu(n,t){return n.length<3?bu(n):n[0]+Au((n.push(n[0]),n),Cu([n[n.length-2]].concat(n,[n[1]]),t))}function Eu(n,t){return n.length<3?xu(n):n[0]+Au(n,Cu(n,t))}function Au(n,t){if(t.length<1||n.length!=t.length&&n.length!=t.length+2)return xu(n);var e=n.length!=t.length,r="",i=n[0],u=n[1],o=t[0],a=o,l=1;if(e&&(r+="Q"+(u[0]-2*o[0]/3)+","+(u[1]-2*o[1]/3)+","+u[0]+","+u[1],i=n[1],l=2),t.length>1){a=t[1],u=n[l],l++,r+="C"+(i[0]+o[0])+","+(i[1]+o[1])+","+(u[0]-a[0])+","+(u[1]-a[1])+","+u[0]+","+u[1];for(var c=2;c<t.length;c++,l++)u=n[l],a=t[c],r+="S"+(u[0]-a[0])+","+(u[1]-a[1])+","+u[0]+","+u[1]}if(e){var f=n[l];r+="Q"+(u[0]+2*a[0]/3
 )+","+(u[1]+2*a[1]/3)+","+f[0]+","+f[1]}return r}function Cu(n,t){for(var e,r=[],i=(1-t)/2,u=n[0],o=n[1],a=1,l=n.length;++a<l;)e=u,u=o,o=n[a],r.push([i*(o[0]-e[0]),i*(o[1]-e[1])]);return r}function zu(n){if(n.length<3)return xu(n);var t=1,e=n.length,r=n[0],i=r[0],u=r[1],o=[i,i,i,(r=n[1])[0]],a=[u,u,u,r[1]],l=[i,",",u,"L",Ru(Pl,o),",",Ru(Pl,a)];for(n.push(n[e-1]);++t<=e;)r=n[t],o.shift(),o.push(r[0]),a.shift(),a.push(r[1]),Du(l,o,a);return n.pop(),l.push("L",r),l.join("")}function Lu(n){if(n.length<4)return xu(n);for(var t,e=[],r=-1,i=n.length,u=[0],o=[0];++r<3;)t=n[r],u.push(t[0]),o.push(t[1]);for(e.push(Ru(Pl,u)+","+Ru(Pl,o)),--r;++r<i;)t=n[r],u.shift(),u.push(t[0]),o.shift(),o.push(t[1]),Du(e,u,o);return e.join("")}function qu(n){for(var t,e,r=-1,i=n.length,u=i+4,o=[],a=[];++r<4;)e=n[r%i],o.push(e[0]),a.push(e[1]);for(t=[Ru(Pl,o),",",Ru(Pl,a)],--r;++r<u;)e=n[r%i],o.shift(),o.push(e[0]),a.shift(),a.push(e[1]),Du(t,o,a);return t.join("")}function Tu(n,t){var e=n.length-1;if(e)for(va
 r r,i,u=n[0][0],o=n[0][1],a=n[e][0]-u,l=n[e][1]-o,c=-1;++c<=e;)r=n[c],i=c/e,r[0]=t*r[0]+(1-t)*(u+i*a),r[1]=t*r[1]+(1-t)*(o+i*l);return zu(n)}function Ru(n,t){return n[0]*t[0]+n[1]*t[1]+n[2]*t[2]+n[3]*t[3]}function Du(n,t,e){n.push("C",Ru(Rl,t),",",Ru(Rl,e),",",Ru(Dl,t),",",Ru(Dl,e),",",Ru(Pl,t),",",Ru(Pl,e))}function Pu(n,t){return(t[1]-n[1])/(t[0]-n[0])}function Uu(n){for(var t=0,e=n.length-1,r=[],i=n[0],u=n[1],o=r[0]=Pu(i,u);++t<e;)r[t]=(o+(o=Pu(i=u,u=n[t+1])))/2;return r[t]=o,r}function ju(n){for(var t,e,r,i,u=[],o=Uu(n),a=-1,l=n.length-1;++a<l;)t=Pu(n[a],n[a+1]),xo(t)<Uo?o[a]=o[a+1]=0:(e=o[a]/t,r=o[a+1]/t,i=e*e+r*r,i>9&&(i=3*t/Math.sqrt(i),o[a]=i*e,o[a+1]=i*r));for(a=-1;++a<=l;)i=(n[Math.min(l,a+1)][0]-n[Math.max(0,a-1)][0])/(6*(1+o[a]*o[a])),u.push([i||0,o[a]*i||0]);return u}function Fu(n){return n.length<3?xu(n):n[0]+Au(n,ju(n))}function Hu(n){for(var t,e,r,i=-1,u=n.length;++i<u;)t=n[i],e=t[0],r=t[1]-Io,t[0]=e*Math.cos(r),t[1]=e*Math.sin(r);return n}function Ou(n){function t(t
 ){function l(){v.push("M",a(n(y),s),f,c(n(d.reverse()),s),"Z")}for(var h,p,g,v=[],d=[],y=[],m=-1,M=t.length,x=En(e),b=En(i),_=e===r?function(){
-return p}:En(r),w=i===u?function(){return g}:En(u);++m<M;)o.call(this,h=t[m],m)?(d.push([p=+x.call(this,h,m),g=+b.call(this,h,m)]),y.push([+_.call(this,h,m),+w.call(this,h,m)])):d.length&&(l(),d=[],y=[]);return d.length&&l(),v.length?v.join(""):null}var e=Ce,r=Ce,i=0,u=ze,o=zt,a=xu,l=a.key,c=a,f="L",s=.7;return t.x=function(n){return arguments.length?(e=r=n,t):r},t.x0=function(n){return arguments.length?(e=n,t):e},t.x1=function(n){return arguments.length?(r=n,t):r},t.y=function(n){return arguments.length?(i=u=n,t):u},t.y0=function(n){return arguments.length?(i=n,t):i},t.y1=function(n){return arguments.length?(u=n,t):u},t.defined=function(n){return arguments.length?(o=n,t):o},t.interpolate=function(n){return arguments.length?(l="function"==typeof n?a=n:(a=Tl.get(n)||xu).key,c=a.reverse||a,f=a.closed?"M":"L",t):l},t.tension=function(n){return arguments.length?(s=n,t):s},t}function Iu(n){return n.radius}function Yu(n){return[n.x,n.y]}function Zu(n){return function(){var t=n.apply(this,
 arguments),e=t[0],r=t[1]-Io;return[e*Math.cos(r),e*Math.sin(r)]}}function Vu(){return 64}function Xu(){return"circle"}function $u(n){var t=Math.sqrt(n/Fo);return"M0,"+t+"A"+t+","+t+" 0 1,1 0,"+-t+"A"+t+","+t+" 0 1,1 0,"+t+"Z"}function Bu(n){return function(){var t,e,r;(t=this[n])&&(r=t[e=t.active])&&(r.timer.c=null,r.timer.t=NaN,--t.count?delete t[e]:delete this[n],t.active+=.5,r.event&&r.event.interrupt.call(this,this.__data__,r.index))}}function Wu(n,t,e){return ko(n,Yl),n.namespace=t,n.id=e,n}function Ju(n,t,e,r){var i=n.id,u=n.namespace;return Y(n,"function"==typeof e?function(n,o,a){n[u][i].tween.set(t,r(e.call(n,n.__data__,o,a)))}:(e=r(e),function(n){n[u][i].tween.set(t,e)}))}function Gu(n){return null==n&&(n=""),function(){this.textContent=n}}function Ku(n){return null==n?"__transition__":"__transition_"+n+"__"}function Qu(n,t,e,r,i){function u(n){var t=v.delay;return f.t=t+l,n>=t?o(n-t):void(f.c=o)}function o(e){var i=g.active,u=g[i];u&&(u.timer.c=null,u.timer.t=NaN,--g.coun
 t,delete g[i],u.event&&u.event.interrupt.call(n,n.__data__,u.index));for(var o in g)if(r>+o){var c=g[o];c.timer.c=null,c.timer.t=NaN,--g.count,delete g[o]}f.c=a,qn(function(){return f.c&&a(e||1)&&(f.c=null,f.t=NaN),1},0,l),g.active=r,v.event&&v.event.start.call(n,n.__data__,t),p=[],v.tween.forEach(function(e,r){(r=r.call(n,n.__data__,t))&&p.push(r)}),h=v.ease,s=v.duration}function a(i){for(var u=i/s,o=h(u),a=p.length;a>0;)p[--a].call(n,o);return u>=1?(v.event&&v.event.end.call(n,n.__data__,t),--g.count?delete g[r]:delete n[e],1):void 0}var l,f,s,h,p,g=n[e]||(n[e]={active:0,count:0}),v=g[r];v||(l=i.time,f=qn(u,0,l),v=g[r]={tween:new c,time:l,timer:f,delay:i.delay,duration:i.duration,ease:i.ease,index:t},i=null,++g.count)}function no(n,t,e){n.attr("transform",function(n){var r=t(n);return"translate("+(isFinite(r)?r:e(n))+",0)"})}function to(n,t,e){n.attr("transform",function(n){var r=t(n);return"translate(0,"+(isFinite(r)?r:e(n))+")"})}function eo(n){return n.toISOString()}function ro
 (n,t,e){function r(t){return n(t)}function i(n,e){var r=n[1]-n[0],i=r/e,u=ao.bisect(Kl,i);return u==Kl.length?[t.year,Ki(n.map(function(n){return n/31536e6}),e)[2]]:u?t[i/Kl[u-1]<Kl[u]/i?u-1:u]:[tc,Ki(n,e)[2]]}return r.invert=function(t){return io(n.invert(t))},r.domain=function(t){return arguments.length?(n.domain(t),r):n.domain().map(io)},r.nice=function(n,t){function e(e){return!isNaN(e)&&!n.range(e,io(+e+1),t).length}var u=r.domain(),o=Yi(u),a=null==n?i(o,10):"number"==typeof n&&i(o,n);return a&&(n=a[0],t=a[1]),r.domain(Xi(u,t>1?{floor:function(t){for(;e(t=n.floor(t));)t=io(t-1);return t},ceil:function(t){for(;e(t=n.ceil(t));)t=io(+t+1);return t}}:n))},r.ticks=function(n,t){var e=Yi(r.domain()),u=null==n?i(e,10):"number"==typeof n?i(e,n):!n.range&&[{range:n},t];return u&&(n=u[0],t=u[1]),n.range(e[0],io(+e[1]+1),1>t?1:t)},r.tickFormat=function(){return e},r.copy=function(){return ro(n.copy(),t,e)},Ji(r,n)}function io(n){return new Date(n)}function uo(n){return JSON.parse(n.respon
 seText)}function oo(n){var t=fo.createRange();return t.selectNode(fo.body),t.createContextualFragment(n.responseText)}var ao={version:"3.5.17"},lo=[].slice,co=function(n){return lo.call(n)},fo=this.document;if(fo)try{co(fo.documentElement.childNodes)[0].nodeType}catch(so){co=function(n){for(var t=n.length,e=new Array(t);t--;)e[t]=n[t];return e}}if(Date.now||(Date.now=function(){return+new Date}),fo)try{fo.createElement("DIV").style.setProperty("opacity",0,"")}catch(ho){var po=this.Element.prototype,go=po.setAttribute,vo=po.setAttributeNS,yo=this.CSSStyleDeclaration.prototype,mo=yo.setProperty;po.setAttribute=function(n,t){go.call(this,n,t+"")},po.setAttributeNS=function(n,t,e){vo.call(this,n,t,e+"")},yo.setProperty=function(n,t,e){mo.call(this,n,t+"",e)}}ao.ascending=e,ao.descending=function(n,t){return n>t?-1:t>n?1:t>=n?0:NaN},ao.min=function(n,t){var e,r,i=-1,u=n.length;if(1===arguments.length){for(;++i<u;)if(null!=(r=n[i])&&r>=r){e=r;break}for(;++i<u;)null!=(r=n[i])&&e>r&&(e=r)}e
 lse{for(;++i<u;)if(null!=(r=t.call(n,n[i],i))&&r>=r){e=r;break}for(;++i<u;)null!=(r=t.call(n,n[i],i))&&e>r&&(e=r)}return e},ao.max=function(n,t){var e,r,i=-1,u=n.length;if(1===arguments.length){for(;++i<u;)if(null!=(r=n[i])&&r>=r){e=r;break}for(;++i<u;)null!=(r=n[i])&&r>e&&(e=r)}else{for(;++i<u;)if(null!=(r=t.call(n,n[i],i))&&r>=r){e=r;break}for(;++i<u;)null!=(r=t.call(n,n[i],i))&&r>e&&(e=r)}return e},ao.extent=function(n,t){var e,r,i,u=-1,o=n.length;if(1===arguments.length){for(;++u<o;)if(null!=(r=n[u])&&r>=r){e=i=r;break}for(;++u<o;)null!=(r=n[u])&&(e>r&&(e=r),r>i&&(i=r))}else{for(;++u<o;)if(null!=(r=t.call(n,n[u],u))&&r>=r){e=i=r;break}for(;++u<o;)null!=(r=t.call(n,n[u],u))&&(e>r&&(e=r),r>i&&(i=r))}return[e,i]},ao.sum=function(n,t){var e,r=0,u=n.length,o=-1;if(1===arguments.length)for(;++o<u;)i(e=+n[o])&&(r+=e);else for(;++o<u;)i(e=+t.call(n,n[o],o))&&(r+=e);return r},ao.mean=function(n,t){var e,u=0,o=n.length,a=-1,l=o;if(1===arguments.length)for(;++a<o;)i(e=r(n[a]))?u+=e:--l;els
 e for(;++a<o;)i(e=r(t.call(n,n[a],a)))?u+=e:--l;return l?u/l:void 0},ao.quantile=function(n,t){var e=(n.length-1)*t+1,r=Math.floor(e),i=+n[r-1],u=e-r;return u?i+u*(n[r]-i):i},ao.median=function(n,t){var u,o=[],a=n.length,l=-1;if(1===arguments.length)for(;++l<a;)i(u=r(n[l]))&&o.push(u);else for(;++l<a;)i(u=r(t.call(n,n[l],l)))&&o.push(u);return o.length?ao.quantile(o.sort(e),.5):void 0},ao.variance=function(n,t){var e,u,o=n.length,a=0,l=0,c=-1,f=0;if(1===arguments.length)for(;++c<o;)i(e=r(n[c]))&&(u=e-a,a+=u/++f,l+=u*(e-a));else for(;++c<o;)i(e=r(t.call(n,n[c],c)))&&(u=e-a,a+=u/++f,l+=u*(e-a));return f>1?l/(f-1):void 0},ao.deviation=function(){var n=ao.variance.apply(this,arguments);return n?Math.sqrt(n):n};var Mo=u(e);ao.bisectLeft=Mo.left,ao.bisect=ao.bisectRight=Mo.right,ao.bisector=function(n){return u(1===n.length?function(t,r){return e(n(t),r)}:n)},ao.shuffle=function(n,t,e){(u=arguments.length)<3&&(e=n.length,2>u&&(t=0));for(var r,i,u=e-t;u;)i=Math.random()*u--|0,r=n[u+t],n[u+
 t]=n[i+t],n[i+t]=r;return n},ao.permute=function(n,t){for(var e=t.length,r=new Array(e);e--;)r[e]=n[t[e]];return r},ao.pairs=function(n){for(var t,e=0,r=n.length-1,i=n[0],u=new Array(0>r?0:r);r>e;)u[e]=[t=i,i=n[++e]];return u},ao.transpose=function(n){if(!(i=n.length))return[];for(var t=-1,e=ao.min(n,o),r=new Array(e);++t<e;)for(var i,u=-1,a=r[t]=new Array(i);++u<i;)a[u]=n[u][t];return r},ao.zip=function(){return ao.transpose(arguments)},ao.keys=function(n){var t=[];for(var e in n)t.push(e);return t},ao.values=function(n){var t=[];for(var e in n)t.push(n[e]);return t},ao.entries=function(n){var t=[];for(var e in n)t.push({key:e,value:n[e]});return t},ao.merge=function(n){for(var t,e,r,i=n.length,u=-1,o=0;++u<i;)o+=n[u].length;for(e=new Array(o);--i>=0;)for(r=n[i],t=r.length;--t>=0;)e[--o]=r[t];return e};var xo=Math.abs;ao.range=function(n,t,e){if(arguments.length<3&&(e=1,arguments.length<2&&(t=n,n=0)),(t-n)/e===1/0)throw new Error("infinite range");var r,i=[],u=a(xo(e)),o=-1;if(n*=u
 ,t*=u,e*=u,0>e)for(;(r=n+e*++o)>t;)i.push(r/u);else for(;(r=n+e*++o)<t;)i.push(r/u);return i},ao.map=function(n,t){var e=new c;if(n instanceof c)n.forEach(function(n,t){e.set(n,t)});else if(Array.isArray(n)){var r,i=-1,u=n.length;if(1===arguments.length)for(;++i<u;)e.set(i,n[i]);else for(;++i<u;)e.set(t.call(n,r=n[i],i),r)}else for(var o in n)e.set(o,n[o]);return e};var bo="__proto__",_o="\x00";l(c,{has:h,get:function(n){return this._[f(n)]},set:function(n,t){return this._[f(n)]=t},remove:p,keys:g,values:function(){var n=[];for(var t in this._)n.push(this._[t]);return n},entries:function(){var n=[];for(var t in this._)n.push({key:s(t),value:this._[t]});return n},size:v,empty:d,forEach:function(n){for(var t in this._)n.call(this,s(t),this._[t])}}),ao.nest=function(){function n(t,o,a){if(a>=u.length)return r?r.call(i,o):e?o.sort(e):o;for(var l,f,s,h,p=-1,g=o.length,v=u[a++],d=new c;++p<g;)(h=d.get(l=v(f=o[p])))?h.push(f):d.set(l,[f]);return t?(f=t(),s=function(e,r){f.set(e,n(t,r,a))})
 :(f={},s=function(e,r){f[e]=n(t,r,a)}),d.forEach(s),f}function t(n,e){if(e>=u.length)return n;var r=[],i=o[e++];return n.forEach(function(n,i){r.push({key:n,values:t(i,e)})}),i?r.sort(function(n,t){return i(n.key,t.key)}):r}var e,r,i={},u=[],o=[];return i.map=function(t,e){return n(e,t,0)},i.entries=function(e){return t(n(ao.map,e,0),0)},i.key=function(n){return u.push(n),i},i.sortKeys=function(n){return o[u.length-1]=n,i},i.sortValues=function(n){return e=n,i},i.rollup=function(n){return r=n,i},i},ao.set=function(n){var t=new y;if(n)for(var e=0,r=n.length;r>e;++e)t.add(n[e]);return t},l(y,{has:h,add:function(n){return this._[f(n+="")]=!0,n},remove:p,values:g,size:v,empty:d,forEach:function(n){for(var t in this._)n.call(this,s(t))}}),ao.behavior={},ao.rebind=function(n,t){for(var e,r=1,i=arguments.length;++r<i;)n[e=arguments[r]]=M(n,t,t[e]);return n};var wo=["webkit","ms","moz","Moz","o","O"];ao.dispatch=function(){for(var n=new _,t=-1,e=arguments.length;++t<e;)n[arguments[t]]=w(n);
 return n},_.prototype.on=function(n,t){var e=n.indexOf("."),r="";if(e>=0&&(r=n.slice(e+1),n=n.slice(0,e)),n)return arguments.length<2?this[n].on(r):this[n].on(r,t);if(2===arguments.length){if(null==t)for(n in this)this.hasOwnProperty(n)&&this[n].on(r,null);return this}},ao.event=null,ao.requote=function(n){return n.replace(So,"\\$&")};var So=/[\\\^\$\*\+\?\|\[\]\(\)\.\{\}]/g,ko={}.__proto__?function(n,t){n.__proto__=t}:function(n,t){for(var e in t)n[e]=t[e]},No=function(n,t){return t.querySelector(n)},Eo=function(n,t){return t.querySelectorAll(n)},Ao=function(n,t){var e=n.matches||n[x(n,"matchesSelector")];return(Ao=function(n,t){return e.call(n,t)})(n,t)};"function"==typeof Sizzle&&(No=function(n,t){return Sizzle(n,t)[0]||null},Eo=Sizzle,Ao=Sizzle.matchesSelector),ao.selection=function(){return ao.select(fo.documentElement)};var Co=ao.selection.prototype=[];Co.select=function(n){var t,e,r,i,u=[];n=A(n);for(var o=-1,a=this.length;++o<a;){u.push(t=[]),t.parentNode=(r=this[o]).parentN
 ode;for(var l=-1,c=r.length;++l<c;)(i=r[l])?(t.push(e=n.call(i,i.__data__,l,o)),e&&"__data__"in i&&(e.__data__=i.__data__)):t.push(null)}return E(u)},Co.selectAll=function(n){var t,e,r=[];n=C(n);for(var i=-1,u=this.length;++i<u;)for(var o=this[i],a=-1,l=o.length;++a<l;)(e=o[a])&&(r.push(t=co(n.call(e,e.__data__,a,i))),t.parentNode=e);return E(r)};var zo="http://www.w3.org/1999/xhtml",Lo={svg:"http://www.w3.org/2000/svg",xhtml:zo,xlink:"http://www.w3.org/1999/xlink",xml:"http://www.w3.org/XML/1998/namespace",xmlns:"http://www.w3.org/2000/xmlns/"};ao.ns={prefix:Lo,qualify:function(n){var t=n.indexOf(":"),e=n;return t>=0&&"xmlns"!==(e=n.slice(0,t))&&(n=n.slice(t+1)),Lo.hasOwnProperty(e)?{space:Lo[e],local:n}:n}},Co.attr=function(n,t){if(arguments.length<2){if("string"==typeof n){var e=this.node();return n=ao.ns.qualify(n),n.local?e.getAttributeNS(n.space,n.local):e.getAttribute(n)}for(t in n)this.each(z(t,n[t]));return this}return this.each(z(n,t))},Co.classed=function(n,t){if(argument
 s.length<2){if("string"==typeof n){var e=this.node(),r=(n=T(n)).length,i=-1;if(t=e.classList){for(;++i<r;)if(!t.contains(n[i]))return!1}else for(t=e.getAttribute("class");++i<r;)if(!q(n[i]).test(t))return!1;return!0}for(t in n)this.each(R(t,n[t]));return this}return this.each(R(n,t))},Co.style=function(n,e,r){var i=arguments.length;if(3>i){if("string"!=typeof n){2>i&&(e="");for(r in n)this.each(P(r,n[r],e));return this}if(2>i){var u=this.node();return t(u).getComputedStyle(u,null).getPropertyValue(n)}r=""}return this.each(P(n,e,r))},Co.property=function(n,t){if(arguments.length<2){if("string"==typeof n)return this.node()[n];for(t in n)this.each(U(t,n[t]));return this}return this.each(U(n,t))},Co.text=function(n){return arguments.length?this.each("function"==typeof n?function(){var t=n.apply(this,arguments);this.textContent=null==t?"":t}:null==n?function(){this.textContent=""}:function(){this.textContent=n}):this.node().textContent},Co.html=function(n){return arguments.length?this.ea
 ch("function"==typeof n?function(){var t=n.apply(this,arguments);this.innerHTML=null==t?"":t}:null==n?function(){this.innerHTML=""}:function(){this.innerHTML=n}):this.node().innerHTML},Co.append=function(n){return n=j(n),this.select(function(){return this.appendChild(n.apply(this,arguments))})},Co.insert=function(n,t){return n=j(n),t=A(t),this.select(function(){return this.insertBefore(n.apply(this,arguments),t.apply(this,arguments)||null)})},Co.remove=function(){return this.each(F)},Co.data=function(n,t){function e(n,e){var r,i,u,o=n.length,s=e.length,h=Math.min(o,s),p=new Array(s),g=new Array(s),v=new Array(o);if(t){var d,y=new c,m=new Array(o);for(r=-1;++r<o;)(i=n[r])&&(y.has(d=t.call(i,i.__data__,r))?v[r]=i:y.set(d,i),m[r]=d);for(r=-1;++r<s;)(i=y.get(d=t.call(e,u=e[r],r)))?i!==!0&&(p[r]=i,i.__data__=u):g[r]=H(u),y.set(d,!0);for(r=-1;++r<o;)r in m&&y.get(m[r])!==!0&&(v[r]=n[r])}else{for(r=-1;++r<h;)i=n[r],u=e[r],i?(i.__data__=u,p[r]=i):g[r]=H(u);for(;s>r;++r)g[r]=H(e[r]);for(;o>r
 ;++r)v[r]=n[r]}g.update=p,g.parentNode=p.parentNode=v.parentNode=n.parentNode,a.push(g),l.push(p),f.push(v)}var r,i,u=-1,o=this.length;if(!arguments.length){for(n=new Array(o=(r=this[0]).length);++u<o;)(i=r[u])&&(n[u]=i.__data__);return n}var a=Z([]),l=E([]),f=E([]);if("function"==typeof n)for(;++u<o;)e(r=this[u],n.call(r,r.parentNode.__data__,u));else for(;++u<o;)e(r=this[u],n);return l.enter=function(){return a},l.exit=function(){return f},l},Co.datum=function(n){return arguments.length?this.property("__data__",n):this.property("__data__")},Co.filter=function(n){var t,e,r,i=[];"function"!=typeof n&&(n=O(n));for(var u=0,o=this.length;o>u;u++){i.push(t=[]),t.parentNode=(e=this[u]).parentNode;for(var a=0,l=e.length;l>a;a++)(r=e[a])&&n.call(r,r.__data__,a,u)&&t.push(r)}return E(i)},Co.order=function(){for(var n=-1,t=this.length;++n<t;)for(var e,r=this[n],i=r.length-1,u=r[i];--i>=0;)(e=r[i])&&(u&&u!==e.nextSibling&&u.parentNode.insertBefore(e,u),u=e);return this},Co.sort=function(n){n=
 I.apply(this,arguments);for(var t=-1,e=this.length;++t<e;)this[t].sort(n);return this.order()},Co.each=function(n){return Y(this,function(t,e,r){n.call(t,t.__data__,e,r)})},Co.call=function(n){var t=co(arguments);return n.apply(t[0]=this,t),this},Co.empty=function(){return!this.node()},Co.node=function(){for(var n=0,t=this.length;t>n;n++)for(var e=this[n],r=0,i=e.length;i>r;r++){var u=e[r];if(u)return u}return null},Co.size=function(){var n=0;return Y(this,function(){++n}),n};var qo=[];ao.selection.enter=Z,ao.selection.enter.prototype=qo,qo.append=Co.append,qo.empty=Co.empty,qo.node=Co.node,qo.call=Co.call,qo.size=Co.size,qo.select=function(n){for(var t,e,r,i,u,o=[],a=-1,l=this.length;++a<l;){r=(i=this[a]).update,o.push(t=[]),t.parentNode=i.parentNode;for(var c=-1,f=i.length;++c<f;)(u=i[c])?(t.push(r[c]=e=n.call(i.parentNode,u.__data__,c,a)),e.__data__=u.__data__):t.push(null)}return E(o)},qo.insert=function(n,t){return arguments.length<2&&(t=V(this)),Co.insert.call(this,n,t)},ao.se
 lect=function(t){var e;return"string"==typeof t?(e=[No(t,fo)],e.parentNode=fo.documentElement):(e=[t],e.parentNode=n(t)),E([e])},ao.selectAll=function(n){var t;return"string"==typeof n?(t=co(Eo(n,fo)),t.parentNode=fo.documentElement):(t=co(n),t.parentNode=null),E([t])},Co.on=function(n,t,e){var r=arguments.length;if(3>r){if("string"!=typeof n){2>r&&(t=!1);for(e in n)this.each(X(e,n[e],t));return this}if(2>r)return(r=this.node()["__on"+n])&&r._;e=!1}return this.each(X(n,t,e))};var To=ao.map({mouseenter:"mouseover",mouseleave:"mouseout"});fo&&To.forEach(function(n){"on"+n in fo&&To.remove(n)});var Ro,Do=0;ao.mouse=function(n){return J(n,k())};var Po=this.navigator&&/WebKit/.test(this.navigator.userAgent)?-1:0;ao.touch=function(n,t,e){if(arguments.length<3&&(e=t,t=k().changedTouches),t)for(var r,i=0,u=t.length;u>i;++i)if((r=t[i]).identifier===e)return J(n,r)},ao.behavior.drag=function(){function n(){this.on("mousedown.drag",u).on("touchstart.drag",o)}function e(n,t,e,u,o){return functi
 on(){function a(){var n,e,r=t(h,v);r&&(n=r[0]-M[0],e=r[1]-M[1],g|=n|e,M=r,p({type:"drag",x:r[0]+c[0],y:r[1]+c[1],dx:n,dy:e}))}function l(){t(h,v)&&(y.on(u+d,null).on(o+d,null),m(g),p({type:"dragend"}))}var c,f=this,s=ao.event.target.correspondingElement||ao.event.target,h=f.parentNode,p=r.of(f,arguments),g=0,v=n(),d=".drag"+(null==v?"":"-"+v),y=ao.select(e(s)).on(u+d,a).on(o+d,l),m=W(s),M=t(h,v);i?(c=i.apply(f,arguments),c=[c.x-M[0],c.y-M[1]]):c=[0,0],p({type:"dragstart"})}}var r=N(n,"drag","dragstart","dragend"),i=null,u=e(b,ao.mouse,t,"mousemove","mouseup"),o=e(G,ao.touch,m,"touchmove","touchend");return n.origin=function(t){return arguments.length?(i=t,n):i},ao.rebind(n,r,"on")},ao.touches=function(n,t){return arguments.length<2&&(t=k().touches),t?co(t).map(function(t){var e=J(n,t);return e.identifier=t.identifier,e}):[]};var Uo=1e-6,jo=Uo*Uo,Fo=Math.PI,Ho=2*Fo,Oo=Ho-Uo,Io=Fo/2,Yo=Fo/180,Zo=180/Fo,Vo=Math.SQRT2,Xo=2,$o=4;ao.interpolateZoom=function(n,t){var e,r,i=n[0],u=n[1],o=n[
 2],a=t[0],l=t[1],c=t[2],f=a-i,s=l-u,h=f*f+s*s;if(jo>h)r=Math.log(c/o)/Vo,e=function(n){return[i+n*f,u+n*s,o*Math.exp(Vo*n*r)]};else{var p=Math.sqrt(h),g=(c*c-o*o+$o*h)/(2*o*Xo*p),v=(c*c-o*o-$o*h)/(2*c*Xo*p),d=Math.log(Math.sqrt(g*g+1)-g),y=Math.log(Math.sqrt(v*v+1)-v);r=(y-d)/Vo,e=function(n){var t=n*r,e=rn(d),a=o/(Xo*p)*(e*un(Vo*t+d)-en(d));return[i+a*f,u+a*s,o*e/rn(Vo*t+d)]}}return e.duration=1e3*r,e},ao.behavior.zoom=function(){function n(n){n.on(L,s).on(Wo+".zoom",p).on("dblclick.zoom",g).on(R,h)}function e(n){return[(n[0]-k.x)/k.k,(n[1]-k.y)/k.k]}function r(n){return[n[0]*k.k+k.x,n[1]*k.k+k.y]}function i(n){k.k=Math.max(A[0],Math.min(A[1],n))}function u(n,t){t=r(t),k.x+=n[0]-t[0],k.y+=n[1]-t[1]}function o(t,e,r,o){t.__chart__={x:k.x,y:k.y,k:k.k},i(Math.pow(2,o)),u(d=e,r),t=ao.select(t),C>0&&(t=t.transition().duration(C)),t.call(n.event)}function a(){b&&b.domain(x.range().map(function(n){return(n-k.x)/k.k}).map(x.invert)),w&&w.domain(_.range().map(function(n){return(n-k.y)/k.k})
 .map(_.invert))}function l(n){z++||n({type:"zoomstart"})}function c(n){a(),n({type:"zoom",scale:k.k,translate:[k.x,k.y]})}function f(n){--z||(n({type:"zoomend"}),d=null)}function s(){function n(){a=1,u(ao.mouse(i),h),c(o)}function r(){s.on(q,null).on(T,null),p(a),f(o)}var i=this,o=D.of(i,arguments),a=0,s=ao.select(t(i)).on(q,n).on(T,r),h=e(ao.mouse(i)),p=W(i);Il.call(i),l(o)}function h(){function n(){var n=ao.touches(g);return p=k.k,n.forEach(function(n){n.identifier in d&&(d[n.identifier]=e(n))}),n}function t(){var t=ao.event.target;ao.select(t).on(x,r).on(b,a),_.push(t);for(var e=ao.event.changedTouches,i=0,u=e.length;u>i;++i)d[e[i].identifier]=null;var l=n(),c=Date.now();if(1===l.length){if(500>c-M){var f=l[0];o(g,f,d[f.identifier],Math.floor(Math.log(k.k)/Math.LN2)+1),S()}M=c}else if(l.length>1){var f=l[0],s=l[1],h=f[0]-s[0],p=f[1]-s[1];y=h*h+p*p}}function r(){var n,t,e,r,o=ao.touches(g);Il.call(g);for(var a=0,l=o.length;l>a;++a,r=null)if(e=o[a],r=d[e.identifier]){if(t)break;n=e
 ,t=r}if(r){var f=(f=e[0]-n[0])*f+(f=e[1]-n[1])*f,s=y&&Math.sqrt(f/y);n=[(n[0]+e[0])/2,(n[1]+e[1])/2],t=[(t[0]+r[0])/2,(t[1]+r[1])/2],i(s*p)}M=null,u(n,t),c(v)}function a(){if(ao.event.touches.length){for(var t=ao.event.changedTouches,e=0,r=t.length;r>e;++e)delete d[t[e].identifier];for(var i in d)return void n()}ao.selectAll(_).on(m,null),w.on(L,s).on(R,h),N(),f(v)}var p,g=this,v=D.of(g,arguments),d={},y=0,m=".zoom-"+ao.event.changedTouches[0].identifier,x="touchmove"+m,b="touchend"+m,_=[],w=ao.select(g),N=W(g);t(),l(v),w.on(L,null).on(R,t)}function p(){var n=D.of(this,arguments);m?clearTimeout(m):(Il.call(this),v=e(d=y||ao.mouse(this)),l(n)),m=setTimeout(function(){m=null,f(n)},50),S(),i(Math.pow(2,.002*Bo())*k.k),u(d,v),c(n)}function g(){var n=ao.mouse(this),t=Math.log(k.k)/Math.LN2;o(this,n,e(n),ao.event.shiftKey?Math.ceil(t)-1:Math.floor(t)+1)}var v,d,y,m,M,x,b,_,w,k={x:0,y:0,k:1},E=[960,500],A=Jo,C=250,z=0,L="mousedown.zoom",q="mousemove.zoom",T="mouseup.zoom",R="touchstart.zoo
 m",D=N(n,"zoomstart","zoom","zoomend");return Wo||(Wo="onwheel"in fo?(Bo=function(){return-ao.event.deltaY*(ao.event.deltaMode?120:1)},"wheel"):"onmousewheel"in fo?(Bo=function(){return ao.event.wheelDelta},"mousewheel"):(Bo=function(){return-ao.event.detail},"MozMousePixelScroll")),n.event=function(n){n.each(function(){var n=D.of(this,arguments),t=k;Hl?ao.select(this).transition().each("start.zoom",function(){k=this.__chart__||{x:0,y:0,k:1},l(n)}).tween("zoom:zoom",function(){var e=E[0],r=E[1],i=d?d[0]:e/2,u=d?d[1]:r/2,o=ao.interpolateZoom([(i-k.x)/k.k,(u-k.y)/k.k,e/k.k],[(i-t.x)/t.k,(u-t.y)/t.k,e/t.k]);return function(t){var r=o(t),a=e/r[2];this.__chart__=k={x:i-r[0]*a,y:u-r[1]*a,k:a},c(n)}}).each("interrupt.zoom",function(){f(n)}).each("end.zoom",function(){f(n)}):(this.__chart__=k,l(n),c(n),f(n))})},n.translate=function(t){return arguments.length?(k={x:+t[0],y:+t[1],k:k.k},a(),n):[k.x,k.y]},n.scale=function(t){return arguments.length?(k={x:k.x,y:k.y,k:null},i(+t),a(),n):k.k},n.s
 caleExtent=function(t){return arguments.length?(A=null==t?Jo:[+t[0],+t[1]],n):A},n.center=function(t){return arguments.length?(y=t&&[+t[0],+t[1]],n):y},n.size=function(t){return arguments.length?(E=t&&[+t[0],+t[1]],n):E},n.duration=function(t){return arguments.length?(C=+t,n):C},n.x=function(t){return arguments.length?(b=t,x=t.copy(),k={x:0,y:0,k:1},n):b},n.y=function(t){return arguments.length?(w=t,_=t.copy(),k={x:0,y:0,k:1},n):w},ao.rebind(n,D,"on")};var Bo,Wo,Jo=[0,1/0];ao.color=an,an.prototype.toString=function(){return this.rgb()+""},ao.hsl=ln;var Go=ln.prototype=new an;Go.brighter=function(n){return n=Math.pow(.7,arguments.length?n:1),new ln(this.h,this.s,this.l/n)},Go.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),new ln(this.h,this.s,n*this.l)},Go.rgb=function(){return cn(this.h,this.s,this.l)},ao.hcl=fn;var Ko=fn.prototype=new an;Ko.brighter=function(n){return new fn(this.h,this.c,Math.min(100,this.l+Qo*(arguments.length?n:1)))},Ko.darker=function(n){return n
 ew fn(this.h,this.c,Math.max(0,this.l-Qo*(arguments.length?n:1)))},Ko.rgb=function(){return sn(this.h,this.c,this.l).rgb()},ao.lab=hn;var Qo=18,na=.95047,ta=1,ea=1.08883,ra=hn.prototype=new an;ra.brighter=function(n){return new hn(Math.min(100,this.l+Qo*(arguments.length?n:1)),this.a,this.b)},ra.darker=function(n){return new hn(Math.max(0,this.l-Qo*(arguments.length?n:1)),this.a,this.b)},ra.rgb=function(){return pn(this.l,this.a,this.b)},ao.rgb=mn;var ia=mn.prototype=new an;ia.brighter=function(n){n=Math.pow(.7,arguments.length?n:1);var t=this.r,e=this.g,r=this.b,i=30;return t||e||r?(t&&i>t&&(t=i),e&&i>e&&(e=i),r&&i>r&&(r=i),new mn(Math.min(255,t/n),Math.min(255,e/n),Math.min(255,r/n))):new mn(i,i,i)},ia.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),new mn(n*this.r,n*this.g,n*this.b)},ia.hsl=function(){return wn(this.r,this.g,this.b)},ia.toString=function(){return"#"+bn(this.r)+bn(this.g)+bn(this.b)};var ua=ao.map({aliceblue:15792383,antiquewhite:16444375,aqua:65535,
 aquamarine:8388564,azure:15794175,beige:16119260,bisque:16770244,black:0,blanchedalmond:16772045,blue:255,blueviolet:9055202,brown:10824234,burlywood:14596231,cadetblue:6266528,chartreuse:8388352,chocolate:13789470,coral:16744272,cornflowerblue:6591981,cornsilk:16775388,crimson:14423100,cyan:65535,darkblue:139,darkcyan:35723,darkgoldenrod:12092939,darkgray:11119017,darkgreen:25600,darkgrey:11119017,darkkhaki:12433259,darkmagenta:9109643,darkolivegreen:5597999,darkorange:16747520,darkorchid:10040012,darkred:9109504,darksalmon:15308410,darkseagreen:9419919,darkslateblue:4734347,darkslategray:3100495,darkslategrey:3100495,darkturquoise:52945,darkviolet:9699539,deeppink:16716947,deepskyblue:49151,dimgray:6908265,dimgrey:6908265,dodgerblue:2003199,firebrick:11674146,floralwhite:16775920,forestgreen:2263842,fuchsia:16711935,gainsboro:14474460,ghostwhite:16316671,gold:16766720,goldenrod:14329120,gray:8421504,green:32768,greenyellow:11403055,grey:8421504,honeydew:15794

<TRUNCATED>

---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[29/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
deleted file mode 100644
index 994b245..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
+++ /dev/null
@@ -1,281 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.container.common.states.datanode.InitDatanodeState;
-import org.apache.hadoop.ozone.container.common.states.DatanodeState;
-import org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.LinkedList;
-import java.util.Queue;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import static org.apache.hadoop.ozone.OzoneConsts.INVALID_PORT;
-import static org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ReportState.states
-    .noContainerReports;
-
-/**
- * Current Context of State Machine.
- */
-public class StateContext {
-  static final Logger LOG =
-      LoggerFactory.getLogger(StateContext.class);
-  private final Queue<SCMCommand> commandQueue;
-  private final Lock lock;
-  private final DatanodeStateMachine parent;
-  private final AtomicLong stateExecutionCount;
-  private final Configuration conf;
-  private DatanodeStateMachine.DatanodeStates state;
-  private SCMNodeReport nrState;
-  private ReportState  reportState;
-  private static final ReportState DEFAULT_REPORT_STATE =
-      ReportState.newBuilder().setState(noContainerReports).setCount(0).build();
-
-  /**
-   * Constructs a StateContext.
-   *
-   * @param conf   - Configration
-   * @param state  - State
-   * @param parent Parent State Machine
-   */
-  public StateContext(Configuration conf, DatanodeStateMachine.DatanodeStates
-      state, DatanodeStateMachine parent) {
-    this.conf = conf;
-    this.state = state;
-    this.parent = parent;
-    commandQueue = new LinkedList<>();
-    lock = new ReentrantLock();
-    stateExecutionCount = new AtomicLong(0);
-    nrState = SCMNodeReport.getDefaultInstance();
-  }
-
-  /**
-   * Returns the ContainerStateMachine class that holds this state.
-   *
-   * @return ContainerStateMachine.
-   */
-  public DatanodeStateMachine getParent() {
-    return parent;
-  }
-
-  /**
-   * Get the container server port.
-   * @return The container server port if available, return -1 if otherwise
-   */
-  public int getContainerPort() {
-    return parent == null ?
-        INVALID_PORT : parent.getContainer().getContainerServerPort();
-  }
-
-  /**
-   * Gets the Ratis Port.
-   * @return int , return -1 if not valid.
-   */
-  public int getRatisPort() {
-    return parent == null ?
-        INVALID_PORT : parent.getContainer().getRatisContainerServerPort();
-  }
-
-  /**
-   * Returns true if we are entering a new state.
-   *
-   * @return boolean
-   */
-  boolean isEntering() {
-    return stateExecutionCount.get() == 0;
-  }
-
-  /**
-   * Returns true if we are exiting from the current state.
-   *
-   * @param newState - newState.
-   * @return boolean
-   */
-  boolean isExiting(DatanodeStateMachine.DatanodeStates newState) {
-    boolean isExiting = state != newState && stateExecutionCount.get() > 0;
-    if(isExiting) {
-      stateExecutionCount.set(0);
-    }
-    return isExiting;
-  }
-
-  /**
-   * Returns the current state the machine is in.
-   *
-   * @return state.
-   */
-  public DatanodeStateMachine.DatanodeStates getState() {
-    return state;
-  }
-
-  /**
-   * Sets the current state of the machine.
-   *
-   * @param state state.
-   */
-  public void setState(DatanodeStateMachine.DatanodeStates state) {
-    this.state = state;
-  }
-
-  /**
-   * Returns the node report of the datanode state context.
-   * @return the node report.
-   */
-  public SCMNodeReport getNodeReport() {
-    return nrState;
-  }
-
-  /**
-   * Sets the storage location report of the datanode state context.
-   * @param nrReport - node report
-   */
-  public void setReportState(SCMNodeReport nrReport) {
-    this.nrState = nrReport;
-  }
-
-  /**
-   * Returns the next task to get executed by the datanode state machine.
-   * @return A callable that will be executed by the
-   * {@link DatanodeStateMachine}
-   */
-  @SuppressWarnings("unchecked")
-  public DatanodeState<DatanodeStateMachine.DatanodeStates> getTask() {
-    switch (this.state) {
-    case INIT:
-      return new InitDatanodeState(this.conf, parent.getConnectionManager(),
-          this);
-    case RUNNING:
-      return new RunningDatanodeState(this.conf, parent.getConnectionManager(),
-          this);
-    case SHUTDOWN:
-      return null;
-    default:
-      throw new IllegalArgumentException("Not Implemented yet.");
-    }
-  }
-
-  /**
-   * Executes the required state function.
-   *
-   * @param service - Executor Service
-   * @param time    - seconds to wait
-   * @param unit    - Seconds.
-   * @throws InterruptedException
-   * @throws ExecutionException
-   * @throws TimeoutException
-   */
-  public void execute(ExecutorService service, long time, TimeUnit unit)
-      throws InterruptedException, ExecutionException, TimeoutException {
-    stateExecutionCount.incrementAndGet();
-    DatanodeState<DatanodeStateMachine.DatanodeStates> task = getTask();
-    if (this.isEntering()) {
-      task.onEnter();
-    }
-    task.execute(service);
-    DatanodeStateMachine.DatanodeStates newState = task.await(time, unit);
-    if (this.state != newState) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Task {} executed, state transited from {} to {}",
-            task.getClass().getSimpleName(), this.state, newState);
-      }
-      if (isExiting(newState)) {
-        task.onExit();
-      }
-      this.clearReportState();
-      this.setState(newState);
-    }
-  }
-
-  /**
-   * Returns the next command or null if it is empty.
-   *
-   * @return SCMCommand or Null.
-   */
-  public SCMCommand getNextCommand() {
-    lock.lock();
-    try {
-      return commandQueue.poll();
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  /**
-   * Adds a command to the State Machine queue.
-   *
-   * @param command - SCMCommand.
-   */
-  public void addCommand(SCMCommand command) {
-    lock.lock();
-    try {
-      commandQueue.add(command);
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  /**
-   * Returns the count of the Execution.
-   * @return long
-   */
-  public long getExecutionCount() {
-    return stateExecutionCount.get();
-  }
-
-
-  /**
-   * Gets the ReportState.
-   * @return ReportState.
-   */
-  public synchronized  ReportState getContainerReportState() {
-    if (reportState == null) {
-      return DEFAULT_REPORT_STATE;
-    }
-    return reportState;
-  }
-
-  /**
-   * Sets the ReportState.
-   * @param rState - ReportState.
-   */
-  public synchronized  void setContainerReportState(ReportState rState) {
-    this.reportState = rState;
-  }
-
-  /**
-   * Clears report state after it has been communicated.
-   */
-  public synchronized void clearReportState() {
-    if(reportState != null) {
-      setContainerReportState(null);
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
deleted file mode 100644
index 80844cf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.container.common.statemachine.background;
-
-import com.google.common.collect.Lists;
-import com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.BackgroundService;
-import org.apache.hadoop.utils.BackgroundTaskResult;
-import org.apache.hadoop.utils.BackgroundTaskQueue;
-import org.apache.hadoop.utils.BackgroundTask;
-import org.apache.hadoop.utils.BatchOperation;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.List;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT;
-
-/**
- * A per-datanode container block deleting service takes in charge
- * of deleting staled ozone blocks.
- */
-public class BlockDeletingService extends BackgroundService{
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(BlockDeletingService.class);
-
-  private final ContainerManager containerManager;
-  private final Configuration conf;
-
-  // Throttle number of blocks to delete per task,
-  // set to 1 for testing
-  private final int blockLimitPerTask;
-
-  // Throttle the number of containers to process concurrently at a time,
-  private final int containerLimitPerInterval;
-
-  // Task priority is useful when a to-delete block has weight.
-  private final static int TASK_PRIORITY_DEFAULT = 1;
-  // Core pool size for container tasks
-  private final static int BLOCK_DELETING_SERVICE_CORE_POOL_SIZE = 10;
-
-  public BlockDeletingService(ContainerManager containerManager,
-      long serviceInterval, long serviceTimeout, Configuration conf) {
-    super("BlockDeletingService", serviceInterval,
-        TimeUnit.MILLISECONDS, BLOCK_DELETING_SERVICE_CORE_POOL_SIZE,
-        serviceTimeout);
-    this.containerManager = containerManager;
-    this.conf = conf;
-    this.blockLimitPerTask = conf.getInt(
-        OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER,
-        OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER_DEFAULT);
-    this.containerLimitPerInterval = conf.getInt(
-        OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL,
-        OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT);
-  }
-
-  @Override
-  public BackgroundTaskQueue getTasks() {
-    BackgroundTaskQueue queue = new BackgroundTaskQueue();
-    List<ContainerData> containers = Lists.newArrayList();
-    try {
-      // We at most list a number of containers a time,
-      // in case there are too many containers and start too many workers.
-      // We must ensure there is no empty container in this result.
-      // The chosen result depends on what container deletion policy is
-      // configured.
-      containers = containerManager.chooseContainerForBlockDeletion(
-          containerLimitPerInterval);
-      LOG.info("Plan to choose {} containers for block deletion, "
-          + "actually returns {} valid containers.",
-          containerLimitPerInterval, containers.size());
-
-      for(ContainerData container : containers) {
-        BlockDeletingTask containerTask =
-            new BlockDeletingTask(container, TASK_PRIORITY_DEFAULT);
-        queue.add(containerTask);
-      }
-    } catch (StorageContainerException e) {
-      LOG.warn("Failed to initiate block deleting tasks, "
-          + "caused by unable to get containers info. "
-          + "Retry in next interval. ", e);
-    } catch (Exception e) {
-      // In case listContainer call throws any uncaught RuntimeException.
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Unexpected error occurs during deleting blocks.", e);
-      }
-    }
-    return queue;
-  }
-
-  private static class ContainerBackgroundTaskResult
-      implements BackgroundTaskResult {
-    private List<String> deletedBlockIds;
-
-    ContainerBackgroundTaskResult() {
-      deletedBlockIds = new LinkedList<>();
-    }
-
-    public void addBlockId(String blockId) {
-      deletedBlockIds.add(blockId);
-    }
-
-    public void addAll(List<String> blockIds) {
-      deletedBlockIds.addAll(blockIds);
-    }
-
-    public List<String> getDeletedBlocks() {
-      return deletedBlockIds;
-    }
-
-    @Override
-    public int getSize() {
-      return deletedBlockIds.size();
-    }
-  }
-
-  private class BlockDeletingTask
-      implements BackgroundTask<BackgroundTaskResult> {
-
-    private final int priority;
-    private final ContainerData containerData;
-
-    BlockDeletingTask(ContainerData containerName, int priority) {
-      this.priority = priority;
-      this.containerData = containerName;
-    }
-
-    @Override
-    public BackgroundTaskResult call() throws Exception {
-      ContainerBackgroundTaskResult crr = new ContainerBackgroundTaskResult();
-      long startTime = Time.monotonicNow();
-      // Scan container's db and get list of under deletion blocks
-      MetadataStore meta = KeyUtils.getDB(containerData, conf);
-      // # of blocks to delete is throttled
-      KeyPrefixFilter filter = new KeyPrefixFilter(
-          OzoneConsts.DELETING_KEY_PREFIX);
-      List<Map.Entry<byte[], byte[]>> toDeleteBlocks =
-          meta.getSequentialRangeKVs(null, blockLimitPerTask, filter);
-      if (toDeleteBlocks.isEmpty()) {
-        LOG.debug("No under deletion block found in container : {}",
-            containerData.getContainerName());
-      }
-
-      List<String> succeedBlocks = new LinkedList<>();
-      LOG.debug("Container : {}, To-Delete blocks : {}",
-          containerData.getContainerName(), toDeleteBlocks.size());
-      File dataDir = ContainerUtils.getDataDirectory(containerData).toFile();
-      if (!dataDir.exists() || !dataDir.isDirectory()) {
-        LOG.error("Invalid container data dir {} : "
-            + "not exist or not a directory", dataDir.getAbsolutePath());
-        return crr;
-      }
-
-      toDeleteBlocks.forEach(entry -> {
-        String blockName = DFSUtil.bytes2String(entry.getKey());
-        LOG.debug("Deleting block {}", blockName);
-        try {
-          ContainerProtos.KeyData data =
-              ContainerProtos.KeyData.parseFrom(entry.getValue());
-          for (ContainerProtos.ChunkInfo chunkInfo : data.getChunksList()) {
-            File chunkFile = dataDir.toPath()
-                .resolve(chunkInfo.getChunkName()).toFile();
-            if (FileUtils.deleteQuietly(chunkFile)) {
-              LOG.debug("block {} chunk {} deleted", blockName,
-                  chunkFile.getAbsolutePath());
-            }
-          }
-          succeedBlocks.add(blockName);
-        } catch (InvalidProtocolBufferException e) {
-          LOG.error("Failed to parse block info for block {}", blockName, e);
-        }
-      });
-
-      // Once files are deleted ... clean up DB
-      BatchOperation batch = new BatchOperation();
-      succeedBlocks.forEach(entry ->
-          batch.delete(DFSUtil.string2Bytes(entry)));
-      meta.writeBatch(batch);
-      // update count of pending deletion blocks in in-memory container status
-      containerManager.decrPendingDeletionBlocks(succeedBlocks.size(),
-          containerData.getContainerName());
-
-      if (!succeedBlocks.isEmpty()) {
-        LOG.info("Container: {}, deleted blocks: {}, task elapsed time: {}ms",
-            containerData.getContainerName(), succeedBlocks.size(),
-            Time.monotonicNow() - startTime);
-      }
-      crr.addAll(succeedBlocks);
-      return crr;
-    }
-
-    @Override
-    public int getPriority() {
-      return priority;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/package-info.java
deleted file mode 100644
index a9e202e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine.background;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerHandler.java
deleted file mode 100644
index f5fbcd7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerHandler.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-
-import org.apache.hadoop.ozone.container.common.statemachine
-    .SCMConnectionManager;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMCloseContainerCmdResponseProto;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.Type;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Container Report handler.
- */
-public class CloseContainerHandler implements CommandHandler {
-  static final Logger LOG =
-      LoggerFactory.getLogger(CloseContainerHandler.class);
-  private int invocationCount;
-  private long totalTime;
-
-  /**
-   * Constructs a ContainerReport handler.
-   */
-  public CloseContainerHandler() {
-  }
-
-  /**
-   * Handles a given SCM command.
-   *
-   * @param command           - SCM Command
-   * @param container         - Ozone Container.
-   * @param context           - Current Context.
-   * @param connectionManager - The SCMs that we are talking to.
-   */
-  @Override
-  public void handle(SCMCommand command, OzoneContainer container,
-      StateContext context, SCMConnectionManager connectionManager) {
-    LOG.debug("Processing Close Container command.");
-    invocationCount++;
-    long startTime = Time.monotonicNow();
-    String containerName = "UNKNOWN";
-    try {
-
-      SCMCloseContainerCmdResponseProto
-          closeContainerProto =
-          SCMCloseContainerCmdResponseProto
-              .parseFrom(command.getProtoBufMessage());
-      containerName = closeContainerProto.getContainerName();
-
-      container.getContainerManager().closeContainer(containerName);
-
-    } catch (Exception e) {
-      LOG.error("Can't close container " + containerName, e);
-    } finally {
-      long endTime = Time.monotonicNow();
-      totalTime += endTime - startTime;
-    }
-  }
-
-  /**
-   * Returns the command type that this command handler handles.
-   *
-   * @return Type
-   */
-  @Override
-  public Type getCommandType() {
-    return Type.closeContainerCommand;
-  }
-
-  /**
-   * Returns number of times this handler has been invoked.
-   *
-   * @return int
-   */
-  @Override
-  public int getInvocationCount() {
-    return invocationCount;
-  }
-
-  /**
-   * Returns the average time this function takes to run.
-   *
-   * @return long
-   */
-  @Override
-  public long getAverageRunTime() {
-    if (invocationCount > 0) {
-      return totalTime / invocationCount;
-    }
-    return 0;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandDispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandDispatcher.java
deleted file mode 100644
index 6670ea9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandDispatcher.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.Type;
-import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Dispatches command to the correct handler.
- */
-public final class CommandDispatcher {
-  static final Logger LOG =
-      LoggerFactory.getLogger(CommandDispatcher.class);
-  private final StateContext context;
-  private final Map<Type, CommandHandler> handlerMap;
-  private final OzoneContainer container;
-  private final SCMConnectionManager connectionManager;
-
-  /**
-   * Constructs a command Dispatcher.
-   * @param context - Context.
-   */
-  /**
-   * Constructs a command dispatcher.
-   *
-   * @param container - Ozone Container
-   * @param context - Context
-   * @param handlers - Set of handlers.
-   */
-  private CommandDispatcher(OzoneContainer container, SCMConnectionManager
-      connectionManager, StateContext context,
-      CommandHandler... handlers) {
-    Preconditions.checkNotNull(context);
-    Preconditions.checkNotNull(handlers);
-    Preconditions.checkArgument(handlers.length > 0);
-    Preconditions.checkNotNull(container);
-    Preconditions.checkNotNull(connectionManager);
-    this.context = context;
-    this.container = container;
-    this.connectionManager = connectionManager;
-    handlerMap = new HashMap<>();
-    for (CommandHandler h : handlers) {
-      if(handlerMap.containsKey(h.getCommandType())){
-        LOG.error("Duplicate handler for the same command. Exiting. Handle " +
-            "key : { }", h.getCommandType().getDescriptorForType().getName());
-        throw new IllegalArgumentException("Duplicate handler for the same " +
-            "command.");
-      }
-      handlerMap.put(h.getCommandType(), h);
-    }
-  }
-
-  /**
-   * Dispatch the command to the correct handler.
-   *
-   * @param command - SCM Command.
-   */
-  public void handle(SCMCommand command) {
-    Preconditions.checkNotNull(command);
-    CommandHandler handler = handlerMap.get(command.getType());
-    if (handler != null) {
-      handler.handle(command, container, context, connectionManager);
-    } else {
-      LOG.error("Unknown SCM Command queued. There is no handler for this " +
-          "command. Command: {}", command.getType().getDescriptorForType()
-          .getName());
-    }
-  }
-
-  public static Builder newBuilder() {
-    return new Builder();
-  }
-
-  /**
-   * Helper class to construct command dispatcher.
-   */
-  public static class Builder {
-    private final List<CommandHandler> handlerList;
-    private OzoneContainer container;
-    private StateContext context;
-    private SCMConnectionManager connectionManager;
-
-    public Builder() {
-      handlerList = new LinkedList<>();
-    }
-
-    /**
-     * Adds a handler.
-     *
-     * @param handler - handler
-     * @return Builder
-     */
-    public Builder addHandler(CommandHandler handler) {
-      Preconditions.checkNotNull(handler);
-      handlerList.add(handler);
-      return this;
-    }
-
-    /**
-     * Add the OzoneContainer.
-     *
-     * @param ozoneContainer - ozone container.
-     * @return Builder
-     */
-    public Builder setContainer(OzoneContainer ozoneContainer) {
-      Preconditions.checkNotNull(ozoneContainer);
-      this.container = ozoneContainer;
-      return this;
-    }
-
-    /**
-     * Set the Connection Manager.
-     *
-     * @param scmConnectionManager
-     * @return this
-     */
-    public Builder setConnectionManager(SCMConnectionManager
-        scmConnectionManager) {
-      Preconditions.checkNotNull(scmConnectionManager);
-      this.connectionManager = scmConnectionManager;
-      return this;
-    }
-
-    /**
-     * Sets the Context.
-     *
-     * @param stateContext - StateContext
-     * @return this
-     */
-    public Builder setContext(StateContext stateContext) {
-      Preconditions.checkNotNull(stateContext);
-      this.context = stateContext;
-      return this;
-    }
-
-    /**
-     * Builds a command Dispatcher.
-     * @return Command Dispatcher.
-     */
-    public CommandDispatcher build() {
-      Preconditions.checkNotNull(this.connectionManager, "Missing connection" +
-          " manager.");
-      Preconditions.checkNotNull(this.container, "Missing container.");
-      Preconditions.checkNotNull(this.context, "Missing context.");
-      Preconditions.checkArgument(this.handlerList.size() > 0);
-      return new CommandDispatcher(this.container, this.connectionManager,
-          this.context, handlerList.toArray(
-              new CommandHandler[handlerList.size()]));
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandHandler.java
deleted file mode 100644
index b30d481..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandHandler.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.Type;
-import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-
-/**
- * Generic interface for handlers.
- */
-public interface CommandHandler {
-
-  /**
-   * Handles a given SCM command.
-   * @param command - SCM Command
-   * @param container - Ozone Container.
-   * @param context - Current Context.
-   * @param connectionManager - The SCMs that we are talking to.
-   */
-  void handle(SCMCommand command, OzoneContainer container,
-      StateContext context, SCMConnectionManager connectionManager);
-
-  /**
-   * Returns the command type that this command handler handles.
-   * @return Type
-   */
-  Type getCommandType();
-
-  /**
-   * Returns number of times this handler has been invoked.
-   * @return int
-   */
-  int getInvocationCount();
-
-  /**
-   * Returns the average time this function takes to run.
-   * @return  long
-   */
-  long getAverageRunTime();
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java
deleted file mode 100644
index 57a1516..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-
-import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.Type;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-/**
- * Container Report handler.
- */
-public class ContainerReportHandler implements CommandHandler {
-  static final Logger LOG =
-      LoggerFactory.getLogger(ContainerReportHandler.class);
-  private int invocationCount;
-  private long totalTime;
-
-  /**
-   * Constructs a ContainerReport handler.
-   */
-  public ContainerReportHandler() {
-  }
-
-  /**
-   * Handles a given SCM command.
-   *
-   * @param command - SCM Command
-   * @param container - Ozone Container.
-   * @param context - Current Context.
-   * @param connectionManager - The SCMs that we are talking to.
-   */
-  @Override
-  public void handle(SCMCommand command, OzoneContainer container,
-      StateContext context, SCMConnectionManager connectionManager) {
-    LOG.debug("Processing Container Report.");
-    invocationCount++;
-    long startTime = Time.monotonicNow();
-    try {
-      ContainerReportsRequestProto contianerReport =
-          container.getContainerReport();
-
-      // TODO : We send this report to all SCMs.Check if it is enough only to
-      // send to the leader once we have RAFT enabled SCMs.
-      for (EndpointStateMachine endPoint : connectionManager.getValues()) {
-        endPoint.getEndPoint().sendContainerReport(contianerReport);
-      }
-    } catch (IOException ex) {
-      LOG.error("Unable to process the Container Report command.", ex);
-    } finally {
-      long endTime = Time.monotonicNow();
-      totalTime += endTime - startTime;
-    }
-  }
-
-  /**
-   * Returns the command type that this command handler handles.
-   *
-   * @return Type
-   */
-  @Override
-  public Type getCommandType() {
-    return Type.sendContainerReport;
-  }
-
-  /**
-   * Returns number of times this handler has been invoked.
-   *
-   * @return int
-   */
-  @Override
-  public int getInvocationCount() {
-    return invocationCount;
-  }
-
-  /**
-   * Returns the average time this function takes to run.
-   *
-   * @return long
-   */
-  @Override
-  public long getAverageRunTime() {
-    if (invocationCount > 0) {
-      return totalTime / invocationCount;
-    }
-    return 0;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
deleted file mode 100644
index 1c859ea..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.helpers.DeletedContainerBlocksSummary;
-import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
-import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto.DeleteBlockTransactionResult;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.Type;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.BatchOperation;
-import org.apache.hadoop.utils.MetadataStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Handle block deletion commands.
- */
-public class DeleteBlocksCommandHandler implements CommandHandler {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(DeleteBlocksCommandHandler.class);
-
-  private ContainerManager containerManager;
-  private Configuration conf;
-  private int invocationCount;
-  private long totalTime;
-
-  public DeleteBlocksCommandHandler(ContainerManager containerManager,
-      Configuration conf) {
-    this.containerManager = containerManager;
-    this.conf = conf;
-  }
-
-  @Override
-  public void handle(SCMCommand command, OzoneContainer container,
-      StateContext context, SCMConnectionManager connectionManager) {
-    if (command.getType() != Type.deleteBlocksCommand) {
-      LOG.warn("Skipping handling command, expected command "
-              + "type {} but found {}",
-          Type.deleteBlocksCommand, command.getType());
-      return;
-    }
-    LOG.debug("Processing block deletion command.");
-    invocationCount++;
-    long startTime = Time.monotonicNow();
-
-    // move blocks to deleting state.
-    // this is a metadata update, the actual deletion happens in another
-    // recycling thread.
-    DeleteBlocksCommand cmd = (DeleteBlocksCommand) command;
-    List<DeletedBlocksTransaction> containerBlocks = cmd.blocksTobeDeleted();
-
-
-    DeletedContainerBlocksSummary summary =
-        DeletedContainerBlocksSummary.getFrom(containerBlocks);
-    LOG.info("Start to delete container blocks, TXIDs={}, "
-            + "numOfContainers={}, numOfBlocks={}",
-        summary.getTxIDSummary(),
-        summary.getNumOfContainers(),
-        summary.getNumOfBlocks());
-
-    ContainerBlocksDeletionACKProto.Builder resultBuilder =
-        ContainerBlocksDeletionACKProto.newBuilder();
-    containerBlocks.forEach(entry -> {
-      DeleteBlockTransactionResult.Builder txResultBuilder =
-          DeleteBlockTransactionResult.newBuilder();
-      txResultBuilder.setTxID(entry.getTxID());
-      try {
-        deleteContainerBlocks(entry, conf);
-        txResultBuilder.setSuccess(true);
-      } catch (IOException e) {
-        LOG.warn("Failed to delete blocks for container={}, TXID={}",
-            entry.getContainerName(), entry.getTxID(), e);
-        txResultBuilder.setSuccess(false);
-      }
-      resultBuilder.addResults(txResultBuilder.build());
-    });
-    ContainerBlocksDeletionACKProto blockDeletionACK = resultBuilder.build();
-
-    // Send ACK back to SCM as long as meta updated
-    // TODO Or we should wait until the blocks are actually deleted?
-    if (!containerBlocks.isEmpty()) {
-      for (EndpointStateMachine endPoint : connectionManager.getValues()) {
-        try {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Sending following block deletion ACK to SCM");
-            for (DeleteBlockTransactionResult result :
-                blockDeletionACK.getResultsList()) {
-              LOG.debug(result.getTxID() + " : " + result.getSuccess());
-            }
-          }
-          endPoint.getEndPoint()
-              .sendContainerBlocksDeletionACK(blockDeletionACK);
-        } catch (IOException e) {
-          LOG.error("Unable to send block deletion ACK to SCM {}",
-              endPoint.getAddress().toString(), e);
-        }
-      }
-    }
-
-    long endTime = Time.monotonicNow();
-    totalTime += endTime - startTime;
-  }
-
-  /**
-   * Move a bunch of blocks from a container to deleting state.
-   * This is a meta update, the actual deletes happen in async mode.
-   *
-   * @param delTX a block deletion transaction.
-   * @param config configuration.
-   * @throws IOException if I/O error occurs.
-   */
-  private void deleteContainerBlocks(DeletedBlocksTransaction delTX,
-      Configuration config) throws IOException {
-    String containerId = delTX.getContainerName();
-    ContainerData containerInfo = containerManager.readContainer(containerId);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Processing Container : {}, DB path : {}", containerId,
-          containerInfo.getDBPath());
-    }
-
-    int newDeletionBlocks = 0;
-    MetadataStore containerDB = KeyUtils.getDB(containerInfo, config);
-    for (String blk : delTX.getBlockIDList()) {
-      BatchOperation batch = new BatchOperation();
-      byte[] blkBytes = DFSUtil.string2Bytes(blk);
-      byte[] blkInfo = containerDB.get(blkBytes);
-      if (blkInfo != null) {
-        // Found the block in container db,
-        // use an atomic update to change its state to deleting.
-        batch.put(DFSUtil.string2Bytes(OzoneConsts.DELETING_KEY_PREFIX + blk),
-            blkInfo);
-        batch.delete(blkBytes);
-        try {
-          containerDB.writeBatch(batch);
-          newDeletionBlocks++;
-          LOG.debug("Transited Block {} to DELETING state in container {}",
-              blk, containerId);
-        } catch (IOException e) {
-          // if some blocks failed to delete, we fail this TX,
-          // without sending this ACK to SCM, SCM will resend the TX
-          // with a certain number of retries.
-          throw new IOException(
-              "Failed to delete blocks for TXID = " + delTX.getTxID(), e);
-        }
-      } else {
-        LOG.debug("Block {} not found or already under deletion in"
-                + " container {}, skip deleting it.", blk, containerId);
-      }
-    }
-
-    // update pending deletion blocks count in in-memory container status
-    containerManager.incrPendingDeletionBlocks(newDeletionBlocks, containerId);
-  }
-
-  @Override
-  public Type getCommandType() {
-    return Type.deleteBlocksCommand;
-  }
-
-  @Override
-  public int getInvocationCount() {
-    return this.invocationCount;
-  }
-
-  @Override
-  public long getAverageRunTime() {
-    if (invocationCount > 0) {
-      return totalTime / invocationCount;
-    }
-    return 0;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/package-info.java
deleted file mode 100644
index 1e9c8dc..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/package-info.java
deleted file mode 100644
index feb2f81..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine;
-/**
-
- State machine class is used by the container to denote various states a
- container can be in and also is used for command processing.
-
- Container has the following states.
-
- Start - > getVersion -> Register -> Running  -> Shutdown
-
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/DatanodeState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/DatanodeState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/DatanodeState.java
deleted file mode 100644
index 75142af..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/DatanodeState.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.states;
-
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-/**
- * State Interface that allows tasks to maintain states.
- */
-public interface DatanodeState<T> {
-  /**
-   * Called before entering this state.
-   */
-  void onEnter();
-
-  /**
-   * Called After exiting this state.
-   */
-  void onExit();
-
-  /**
-   * Executes one or more tasks that is needed by this state.
-   *
-   * @param executor -  ExecutorService
-   */
-  void execute(ExecutorService executor);
-
-  /**
-   * Wait for execute to finish.
-   *
-   * @param time - Time
-   * @param timeUnit - Unit of time.
-   */
-  T await(long time, TimeUnit timeUnit)
-      throws InterruptedException, ExecutionException, TimeoutException;
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
deleted file mode 100644
index 6ca94bf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.states.datanode;
-
-import com.google.common.base.Strings;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.common.states.DatanodeState;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.Collection;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-/**
- * Init Datanode State is the task that gets run when we are in Init State.
- */
-public class InitDatanodeState implements DatanodeState,
-    Callable<DatanodeStateMachine.DatanodeStates> {
-  static final Logger LOG = LoggerFactory.getLogger(InitDatanodeState.class);
-  private final SCMConnectionManager connectionManager;
-  private final Configuration conf;
-  private final StateContext context;
-  private Future<DatanodeStateMachine.DatanodeStates> result;
-
-  /**
-   *  Create InitDatanodeState Task.
-   *
-   * @param conf - Conf
-   * @param connectionManager - Connection Manager
-   * @param context - Current Context
-   */
-  public InitDatanodeState(Configuration conf,
-                           SCMConnectionManager connectionManager,
-                           StateContext context) {
-    this.conf = conf;
-    this.connectionManager = connectionManager;
-    this.context = context;
-  }
-
-  /**
-   * Computes a result, or throws an exception if unable to do so.
-   *
-   * @return computed result
-   * @throws Exception if unable to compute a result
-   */
-  @Override
-  public DatanodeStateMachine.DatanodeStates call() throws Exception {
-    Collection<InetSocketAddress> addresses = null;
-    try {
-      addresses = OzoneClientUtils.getSCMAddresses(conf);
-    } catch (IllegalArgumentException e) {
-      if(!Strings.isNullOrEmpty(e.getMessage())) {
-        LOG.error("Failed to get SCM addresses: " + e.getMessage());
-      }
-      return DatanodeStateMachine.DatanodeStates.SHUTDOWN;
-    }
-
-    if (addresses == null || addresses.isEmpty()) {
-      LOG.error("Null or empty SCM address list found.");
-      return DatanodeStateMachine.DatanodeStates.SHUTDOWN;
-    } else {
-      for (InetSocketAddress addr : addresses) {
-        connectionManager.addSCMServer(addr);
-      }
-    }
-
-    // If datanode ID is set, persist it to the ID file.
-    persistContainerDatanodeID();
-
-    return this.context.getState().getNextState();
-  }
-
-  /**
-   * Update Ozone container port to the datanode ID,
-   * and persist the ID to a local file.
-   */
-  private void persistContainerDatanodeID() throws IOException {
-    String dataNodeIDPath = OzoneUtils.getDatanodeIDPath(conf);
-    if (Strings.isNullOrEmpty(dataNodeIDPath)) {
-      LOG.error("A valid file path is needed for config setting {}",
-          ScmConfigKeys.OZONE_SCM_DATANODE_ID);
-      this.context.setState(DatanodeStateMachine.DatanodeStates.SHUTDOWN);
-      return;
-    }
-    File idPath = new File(dataNodeIDPath);
-    int containerPort = this.context.getContainerPort();
-    int ratisPort = this.context.getRatisPort();
-    DatanodeID datanodeID = this.context.getParent().getDatanodeID();
-    if (datanodeID != null) {
-      datanodeID.setContainerPort(containerPort);
-      datanodeID.setRatisPort(ratisPort);
-      ContainerUtils.writeDatanodeIDTo(datanodeID, idPath);
-      LOG.info("Datanode ID is persisted to {}", dataNodeIDPath);
-    }
-  }
-
-  /**
-   * Called before entering this state.
-   */
-  @Override
-  public void onEnter() {
-    LOG.trace("Entering init container state");
-  }
-
-  /**
-   * Called After exiting this state.
-   */
-  @Override
-  public void onExit() {
-    LOG.trace("Exiting init container state");
-  }
-
-  /**
-   * Executes one or more tasks that is needed by this state.
-   *
-   * @param executor -  ExecutorService
-   */
-  @Override
-  public void execute(ExecutorService executor) {
-    result = executor.submit(this);
-  }
-
-  /**
-   * Wait for execute to finish.
-   *
-   * @param time     - Time
-   * @param timeUnit - Unit of time.
-   */
-  @Override
-  public DatanodeStateMachine.DatanodeStates await(long time,
-      TimeUnit timeUnit) throws InterruptedException,
-      ExecutionException, TimeoutException {
-    return result.get(time, timeUnit);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java
deleted file mode 100644
index fc9af5b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.states.datanode;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.common.states.DatanodeState;
-import org.apache.hadoop.ozone.container.common.states.endpoint.HeartbeatEndpointTask;
-import org.apache.hadoop.ozone.container.common.states.endpoint.RegisterEndpointTask;
-import org.apache.hadoop.ozone.container.common.states.endpoint.VersionEndpointTask;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-/**
- * Class that implements handshake with SCM.
- */
-public class RunningDatanodeState implements DatanodeState {
-  static final Logger
-      LOG = LoggerFactory.getLogger(RunningDatanodeState.class);
-  private final SCMConnectionManager connectionManager;
-  private final Configuration conf;
-  private final StateContext context;
-  private CompletionService<EndpointStateMachine.EndPointStates> ecs;
-
-  public RunningDatanodeState(Configuration conf,
-      SCMConnectionManager connectionManager,
-      StateContext context) {
-    this.connectionManager = connectionManager;
-    this.conf = conf;
-    this.context = context;
-  }
-
-  /**
-   * Reads a datanode ID from the persisted information.
-   *
-   * @param idPath - Path to the ID File.
-   * @return DatanodeID
-   * @throws IOException
-   */
-  private StorageContainerDatanodeProtocolProtos.ContainerNodeIDProto
-      readPersistedDatanodeID(Path idPath) throws IOException {
-    Preconditions.checkNotNull(idPath);
-    DatanodeID datanodeID = null;
-    List<DatanodeID> datanodeIDs =
-        ContainerUtils.readDatanodeIDsFrom(idPath.toFile());
-    int containerPort = this.context.getContainerPort();
-    for(DatanodeID dnId : datanodeIDs) {
-      if(dnId.getContainerPort() == containerPort) {
-        datanodeID = dnId;
-        break;
-      }
-    }
-
-    if (datanodeID == null) {
-      throw new IOException("No valid datanode ID found from "
-          + idPath.toFile().getAbsolutePath()
-          + " that matches container port "
-          + containerPort);
-    } else {
-      StorageContainerDatanodeProtocolProtos.ContainerNodeIDProto
-          containerIDProto =
-          StorageContainerDatanodeProtocolProtos
-              .ContainerNodeIDProto
-              .newBuilder()
-              .setDatanodeID(datanodeID.getProtoBufMessage())
-              .build();
-      return containerIDProto;
-    }
-  }
-
-  /**
-   * Returns ContainerNodeIDProto or null in case of Error.
-   *
-   * @return ContainerNodeIDProto
-   */
-  private StorageContainerDatanodeProtocolProtos.ContainerNodeIDProto
-      getContainerNodeID() {
-    String dataNodeIDPath = OzoneUtils.getDatanodeIDPath(conf);
-    if (dataNodeIDPath == null || dataNodeIDPath.isEmpty()) {
-      LOG.error("A valid file path is needed for config setting {}",
-          ScmConfigKeys.OZONE_SCM_DATANODE_ID);
-
-      // This is an unrecoverable error.
-      this.context.setState(DatanodeStateMachine.DatanodeStates.SHUTDOWN);
-      return null;
-    }
-    StorageContainerDatanodeProtocolProtos.ContainerNodeIDProto nodeID;
-    // try to read an existing ContainerNode ID.
-    try {
-      nodeID = readPersistedDatanodeID(Paths.get(dataNodeIDPath));
-      if (nodeID != null) {
-        LOG.trace("Read Node ID :", nodeID.getDatanodeID().getDatanodeUuid());
-        return nodeID;
-      }
-    } catch (IOException ex) {
-      LOG.trace("Not able to find container Node ID, creating it.", ex);
-    }
-    this.context.setState(DatanodeStateMachine.DatanodeStates.SHUTDOWN);
-    return null;
-  }
-
-  /**
-   * Called before entering this state.
-   */
-  @Override
-  public void onEnter() {
-    LOG.trace("Entering handshake task.");
-  }
-
-  /**
-   * Called After exiting this state.
-   */
-  @Override
-  public void onExit() {
-    LOG.trace("Exiting handshake task.");
-  }
-
-  /**
-   * Executes one or more tasks that is needed by this state.
-   *
-   * @param executor -  ExecutorService
-   */
-  @Override
-  public void execute(ExecutorService executor) {
-    ecs = new ExecutorCompletionService<>(executor);
-    for (EndpointStateMachine endpoint : connectionManager.getValues()) {
-      Callable<EndpointStateMachine.EndPointStates> endpointTask
-          = getEndPointTask(endpoint);
-      ecs.submit(endpointTask);
-    }
-  }
-  //TODO : Cache some of these tasks instead of creating them
-  //all the time.
-  private Callable<EndpointStateMachine.EndPointStates>
-      getEndPointTask(EndpointStateMachine endpoint) {
-    switch (endpoint.getState()) {
-    case GETVERSION:
-      return new VersionEndpointTask(endpoint, conf);
-    case REGISTER:
-      return  RegisterEndpointTask.newBuilder()
-          .setConfig(conf)
-          .setEndpointStateMachine(endpoint)
-          .setNodeID(getContainerNodeID())
-          .build();
-    case HEARTBEAT:
-      return HeartbeatEndpointTask.newBuilder()
-          .setConfig(conf)
-          .setEndpointStateMachine(endpoint)
-          .setNodeID(getContainerNodeID())
-          .setContext(context)
-          .build();
-    case SHUTDOWN:
-      break;
-    default:
-      throw new IllegalArgumentException("Illegal Argument.");
-    }
-    return null;
-  }
-
-  /**
-   * Computes the next state the container state machine must move to by looking
-   * at all the state of endpoints.
-   * <p>
-   * if any endpoint state has moved to Shutdown, either we have an
-   * unrecoverable error or we have been told to shutdown. Either case the
-   * datanode state machine should move to Shutdown state, otherwise we
-   * remain in the Running state.
-   *
-   * @return next container state.
-   */
-  private DatanodeStateMachine.DatanodeStates
-      computeNextContainerState(
-      List<Future<EndpointStateMachine.EndPointStates>> results) {
-    for (Future<EndpointStateMachine.EndPointStates> state : results) {
-      try {
-        if (state.get() == EndpointStateMachine.EndPointStates.SHUTDOWN) {
-          // if any endpoint tells us to shutdown we move to shutdown state.
-          return DatanodeStateMachine.DatanodeStates.SHUTDOWN;
-        }
-      } catch (InterruptedException | ExecutionException e) {
-        LOG.error("Error in executing end point task.", e);
-      }
-    }
-    return DatanodeStateMachine.DatanodeStates.RUNNING;
-  }
-
-  /**
-   * Wait for execute to finish.
-   *
-   * @param duration - Time
-   * @param timeUnit - Unit of duration.
-   */
-  @Override
-  public DatanodeStateMachine.DatanodeStates
-      await(long duration, TimeUnit timeUnit)
-      throws InterruptedException, ExecutionException, TimeoutException {
-    int count = connectionManager.getValues().size();
-    int returned = 0;
-    long timeLeft = timeUnit.toMillis(duration);
-    long startTime = Time.monotonicNow();
-    List<Future<EndpointStateMachine.EndPointStates>> results = new
-        LinkedList<>();
-
-    while (returned < count && timeLeft > 0) {
-      Future<EndpointStateMachine.EndPointStates> result =
-          ecs.poll(timeLeft, TimeUnit.MILLISECONDS);
-      if (result != null) {
-        results.add(result);
-        returned++;
-      }
-      timeLeft = timeLeft - (Time.monotonicNow() - startTime);
-    }
-    return computeNextContainerState(results);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/package-info.java
deleted file mode 100644
index 6b8d16c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.states.datanode;
-/**
- This package contians files that guide the state transitions from
- Init->Running->Shutdown for the datanode.
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
deleted file mode 100644
index b4cce23..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.states.endpoint;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.container.common.helpers
-    .DeletedContainerBlocksSummary;
-import org.apache.hadoop.ozone.container.common.statemachine
-    .EndpointStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine
-    .EndpointStateMachine.EndPointStates;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
-import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerNodeIDProto;
-import org.apache.hadoop.ozone.protocol.commands.SendContainerCommand;
-import org.apache.hadoop.ozone.protocol.proto
-     .StorageContainerDatanodeProtocolProtos.SCMCommandResponseProto;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.time.ZonedDateTime;
-import java.util.concurrent.Callable;
-
-/**
- * Heartbeat class for SCMs.
- */
-public class HeartbeatEndpointTask
-    implements Callable<EndpointStateMachine.EndPointStates> {
-  static final Logger LOG =
-      LoggerFactory.getLogger(HeartbeatEndpointTask.class);
-  private final EndpointStateMachine rpcEndpoint;
-  private final Configuration conf;
-  private ContainerNodeIDProto containerNodeIDProto;
-  private StateContext context;
-
-  /**
-   * Constructs a SCM heart beat.
-   *
-   * @param conf Config.
-   */
-  public HeartbeatEndpointTask(EndpointStateMachine rpcEndpoint,
-      Configuration conf, StateContext context) {
-    this.rpcEndpoint = rpcEndpoint;
-    this.conf = conf;
-    this.context = context;
-  }
-
-  /**
-   * Get the container Node ID proto.
-   *
-   * @return ContainerNodeIDProto
-   */
-  public ContainerNodeIDProto getContainerNodeIDProto() {
-    return containerNodeIDProto;
-  }
-
-  /**
-   * Set container node ID proto.
-   *
-   * @param containerNodeIDProto - the node id.
-   */
-  public void setContainerNodeIDProto(ContainerNodeIDProto
-      containerNodeIDProto) {
-    this.containerNodeIDProto = containerNodeIDProto;
-  }
-
-  /**
-   * Computes a result, or throws an exception if unable to do so.
-   *
-   * @return computed result
-   * @throws Exception if unable to compute a result
-   */
-  @Override
-  public EndpointStateMachine.EndPointStates call() throws Exception {
-    rpcEndpoint.lock();
-    try {
-      Preconditions.checkState(this.containerNodeIDProto != null);
-      DatanodeID datanodeID = DatanodeID.getFromProtoBuf(this
-          .containerNodeIDProto.getDatanodeID());
-
-      SCMHeartbeatResponseProto reponse = rpcEndpoint.getEndPoint()
-          .sendHeartbeat(datanodeID, this.context.getNodeReport(),
-              this.context.getContainerReportState());
-      processResponse(reponse, datanodeID);
-      rpcEndpoint.setLastSuccessfulHeartbeat(ZonedDateTime.now());
-      rpcEndpoint.zeroMissedCount();
-    } catch (IOException ex) {
-      rpcEndpoint.logIfNeeded(ex);
-    } finally {
-      rpcEndpoint.unlock();
-    }
-    return rpcEndpoint.getState();
-  }
-
-  /**
-   * Returns a builder class for HeartbeatEndpointTask task.
-   * @return   Builder.
-   */
-  public static Builder newBuilder() {
-    return new Builder();
-  }
-
-  /**
-   * Add this command to command processing Queue.
-   *
-   * @param response - SCMHeartbeat response.
-   */
-  private void processResponse(SCMHeartbeatResponseProto response,
-      final DatanodeID datanodeID) {
-    for (SCMCommandResponseProto commandResponseProto : response
-        .getCommandsList()) {
-      // Verify the response is indeed for this datanode.
-      Preconditions.checkState(commandResponseProto.getDatanodeUUID()
-          .equalsIgnoreCase(datanodeID.getDatanodeUuid().toString()),
-          "Unexpected datanode ID in the response.");
-      switch (commandResponseProto.getCmdType()) {
-      case sendContainerReport:
-        this.context.addCommand(SendContainerCommand.getFromProtobuf(
-            commandResponseProto.getSendReport()));
-        break;
-      case reregisterCommand:
-        if (rpcEndpoint.getState() == EndPointStates.HEARTBEAT) {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Received SCM notification to register."
-                + " Interrupt HEARTBEAT and transit to REGISTER state.");
-          }
-          rpcEndpoint.setState(EndPointStates.REGISTER);
-        } else {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Illegal state {} found, expecting {}.",
-                rpcEndpoint.getState().name(), EndPointStates.HEARTBEAT);
-          }
-        }
-        break;
-      case deleteBlocksCommand:
-        DeleteBlocksCommand db = DeleteBlocksCommand
-            .getFromProtobuf(commandResponseProto.getDeleteBlocksProto());
-        if (!db.blocksTobeDeleted().isEmpty()) {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug(DeletedContainerBlocksSummary
-                .getFrom(db.blocksTobeDeleted())
-                .toString());
-          }
-          this.context.addCommand(db);
-        }
-        break;
-      case closeContainerCommand:
-        CloseContainerCommand closeContainer =
-            CloseContainerCommand.getFromProtobuf(
-                commandResponseProto.getCloseContainerProto());
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Received SCM container close request for container {}",
-              closeContainer.getContainerName());
-        }
-        this.context.addCommand(closeContainer);
-        break;
-      default:
-        throw new IllegalArgumentException("Unknown response : "
-            + commandResponseProto.getCmdType().name());
-      }
-    }
-  }
-
-  /**
-   * Builder class for HeartbeatEndpointTask.
-   */
-  public static class Builder {
-    private EndpointStateMachine endPointStateMachine;
-    private Configuration conf;
-    private ContainerNodeIDProto containerNodeIDProto;
-    private StateContext context;
-
-    /**
-     * Constructs the builder class.
-     */
-    public Builder() {
-    }
-
-    /**
-     * Sets the endpoint state machine.
-     *
-     * @param rpcEndPoint - Endpoint state machine.
-     * @return Builder
-     */
-    public Builder setEndpointStateMachine(EndpointStateMachine rpcEndPoint) {
-      this.endPointStateMachine = rpcEndPoint;
-      return this;
-    }
-
-    /**
-     * Sets the Config.
-     *
-     * @param config - config
-     * @return Builder
-     */
-    public Builder setConfig(Configuration config) {
-      this.conf = config;
-      return this;
-    }
-
-    /**
-     * Sets the NodeID.
-     *
-     * @param nodeID - NodeID proto
-     * @return Builder
-     */
-    public Builder setNodeID(ContainerNodeIDProto nodeID) {
-      this.containerNodeIDProto = nodeID;
-      return this;
-    }
-
-    /**
-     * Sets the context.
-     * @param stateContext - State context.
-     * @return this.
-     */
-    public Builder setContext(StateContext stateContext) {
-      this.context = stateContext;
-      return this;
-    }
-
-    public HeartbeatEndpointTask build() {
-      if (endPointStateMachine == null) {
-        LOG.error("No endpoint specified.");
-        throw new IllegalArgumentException("A valid endpoint state machine is" +
-            " needed to construct HeartbeatEndpointTask task");
-      }
-
-      if (conf == null) {
-        LOG.error("No config specified.");
-        throw new IllegalArgumentException("A valid configration is needed to" +
-            " construct HeartbeatEndpointTask task");
-      }
-
-      if (containerNodeIDProto == null) {
-        LOG.error("No nodeID specified.");
-        throw new IllegalArgumentException("A vaild Node ID is needed to " +
-            "construct HeartbeatEndpointTask task");
-      }
-
-      HeartbeatEndpointTask task = new HeartbeatEndpointTask(this
-          .endPointStateMachine, this.conf, this.context);
-      task.setContainerNodeIDProto(containerNodeIDProto);
-      return task;
-    }
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[02/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java
deleted file mode 100644
index 30f87e24..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java
+++ /dev/null
@@ -1,456 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-import com.google.common.primitives.Longs;
-import static java.util.concurrent.TimeUnit.SECONDS;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics;
-import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher;
-import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.MiniOzoneClassicCluster;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.*;
-import java.util.concurrent.TimeoutException;
-
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
-    DFS_CBLOCK_DISK_CACHE_PATH_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
-    DFS_CBLOCK_TRACE_IO;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
-    DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
-    DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL;
-
-/**
- * Tests for Local Cache Buffer Manager.
- */
-public class TestBufferManager {
-  private final static long GB = 1024 * 1024 * 1024;
-  private final static int KB = 1024;
-  private static MiniOzoneCluster cluster;
-  private static OzoneConfiguration config;
-  private static StorageContainerLocationProtocolClientSideTranslatorPB
-      storageContainerLocationClient;
-  private static XceiverClientManager xceiverClientManager;
-
-  @BeforeClass
-  public static void init() throws IOException {
-    config = new OzoneConfiguration();
-    String path = GenericTestUtils.getTempPath(
-        TestBufferManager.class.getSimpleName());
-    config.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
-    config.setBoolean(DFS_CBLOCK_TRACE_IO, true);
-    config.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
-    cluster = new MiniOzoneClassicCluster.Builder(config)
-        .numDataNodes(1).setHandlerType("distributed").build();
-    storageContainerLocationClient = cluster
-        .createStorageContainerLocationClient();
-    xceiverClientManager = new XceiverClientManager(config);
-  }
-
-  @AfterClass
-  public static void shutdown() throws InterruptedException {
-    if (cluster != null) {
-      cluster.shutdown();
-    }
-    IOUtils.cleanupWithLogger(null, storageContainerLocationClient, cluster);
-  }
-
-  /**
-   * createContainerAndGetPipeline creates a set of containers and returns the
-   * Pipelines that define those containers.
-   *
-   * @param count - Number of containers to create.
-   * @return - List of Pipelines.
-   * @throws IOException
-   */
-  private List<Pipeline> createContainerAndGetPipeline(int count)
-      throws IOException {
-    List<Pipeline> containerPipelines = new LinkedList<>();
-    for (int x = 0; x < count; x++) {
-      String traceID = "trace" + RandomStringUtils.randomNumeric(4);
-      String containerName = "container" + RandomStringUtils.randomNumeric(10);
-      Pipeline pipeline =
-          storageContainerLocationClient.allocateContainer(
-              xceiverClientManager.getType(),
-              xceiverClientManager.getFactor(), containerName, "CBLOCK");
-      XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline);
-      ContainerProtocolCalls.createContainer(client, traceID);
-      // This step is needed since we set private data on pipelines, when we
-      // read the list from CBlockServer. So we mimic that action here.
-      pipeline.setData(Longs.toByteArray(x));
-      containerPipelines.add(pipeline);
-      xceiverClientManager.releaseClient(client);
-    }
-    return containerPipelines;
-  }
-
-  /**
-   * This test writes some block to the cache and then shuts down the cache.
-   * The cache is then restarted to check that the
-   * correct number of blocks are read from Dirty Log
-   *
-   * @throws IOException
-   */
-  @Test
-  public void testEmptyBlockBufferHandling() throws IOException,
-      InterruptedException, TimeoutException {
-    // Create a new config so that this tests write metafile to new location
-    OzoneConfiguration flushTestConfig = new OzoneConfiguration();
-    String path = GenericTestUtils
-        .getTempPath(TestBufferManager.class.getSimpleName()
-            + RandomStringUtils.randomNumeric(4));
-    flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
-    flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
-    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
-
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
-    String userName = "user" + RandomStringUtils.randomNumeric(4);
-    String data = RandomStringUtils.random(4 * KB);
-    List<Pipeline> pipelines = createContainerAndGetPipeline(10);
-    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
-        xceiverClientManager, metrics);
-    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
-        .setConfiguration(flushTestConfig)
-        .setVolumeName(volumeName)
-        .setUserName(userName)
-        .setPipelines(pipelines)
-        .setClientManager(xceiverClientManager)
-        .setBlockSize(4 * KB)
-        .setVolumeSize(50 * GB)
-        .setFlusher(flusher)
-        .setCBlockTargetMetrics(metrics)
-        .build();
-    cache.start();
-    // Write data to the cache
-    cache.put(1, data.getBytes(StandardCharsets.UTF_8));
-    Assert.assertEquals(0, metrics.getNumDirectBlockWrites());
-    Assert.assertEquals(1, metrics.getNumWriteOps());
-    cache.put(2, data.getBytes(StandardCharsets.UTF_8));
-    Assert.assertEquals(0, metrics.getNumDirectBlockWrites());
-    Assert.assertEquals(2, metrics.getNumWriteOps());
-
-    // Store the previous block buffer position
-    Assert.assertEquals(2, metrics.getNumBlockBufferUpdates());
-    // Simulate a shutdown by closing the cache
-    cache.close();
-    Thread.sleep(1000);
-    Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
-    Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
-    Assert.assertEquals(2 * (Long.SIZE/ Byte.SIZE),
-                                metrics.getNumBytesDirtyLogWritten());
-    Assert.assertEquals(0, metrics.getNumFailedBlockBufferFlushes());
-    Assert.assertEquals(0, metrics.getNumInterruptedBufferWaits());
-
-    // Restart cache and check that right number of entries are read
-    CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher newFlusher =
-        new ContainerCacheFlusher(flushTestConfig,
-            xceiverClientManager, newMetrics);
-    CBlockLocalCache newCache = CBlockLocalCache.newBuilder()
-        .setConfiguration(flushTestConfig)
-        .setVolumeName(volumeName)
-        .setUserName(userName)
-        .setPipelines(pipelines)
-        .setClientManager(xceiverClientManager)
-        .setBlockSize(4 * KB)
-        .setVolumeSize(50 * GB)
-        .setFlusher(newFlusher)
-        .setCBlockTargetMetrics(newMetrics)
-        .build();
-    newCache.start();
-    Thread fllushListenerThread = new Thread(newFlusher);
-    fllushListenerThread.setDaemon(true);
-    fllushListenerThread.start();
-
-    Thread.sleep(5000);
-    Assert.assertEquals(metrics.getNumBlockBufferUpdates(),
-                                      newMetrics.getNumDirtyLogBlockRead());
-    Assert.assertEquals(newMetrics.getNumDirtyLogBlockRead()
-            * (Long.SIZE/ Byte.SIZE), newMetrics.getNumBytesDirtyLogReads());
-    // Now shutdown again, nothing should be flushed
-    newFlusher.shutdown();
-    Assert.assertEquals(0, newMetrics.getNumBlockBufferUpdates());
-    Assert.assertEquals(0, newMetrics.getNumBytesDirtyLogWritten());
-  }
-
-  @Test
-  public void testPeriodicFlush() throws IOException,
-      InterruptedException, TimeoutException{
-    // Create a new config so that this tests write metafile to new location
-    OzoneConfiguration flushTestConfig = new OzoneConfiguration();
-    String path = GenericTestUtils
-        .getTempPath(TestBufferManager.class.getSimpleName()
-            + RandomStringUtils.randomNumeric(4));
-    flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
-    flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
-    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
-    flushTestConfig
-        .setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, 5, SECONDS);
-
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
-    String userName = "user" + RandomStringUtils.randomNumeric(4);
-    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
-        xceiverClientManager, metrics);
-    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
-        .setConfiguration(flushTestConfig)
-        .setVolumeName(volumeName)
-        .setUserName(userName)
-        .setPipelines(createContainerAndGetPipeline(10))
-        .setClientManager(xceiverClientManager)
-        .setBlockSize(4 * KB)
-        .setVolumeSize(50 * GB)
-        .setFlusher(flusher)
-        .setCBlockTargetMetrics(metrics)
-        .build();
-    cache.start();
-    Thread.sleep(8000);
-    // Ticks will be at 5s, 10s and so on, so this count should be 1
-    Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
-    // Nothing pushed to cache, so nothing should be written
-    Assert.assertEquals(0, metrics.getNumBytesDirtyLogWritten());
-    Assert.assertEquals(0, metrics.getNumBlockBufferFlushCompleted());
-    cache.close();
-    // After close, another trigger should happen but still no data written
-    Assert.assertEquals(2, metrics.getNumBlockBufferFlushTriggered());
-    Assert.assertEquals(0, metrics.getNumBytesDirtyLogWritten());
-    Assert.assertEquals(0, metrics.getNumBlockBufferFlushCompleted());
-    Assert.assertEquals(0, metrics.getNumFailedBlockBufferFlushes());
-  }
-
-  @Test
-  public void testSingleBufferFlush() throws IOException,
-      InterruptedException, TimeoutException {
-    // Create a new config so that this tests write metafile to new location
-    OzoneConfiguration flushTestConfig = new OzoneConfiguration();
-    String path = GenericTestUtils
-        .getTempPath(TestBufferManager.class.getSimpleName()
-            + RandomStringUtils.randomNumeric(4));
-    flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
-    flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
-    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
-
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
-    String userName = "user" + RandomStringUtils.randomNumeric(4);
-    String data = RandomStringUtils.random(4 * KB);
-    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
-        xceiverClientManager, metrics);
-    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
-        .setConfiguration(flushTestConfig)
-        .setVolumeName(volumeName)
-        .setUserName(userName)
-        .setPipelines(createContainerAndGetPipeline(10))
-        .setClientManager(xceiverClientManager)
-        .setBlockSize(4 * KB)
-        .setVolumeSize(50 * GB)
-        .setFlusher(flusher)
-        .setCBlockTargetMetrics(metrics)
-        .build();
-    cache.start();
-
-    for (int i = 0; i < 511; i++) {
-      cache.put(i, data.getBytes(StandardCharsets.UTF_8));
-    }
-    // After writing 511 block no flush should happen
-    Assert.assertEquals(0, metrics.getNumBlockBufferFlushTriggered());
-    Assert.assertEquals(0, metrics.getNumBlockBufferFlushCompleted());
-
-
-    // After one more block it should
-    cache.put(512, data.getBytes(StandardCharsets.UTF_8));
-    Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
-    Thread.sleep(1000);
-    Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
-    cache.close();
-    Assert.assertEquals(512 * (Long.SIZE / Byte.SIZE),
-                                metrics.getNumBytesDirtyLogWritten());
-  }
-
-  @Test
-  public void testMultipleBuffersFlush() throws IOException,
-      InterruptedException, TimeoutException {
-    // Create a new config so that this tests write metafile to new location
-    OzoneConfiguration flushTestConfig = new OzoneConfiguration();
-    String path = GenericTestUtils
-        .getTempPath(TestBufferManager.class.getSimpleName()
-            + RandomStringUtils.randomNumeric(4));
-    flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
-    flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
-    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
-    flushTestConfig
-        .setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, 120, SECONDS);
-
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
-    String userName = "user" + RandomStringUtils.randomNumeric(4);
-    String data = RandomStringUtils.random(4 * KB);
-    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
-        xceiverClientManager, metrics);
-    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
-        .setConfiguration(flushTestConfig)
-        .setVolumeName(volumeName)
-        .setUserName(userName)
-        .setPipelines(createContainerAndGetPipeline(10))
-        .setClientManager(xceiverClientManager)
-        .setBlockSize(4 * KB)
-        .setVolumeSize(50 * GB)
-        .setFlusher(flusher)
-        .setCBlockTargetMetrics(metrics)
-        .build();
-    cache.start();
-
-    for (int i = 0; i < 4; i++) {
-      for (int j = 0; j < 512; j++) {
-        cache.put(i * 512 + j, data.getBytes(StandardCharsets.UTF_8));
-      }
-      // Flush should be triggered after every 512 block write
-      Assert.assertEquals(i + 1, metrics.getNumBlockBufferFlushTriggered());
-    }
-    Assert.assertEquals(0, metrics.getNumIllegalDirtyLogFiles());
-    Assert.assertEquals(0, metrics.getNumFailedDirtyLogFileDeletes());
-    cache.close();
-    Assert.assertEquals(4 * 512 * (Long.SIZE / Byte.SIZE),
-        metrics.getNumBytesDirtyLogWritten());
-    Assert.assertEquals(5, metrics.getNumBlockBufferFlushTriggered());
-    Assert.assertEquals(4, metrics.getNumBlockBufferFlushCompleted());
-  }
-
-  @Test
-  public void testSingleBlockFlush() throws IOException,
-      InterruptedException, TimeoutException{
-    // Create a new config so that this tests write metafile to new location
-    OzoneConfiguration flushTestConfig = new OzoneConfiguration();
-    String path = GenericTestUtils
-        .getTempPath(TestBufferManager.class.getSimpleName()
-            + RandomStringUtils.randomNumeric(4));
-    flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
-    flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
-    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
-    flushTestConfig
-        .setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL,
-            5, SECONDS);
-
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
-    String userName = "user" + RandomStringUtils.randomNumeric(4);
-    String data = RandomStringUtils.random(4 * KB);
-    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
-        xceiverClientManager, metrics);
-    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
-        .setConfiguration(flushTestConfig)
-        .setVolumeName(volumeName)
-        .setUserName(userName)
-        .setPipelines(createContainerAndGetPipeline(10))
-        .setClientManager(xceiverClientManager)
-        .setBlockSize(4 * KB)
-        .setVolumeSize(50 * GB)
-        .setFlusher(flusher)
-        .setCBlockTargetMetrics(metrics)
-        .build();
-    cache.start();
-    cache.put(0, data.getBytes(StandardCharsets.UTF_8));
-    Thread.sleep(8000);
-    // Ticks will be at 5s, 10s and so on, so this count should be 1
-    Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
-    // 1 block written to cache, which should be flushed
-    Assert.assertEquals(8, metrics.getNumBytesDirtyLogWritten());
-    Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
-    cache.close();
-    // After close, another trigger should happen but no data should be written
-    Assert.assertEquals(2, metrics.getNumBlockBufferFlushTriggered());
-    Assert.assertEquals(8, metrics.getNumBytesDirtyLogWritten());
-    Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
-    Assert.assertEquals(0, metrics.getNumFailedBlockBufferFlushes());
-  }
-
-  @Test
-  public void testRepeatedBlockWrites() throws IOException,
-      InterruptedException, TimeoutException{
-    // Create a new config so that this tests write metafile to new location
-    OzoneConfiguration flushTestConfig = new OzoneConfiguration();
-    String path = GenericTestUtils
-        .getTempPath(TestBufferManager.class.getSimpleName()
-            + RandomStringUtils.randomNumeric(4));
-    flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
-    flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
-    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
-
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
-    String userName = "user" + RandomStringUtils.randomNumeric(4);
-    String data = RandomStringUtils.random(4 * KB);
-    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
-        xceiverClientManager, metrics);
-    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
-        .setConfiguration(flushTestConfig)
-        .setVolumeName(volumeName)
-        .setUserName(userName)
-        .setPipelines(createContainerAndGetPipeline(10))
-        .setClientManager(xceiverClientManager)
-        .setBlockSize(4 * KB)
-        .setVolumeSize(50 * GB)
-        .setFlusher(flusher)
-        .setCBlockTargetMetrics(metrics)
-        .build();
-    Thread fllushListenerThread = new Thread(flusher);
-    fllushListenerThread.setDaemon(true);
-    fllushListenerThread.start();
-    cache.start();
-    for (int i = 0; i < 512; i++) {
-      cache.put(i, data.getBytes(StandardCharsets.UTF_8));
-    }
-    Assert.assertEquals(512, metrics.getNumWriteOps());
-    Assert.assertEquals(512, metrics.getNumBlockBufferUpdates());
-    Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
-    Thread.sleep(5000);
-    Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
-
-
-    for (int i = 0; i < 512; i++) {
-      cache.put(i, data.getBytes(StandardCharsets.UTF_8));
-    }
-    Assert.assertEquals(1024, metrics.getNumWriteOps());
-    Assert.assertEquals(1024, metrics.getNumBlockBufferUpdates());
-    Assert.assertEquals(2, metrics.getNumBlockBufferFlushTriggered());
-
-    Thread.sleep(5000);
-    Assert.assertEquals(0, metrics.getNumWriteIOExceptionRetryBlocks());
-    Assert.assertEquals(0, metrics.getNumWriteGenericExceptionRetryBlocks());
-    Assert.assertEquals(2, metrics.getNumBlockBufferFlushCompleted());
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockCLI.java
deleted file mode 100644
index f8b05ed..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockCLI.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-import org.apache.hadoop.cblock.cli.CBlockCli;
-import org.apache.hadoop.cblock.meta.VolumeDescriptor;
-import org.apache.hadoop.cblock.util.MockStorageClient;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.List;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * A testing class for cblock command line tool.
- */
-public class TestCBlockCLI {
-  private static final long GB = 1 * 1024 * 1024 * 1024L;
-  private static final int KB = 1024;
-  private static CBlockCli cmd;
-  private static OzoneConfiguration conf;
-  private static CBlockManager cBlockManager;
-  private static ByteArrayOutputStream outContent;
-  private static PrintStream testPrintOut;
-
-  @BeforeClass
-  public static void setup() throws IOException {
-    outContent = new ByteArrayOutputStream();
-    ScmClient storageClient = new MockStorageClient();
-    conf = new OzoneConfiguration();
-    String path = GenericTestUtils
-        .getTempPath(TestCBlockCLI.class.getSimpleName());
-    File filePath = new File(path);
-    if (!filePath.exists() && !filePath.mkdirs()) {
-      throw new IOException("Unable to create test DB dir");
-    }
-    conf.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0");
-    conf.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0");
-    conf.set(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY, path.concat(
-        "/testCblockCli.dat"));
-    cBlockManager = new CBlockManager(conf, storageClient);
-    cBlockManager.start();
-    testPrintOut = new PrintStream(outContent);
-    cmd = new CBlockCli(conf, testPrintOut);
-  }
-
-  @AfterClass
-  public static void clean() {
-    if (cBlockManager != null) {
-      cBlockManager.stop();
-      cBlockManager.join();
-      cBlockManager.clean();
-    }
-  }
-
-  @After
-  public void reset() {
-    outContent.reset();
-  }
-
-  /**
-   * Test the help command.
-   * @throws Exception
-   */
-  @Test
-  public void testCliHelp() throws Exception {
-    PrintStream initialStdOut = System.out;
-    System.setOut(testPrintOut);
-    String[] args = {"-h"};
-    cmd.run(args);
-    String helpPrints =
-        "usage: cblock\n" +
-        " -c,--createVolume <user> <volume> <volumeSize in [GB/TB]> " +
-            "<blockSize>   create a fresh new volume\n" +
-        " -d,--deleteVolume <user> <volume>                         " +
-            "              delete a volume\n" +
-        " -h,--help                                                 " +
-            "              help\n" +
-        " -i,--infoVolume <user> <volume>                           " +
-            "              info a volume\n" +
-        " -l,--listVolume <user>                                    " +
-            "              list all volumes\n" +
-        " -s,--serverAddr <serverAddress>:<serverPort>              " +
-            "              specify server address:port\n";
-    assertEquals(helpPrints, outContent.toString());
-    outContent.reset();
-    System.setOut(initialStdOut);
-  }
-
-  /**
-   * Test volume listing command.
-   * @throws Exception
-   */
-  @Test
-  public void testCliList() throws Exception {
-    String userName0 = "userTestCliList0";
-    String userName1 = "userTestCliList1";
-    String userTestNotExist = "userTestNotExist";
-    String volumeName0 = "volumeTest0";
-    String volumeName1 = "volumeTest1";
-    String volumeSize0 = "30GB";
-    String volumeSize1 = "40GB";
-    String blockSize = Integer.toString(4);
-    String[] argsCreate0 =
-        {"-c", userName0, volumeName0, volumeSize0, blockSize};
-    cmd.run(argsCreate0);
-    String[] argsCreate1 =
-        {"-c", userName0, volumeName1, volumeSize1, blockSize};
-    cmd.run(argsCreate1);
-    String[] argsCreate2 =
-        {"-c", userName1, volumeName0, volumeSize0, blockSize};
-    cmd.run(argsCreate2);
-    String[] argsList0 = {"-l"};
-    cmd.run(argsList0);
-    String[] outExpected1 = {
-        "userTestCliList1:volumeTest0\t32212254720\t4096\n",
-        "userTestCliList0:volumeTest0\t32212254720\t4096\n",
-        "userTestCliList0:volumeTest1\t42949672960\t4096\n"};
-    int length = 0;
-    for (String str : outExpected1) {
-      assertTrue(outContent.toString().contains(str));
-      length += str.length();
-    }
-    assertEquals(length, outContent.toString().length());
-    outContent.reset();
-
-    String[] argsList1 = {"-l", userName1};
-    cmd.run(argsList1);
-    String outExpected2 = "userTestCliList1:volumeTest0\t32212254720\t4096\n";
-    assertEquals(outExpected2, outContent.toString());
-    outContent.reset();
-
-    String[] argsList2 = {"-l", userTestNotExist};
-    cmd.run(argsList2);
-    String outExpected3 = "\n";
-    assertEquals(outExpected3, outContent.toString());
-  }
-
-  /**
-   * Test create volume command.
-   * @throws Exception
-   */
-  @Test
-  public void testCliCreate() throws Exception {
-    String userName = "userTestCliCreate";
-    String volumeName = "volumeTest";
-    String volumeSize = "30GB";
-    String blockSize = "4";
-    String[] argsCreate = {"-c", userName, volumeName, volumeSize, blockSize};
-    cmd.run(argsCreate);
-    List<VolumeDescriptor> allVolumes = cBlockManager.getAllVolumes(userName);
-    assertEquals(1, allVolumes.size());
-    VolumeDescriptor volume = allVolumes.get(0);
-    assertEquals(userName, volume.getUserName());
-    assertEquals(volumeName, volume.getVolumeName());
-    long volumeSizeB = volume.getVolumeSize();
-    assertEquals(30, (int)(volumeSizeB/ GB));
-    assertEquals(4, volume.getBlockSize()/ KB);
-  }
-
-  /**
-   * Test delete volume command.
-   * @throws Exception
-   */
-  @Test
-  public void testCliDelete() throws Exception {
-    String userName = "userTestCliDelete";
-    String volumeName = "volumeTest";
-    String volumeSize = "30GB";
-    String blockSize = "4";
-    String[] argsCreate = {"-c", userName, volumeName, volumeSize, blockSize};
-    cmd.run(argsCreate);
-    List<VolumeDescriptor> allVolumes = cBlockManager.getAllVolumes(userName);
-    assertEquals(1, allVolumes.size());
-    VolumeDescriptor volume = allVolumes.get(0);
-    assertEquals(userName, volume.getUserName());
-    assertEquals(volumeName, volume.getVolumeName());
-    long volumeSizeB = volume.getVolumeSize();
-    assertEquals(30, (int)(volumeSizeB/ GB));
-    assertEquals(4, volume.getBlockSize()/ KB);
-
-    String[] argsDelete = {"-d", userName, volumeName};
-    cmd.run(argsDelete);
-    allVolumes = cBlockManager.getAllVolumes(userName);
-    assertEquals(0, allVolumes.size());
-  }
-
-  /**
-   * Test info volume command.
-   * @throws Exception
-   */
-  @Test
-  public void testCliInfoVolume() throws Exception {
-    String userName0 = "userTestCliInfo";
-    String volumeName0 = "volumeTest0";
-    String volumeSize = "8000GB";
-    String blockSize = "4";
-    String[] argsCreate0 = {
-        "-c", userName0, volumeName0, volumeSize, blockSize};
-    cmd.run(argsCreate0);
-    String[] argsInfo = {"-i", userName0, volumeName0};
-    cmd.run(argsInfo);
-    // TODO : the usage field is not implemented yet, always 0 now.
-    String outExpected = " userName:userTestCliInfo " +
-        "volumeName:volumeTest0 " +
-        "volumeSize:8589934592000 " +
-        "blockSize:4096 (sizeInBlocks:2097152000) usageInBlocks:0\n";
-    assertEquals(outExpected, outContent.toString());
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java
deleted file mode 100644
index cdd65c5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java
+++ /dev/null
@@ -1,377 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-import com.google.common.primitives.Longs;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics;
-import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher;
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.MiniOzoneClassicCluster;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.PipelineChannel;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocolPB
-    .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_DISK_CACHE_PATH_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_TRACE_IO;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE;
-
-/**
- * Tests for Cblock read write functionality.
- */
-public class TestCBlockReadWrite {
-  private final static long GB = 1024 * 1024 * 1024;
-  private final static int KB = 1024;
-  private static MiniOzoneCluster cluster;
-  private static OzoneConfiguration config;
-  private static StorageContainerLocationProtocolClientSideTranslatorPB
-      storageContainerLocationClient;
-  private static XceiverClientManager xceiverClientManager;
-
-  @BeforeClass
-  public static void init() throws IOException {
-    config = new OzoneConfiguration();
-    String path = GenericTestUtils
-        .getTempPath(TestCBlockReadWrite.class.getSimpleName());
-    config.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
-    config.setBoolean(DFS_CBLOCK_TRACE_IO, true);
-    config.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
-    cluster = new MiniOzoneClassicCluster.Builder(config)
-        .numDataNodes(1)
-        .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
-    storageContainerLocationClient = cluster
-        .createStorageContainerLocationClient();
-    xceiverClientManager = new XceiverClientManager(config);
-  }
-
-  @AfterClass
-  public static void shutdown() throws InterruptedException {
-    if (cluster != null) {
-      cluster.shutdown();
-    }
-    IOUtils.cleanupWithLogger(null, storageContainerLocationClient, cluster);
-  }
-
-  /**
-   * getContainerPipelines creates a set of containers and returns the
-   * Pipelines that define those containers.
-   *
-   * @param count - Number of containers to create.
-   * @return - List of Pipelines.
-   * @throws IOException throws Exception
-   */
-  private List<Pipeline> getContainerPipeline(int count) throws IOException {
-    List<Pipeline> containerPipelines = new LinkedList<>();
-    for (int x = 0; x < count; x++) {
-      String traceID = "trace" + RandomStringUtils.randomNumeric(4);
-      String containerName = "container" + RandomStringUtils.randomNumeric(10);
-      Pipeline pipeline =
-          storageContainerLocationClient.allocateContainer(
-              xceiverClientManager.getType(),
-              xceiverClientManager.getFactor(), containerName, "CBLOCK");
-      XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline);
-      ContainerProtocolCalls.createContainer(client, traceID);
-      // This step is needed since we set private data on pipelines, when we
-      // read the list from CBlockServer. So we mimic that action here.
-      pipeline.setData(Longs.toByteArray(x));
-      containerPipelines.add(pipeline);
-    }
-    return containerPipelines;
-  }
-
-  /**
-   * This test creates a cache and performs a simple write / read.
-   * The operations are done by bypassing the cache.
-   *
-   * @throws IOException
-   */
-  @Test
-  public void testDirectIO() throws IOException,
-      InterruptedException, TimeoutException {
-    OzoneConfiguration cConfig = new OzoneConfiguration();
-    cConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, false);
-    cConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
-    final long blockID = 0;
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
-    String userName = "user" + RandomStringUtils.randomNumeric(4);
-    String data = RandomStringUtils.random(4 * KB);
-    String dataHash = DigestUtils.sha256Hex(data);
-    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher flusher = new ContainerCacheFlusher(cConfig,
-        xceiverClientManager, metrics);
-    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
-        .setConfiguration(cConfig)
-        .setVolumeName(volumeName)
-        .setUserName(userName)
-        .setPipelines(getContainerPipeline(10))
-        .setClientManager(xceiverClientManager)
-        .setBlockSize(4 * KB)
-        .setVolumeSize(50 * GB)
-        .setFlusher(flusher)
-        .setCBlockTargetMetrics(metrics)
-        .build();
-    cache.start();
-    Assert.assertFalse(cache.isShortCircuitIOEnabled());
-    cache.put(blockID, data.getBytes(StandardCharsets.UTF_8));
-    Assert.assertEquals(1, metrics.getNumDirectBlockWrites());
-    Assert.assertEquals(1, metrics.getNumWriteOps());
-    // Please note that this read is directly from remote container
-    LogicalBlock block = cache.get(blockID);
-    Assert.assertEquals(1, metrics.getNumReadOps());
-    Assert.assertEquals(0, metrics.getNumReadCacheHits());
-    Assert.assertEquals(1, metrics.getNumReadCacheMiss());
-    Assert.assertEquals(0, metrics.getNumReadLostBlocks());
-    Assert.assertEquals(0, metrics.getNumFailedDirectBlockWrites());
-
-    cache.put(blockID + 1, data.getBytes(StandardCharsets.UTF_8));
-    Assert.assertEquals(2, metrics.getNumDirectBlockWrites());
-    Assert.assertEquals(2, metrics.getNumWriteOps());
-    Assert.assertEquals(0, metrics.getNumFailedDirectBlockWrites());
-    // Please note that this read is directly from remote container
-    block = cache.get(blockID + 1);
-    Assert.assertEquals(2, metrics.getNumReadOps());
-    Assert.assertEquals(0, metrics.getNumReadCacheHits());
-    Assert.assertEquals(2, metrics.getNumReadCacheMiss());
-    Assert.assertEquals(0, metrics.getNumReadLostBlocks());
-    String readHash = DigestUtils.sha256Hex(block.getData().array());
-    Assert.assertEquals("File content does not match.", dataHash, readHash);
-    GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
-    cache.close();
-  }
-
-  /**
-   * This test writes some block to the cache and then shuts down the cache
-   * The cache is then restarted with "short.circuit.io" disable to check
-   * that the blocks are read correctly from the container.
-   *
-   * @throws IOException
-   */
-  @Test
-  public void testContainerWrites() throws IOException,
-      InterruptedException, TimeoutException {
-    // Create a new config so that this tests write metafile to new location
-    OzoneConfiguration flushTestConfig = new OzoneConfiguration();
-    String path = GenericTestUtils
-        .getTempPath(TestCBlockReadWrite.class.getSimpleName());
-    flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
-    flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
-    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
-    flushTestConfig.setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, 3,
-        TimeUnit.SECONDS);
-    XceiverClientManager xcm = new XceiverClientManager(flushTestConfig);
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
-    String userName = "user" + RandomStringUtils.randomNumeric(4);
-
-    int numUniqueBlocks = 4;
-    String[] data = new String[numUniqueBlocks];
-    String[] dataHash = new String[numUniqueBlocks];
-    for (int i = 0; i < numUniqueBlocks; i++) {
-      data[i] = RandomStringUtils.random(4 * KB);
-      dataHash[i] = DigestUtils.sha256Hex(data[i]);
-    }
-
-    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
-        xcm, metrics);
-    List<Pipeline> pipelines = getContainerPipeline(10);
-    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
-        .setConfiguration(flushTestConfig)
-        .setVolumeName(volumeName)
-        .setUserName(userName)
-        .setPipelines(pipelines)
-        .setClientManager(xcm)
-        .setBlockSize(4 * KB)
-        .setVolumeSize(50 * GB)
-        .setFlusher(flusher)
-        .setCBlockTargetMetrics(metrics)
-        .build();
-    cache.start();
-    Thread flushListenerThread = new Thread(flusher);
-    flushListenerThread.setDaemon(true);
-    flushListenerThread.start();
-    Assert.assertTrue(cache.isShortCircuitIOEnabled());
-    // Write data to the cache
-    for (int i = 0; i < 512; i++) {
-      cache.put(i, data[i % numUniqueBlocks].getBytes(StandardCharsets.UTF_8));
-    }
-    // Close the cache and flush the data to the containers
-    cache.close();
-    Assert.assertEquals(0, metrics.getNumDirectBlockWrites());
-    Assert.assertEquals(512, metrics.getNumWriteOps());
-    Thread.sleep(3000);
-    flusher.shutdown();
-    Assert.assertTrue(metrics.getNumBlockBufferFlushTriggered() > 1);
-    Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
-    Assert.assertEquals(0, metrics.getNumWriteIOExceptionRetryBlocks());
-    Assert.assertEquals(0, metrics.getNumWriteGenericExceptionRetryBlocks());
-    Assert.assertEquals(0, metrics.getNumFailedReleaseLevelDB());
-    // Now disable DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO and restart cache
-    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, false);
-    CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher newFlusher =
-        new ContainerCacheFlusher(flushTestConfig, xcm, newMetrics);
-    CBlockLocalCache newCache = CBlockLocalCache.newBuilder()
-        .setConfiguration(flushTestConfig)
-        .setVolumeName(volumeName)
-        .setUserName(userName)
-        .setPipelines(pipelines)
-        .setClientManager(xcm)
-        .setBlockSize(4 * KB)
-        .setVolumeSize(50 * GB)
-        .setFlusher(newFlusher)
-        .setCBlockTargetMetrics(newMetrics)
-        .build();
-    newCache.start();
-    Assert.assertFalse(newCache.isShortCircuitIOEnabled());
-    // this read will be from the container, also match the hash
-    for (int i = 0; i < 512; i++) {
-      LogicalBlock block = newCache.get(i);
-      String readHash = DigestUtils.sha256Hex(block.getData().array());
-      Assert.assertEquals("File content does not match, for index:"
-          + i, dataHash[i % numUniqueBlocks], readHash);
-    }
-    Assert.assertEquals(0, newMetrics.getNumReadLostBlocks());
-    Assert.assertEquals(0, newMetrics.getNumFailedReadBlocks());
-    newCache.close();
-    newFlusher.shutdown();
-  }
-
-  @Test
-  public void testRetryLog() throws IOException,
-      InterruptedException, TimeoutException {
-    // Create a new config so that this tests write metafile to new location
-    OzoneConfiguration flushTestConfig = new OzoneConfiguration();
-    String path = GenericTestUtils
-        .getTempPath(TestCBlockReadWrite.class.getSimpleName());
-    flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
-    flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
-    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
-    flushTestConfig.setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL,
-        3,
-        TimeUnit.SECONDS);
-
-    int numblocks = 10;
-    flushTestConfig.setInt(DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE, numblocks);
-
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
-    String userName = "user" + RandomStringUtils.randomNumeric(4);
-    String data = RandomStringUtils.random(4 * KB);
-
-    List<Pipeline> fakeContainerPipelines = new LinkedList<>();
-    PipelineChannel pipelineChannel = new PipelineChannel("fake",
-        LifeCycleState.OPEN, ReplicationType.STAND_ALONE, ReplicationFactor.ONE,
-        "fake");
-    Pipeline fakePipeline = new Pipeline("fake", pipelineChannel);
-    fakePipeline.setData(Longs.toByteArray(1));
-    fakeContainerPipelines.add(fakePipeline);
-
-    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
-        xceiverClientManager, metrics);
-    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
-        .setConfiguration(flushTestConfig)
-        .setVolumeName(volumeName)
-        .setUserName(userName)
-        .setPipelines(fakeContainerPipelines)
-        .setClientManager(xceiverClientManager)
-        .setBlockSize(4 * KB)
-        .setVolumeSize(50 * GB)
-        .setFlusher(flusher)
-        .setCBlockTargetMetrics(metrics)
-        .build();
-    cache.start();
-    Thread flushListenerThread = new Thread(flusher);
-    flushListenerThread.setDaemon(true);
-    flushListenerThread.start();
-
-    for (int i = 0; i < numblocks; i++) {
-      cache.put(i, data.getBytes(StandardCharsets.UTF_8));
-    }
-    Assert.assertEquals(numblocks, metrics.getNumWriteOps());
-    Thread.sleep(3000);
-
-    // all the writes to the container will fail because of fake pipelines
-    Assert.assertEquals(numblocks, metrics.getNumDirtyLogBlockRead());
-    Assert.assertTrue(
-        metrics.getNumWriteGenericExceptionRetryBlocks() >= numblocks);
-    Assert.assertEquals(0, metrics.getNumWriteIOExceptionRetryBlocks());
-    Assert.assertEquals(0, metrics.getNumFailedRetryLogFileWrites());
-    Assert.assertEquals(0, metrics.getNumFailedReleaseLevelDB());
-    cache.close();
-    flusher.shutdown();
-
-    // restart cache with correct pipelines, now blocks should be uploaded
-    // correctly
-    CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher newFlusher =
-        new ContainerCacheFlusher(flushTestConfig,
-            xceiverClientManager, newMetrics);
-    CBlockLocalCache newCache = CBlockLocalCache.newBuilder()
-        .setConfiguration(flushTestConfig)
-        .setVolumeName(volumeName)
-        .setUserName(userName)
-        .setPipelines(getContainerPipeline(10))
-        .setClientManager(xceiverClientManager)
-        .setBlockSize(4 * KB)
-        .setVolumeSize(50 * GB)
-        .setFlusher(newFlusher)
-        .setCBlockTargetMetrics(newMetrics)
-        .build();
-    newCache.start();
-    Thread newFlushListenerThread = new Thread(newFlusher);
-    newFlushListenerThread.setDaemon(true);
-    newFlushListenerThread.start();
-    Thread.sleep(3000);
-    Assert.assertTrue(newMetrics.getNumRetryLogBlockRead() >= numblocks);
-    Assert.assertEquals(0, newMetrics.getNumWriteGenericExceptionRetryBlocks());
-    Assert.assertEquals(0, newMetrics.getNumWriteIOExceptionRetryBlocks());
-    Assert.assertEquals(0, newMetrics.getNumFailedReleaseLevelDB());
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java
deleted file mode 100644
index b5695fc..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.cblock.util.MockStorageClient;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-/**
- * This class tests the basics of CBlock server. Mainly about the four
- * operations on volumes: create, delete, info and list.
- */
-public class TestCBlockServer {
-  private static CBlockManager cBlockManager;
-  private static OzoneConfiguration conf;
-
-  @Before
-  public void setup() throws Exception {
-    ScmClient storageClient = new MockStorageClient();
-    conf = new OzoneConfiguration();
-    conf.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0");
-    conf.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0");
-    cBlockManager = new CBlockManager(conf, storageClient);
-    cBlockManager.start();
-  }
-
-  @After
-  public void clean() {
-    cBlockManager.stop();
-    cBlockManager.join();
-    cBlockManager.clean();
-  }
-
-  /**
-   * Test create volume for different users.
-   * @throws Exception
-   */
-  @Test
-  public void testCreateVolume() throws Exception {
-    String userName1 = "user" + RandomStringUtils.randomNumeric(5);
-    String userName2 = "user" + RandomStringUtils.randomNumeric(5);
-    String volumeName1 = "volume" +  RandomStringUtils.randomNumeric(5);
-    String volumeName2 = "volume" +   RandomStringUtils.randomNumeric(5);
-    long volumeSize = 1L*1024*1024;
-    int blockSize = 4096;
-    cBlockManager.createVolume(userName1, volumeName1, volumeSize, blockSize);
-    List<VolumeInfo> volumes = cBlockManager.listVolume(userName1);
-    assertEquals(1, volumes.size());
-    VolumeInfo existingVolume = volumes.get(0);
-    assertEquals(userName1, existingVolume.getUserName());
-    assertEquals(volumeName1, existingVolume.getVolumeName());
-    assertEquals(volumeSize, existingVolume.getVolumeSize());
-    assertEquals(blockSize, existingVolume.getBlockSize());
-
-    cBlockManager.createVolume(userName1, volumeName2, volumeSize, blockSize);
-    cBlockManager.createVolume(userName2, volumeName1, volumeSize, blockSize);
-    volumes = cBlockManager.listVolume(userName1);
-    assertEquals(2, volumes.size());
-    volumes = cBlockManager.listVolume(userName2);
-    assertEquals(1, volumes.size());
-  }
-
-  /**
-   * Test delete volume.
-   * @throws Exception
-   */
-  @Test
-  public void testDeleteVolume() throws Exception {
-    String userName = "user" + RandomStringUtils.randomNumeric(5);
-    String volumeName1 = "volume" +  RandomStringUtils.randomNumeric(5);
-    String volumeName2 = "volume" +  RandomStringUtils.randomNumeric(5);
-    long volumeSize = 1L*1024*1024;
-    int blockSize = 4096;
-    cBlockManager.createVolume(userName, volumeName1, volumeSize, blockSize);
-    cBlockManager.createVolume(userName, volumeName2, volumeSize, blockSize);
-    cBlockManager.deleteVolume(userName, volumeName1, true);
-    List<VolumeInfo> volumes = cBlockManager.listVolume(userName);
-    assertEquals(1, volumes.size());
-
-    VolumeInfo existingVolume = volumes.get(0);
-    assertEquals(userName, existingVolume.getUserName());
-    assertEquals(volumeName2, existingVolume.getVolumeName());
-    assertEquals(volumeSize, existingVolume.getVolumeSize());
-    assertEquals(blockSize, existingVolume.getBlockSize());
-  }
-
-  /**
-   * Test info volume.
-   *
-   * TODO : usage field is not being tested (as it is not implemented yet)
-   * @throws Exception
-   */
-  @Test
-  public void testInfoVolume() throws Exception {
-    String userName = "user" + RandomStringUtils.randomNumeric(5);
-    String volumeName = "volume" +  RandomStringUtils.randomNumeric(5);
-    long volumeSize = 1L*1024*1024;
-    int blockSize = 4096;
-    cBlockManager.createVolume(userName, volumeName, volumeSize, blockSize);
-    VolumeInfo info = cBlockManager.infoVolume(userName, volumeName);
-    assertEquals(userName, info.getUserName());
-    assertEquals(volumeName, info.getVolumeName());
-    assertEquals(volumeSize, info.getVolumeSize());
-    assertEquals(blockSize, info.getBlockSize());
-  }
-
-  /**
-   * Test listing a number of volumes.
-   * @throws Exception
-   */
-  @Test
-  public void testListVolume() throws Exception {
-    String userName = "user" + RandomStringUtils.randomNumeric(5);
-    String volumeName ="volume" +  RandomStringUtils.randomNumeric(5);
-    long volumeSize = 1L*1024*1024;
-    int blockSize = 4096;
-    int volumeNum = 100;
-    for (int i = 0; i<volumeNum; i++) {
-      cBlockManager.createVolume(userName, volumeName + i,
-          volumeSize, blockSize);
-    }
-    List<VolumeInfo> volumes = cBlockManager.listVolume(userName);
-    assertEquals(volumeNum, volumes.size());
-    Set<String> volumeIds = new HashSet<>();
-    for (int i = 0; i<volumeNum; i++) {
-      VolumeInfo volumeInfo = volumes.get(i);
-      assertEquals(userName, volumeInfo.getUserName());
-      assertFalse(volumeIds.contains(volumeName + i));
-      volumeIds.add(volumeName + i);
-      assertEquals(volumeSize, volumeInfo.getVolumeSize());
-      assertEquals(blockSize, volumeInfo.getBlockSize());
-    }
-    for (int i = 0; i<volumeNum; i++) {
-      assertTrue(volumeIds.contains(volumeName + i));
-    }
-  }
-
-  /**
-   * Test listing a number of volumes.
-   * @throws Exception
-   */
-  @Test
-  public void testListVolumes() throws Exception {
-    String volumeName ="volume" +  RandomStringUtils.randomNumeric(5);
-    long volumeSize = 1L*1024*1024;
-    int blockSize = 4096;
-    int volumeNum = 100;
-    int userCount = 10;
-
-    assertTrue("We need at least one volume for each user",
-        userCount < volumeNum);
-
-    for (int i = 0; i<volumeNum; i++) {
-      String userName =
-          "user-" + (i % userCount);
-      cBlockManager.createVolume(userName, volumeName + i,
-          volumeSize, blockSize);
-    }
-    List<VolumeInfo> allVolumes = cBlockManager.listVolumes();
-    //check if we have the volumes from all the users.
-
-    Set<String> volumeIds = new HashSet<>();
-    Set<String> usernames = new HashSet<>();
-    for (int i = 0; i < allVolumes.size(); i++) {
-      VolumeInfo volumeInfo = allVolumes.get(i);
-      assertFalse(volumeIds.contains(volumeName + i));
-      usernames.add(volumeInfo.getUserName());
-      volumeIds.add(volumeName + i);
-      assertEquals(volumeSize, volumeInfo.getVolumeSize());
-      assertEquals(blockSize, volumeInfo.getBlockSize());
-    }
-
-    assertEquals(volumeNum, volumeIds.size());
-    for (int i = 0; i<volumeNum; i++) {
-      assertTrue(volumeIds.contains(volumeName + i));
-    }
-
-    assertEquals(userCount, usernames.size());
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockServerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockServerPersistence.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockServerPersistence.java
deleted file mode 100644
index c6896a1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockServerPersistence.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-import org.apache.hadoop.cblock.meta.VolumeDescriptor;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.cblock.util.MockStorageClient;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY;
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test the CBlock server state is maintained in persistent storage and can be
- * recovered on CBlock server restart.
- */
-public class TestCBlockServerPersistence {
-
-  /**
-   * Test when cblock server fails with volume meta data, the meta data can be
-   * restored correctly.
-   * @throws Exception
-   */
-  @Test
-  public void testWriteToPersistentStore() throws Exception {
-    OzoneConfiguration conf = new OzoneConfiguration();
-    String userName = "testWriteToPersistentStore";
-    String volumeName1 = "testVolume1";
-    String volumeName2 = "testVolume2";
-    long volumeSize1 = 30L*1024*1024*1024;
-    long volumeSize2 = 15L*1024*1024*1024;
-    int blockSize = 4096;
-    CBlockManager cBlockManager = null;
-    CBlockManager cBlockManager1 = null;
-    String path = GenericTestUtils
-        .getTempPath(TestCBlockServerPersistence.class.getSimpleName());
-    File filePath = new File(path);
-    if(!filePath.exists() && !filePath.mkdirs()) {
-      throw new IOException("Unable to create test DB dir");
-    }
-    conf.set(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY, path.concat(
-        "/testCblockPersistence.dat"));
-    try {
-      ScmClient storageClient = new MockStorageClient();
-      conf.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0");
-      conf.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0");
-      cBlockManager = new CBlockManager(conf, storageClient);
-      cBlockManager.start();
-      cBlockManager.createVolume(userName, volumeName1, volumeSize1, blockSize);
-      cBlockManager.createVolume(userName, volumeName2, volumeSize2, blockSize);
-      List<VolumeDescriptor> allVolumes = cBlockManager.getAllVolumes();
-      // close the cblock server. Since meta data is written to disk on volume
-      // creation, closing server here is the same as a cblock server crash.
-      cBlockManager.close();
-      cBlockManager.stop();
-      cBlockManager.join();
-      cBlockManager = null;
-      assertEquals(2, allVolumes.size());
-      VolumeDescriptor volumeDescriptor1 = allVolumes.get(0);
-      VolumeDescriptor volumeDescriptor2 = allVolumes.get(1);
-
-      // create a new cblock server instance. This is just the
-      // same as restarting cblock server.
-      ScmClient storageClient1 = new MockStorageClient();
-      OzoneConfiguration conf1 = new OzoneConfiguration();
-      conf1.set(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY, path.concat(
-          "/testCblockPersistence.dat"));
-      conf1.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0");
-      conf1.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0");
-      cBlockManager1 = new CBlockManager(conf1, storageClient1);
-      cBlockManager1.start();
-      List<VolumeDescriptor> allVolumes1 = cBlockManager1.getAllVolumes();
-      assertEquals(2, allVolumes1.size());
-      VolumeDescriptor newvolumeDescriptor1 = allVolumes1.get(0);
-      VolumeDescriptor newvolumeDescriptor2 = allVolumes1.get(1);
-
-      // It seems levelDB iterator gets keys in the same order as keys
-      // are inserted, in which case the else clause should never happen.
-      // But still kept the second clause if it is possible to get different
-      // key ordering from leveldb. And we do not rely on the ordering of keys
-      // here.
-      if (volumeDescriptor1.getVolumeName().equals(
-          newvolumeDescriptor1.getVolumeName())) {
-        assertEquals(volumeDescriptor1.toString(),
-            newvolumeDescriptor1.toString());
-        assertEquals(volumeDescriptor2.toString(),
-            newvolumeDescriptor2.toString());
-      } else {
-        assertEquals(volumeDescriptor1.toString(),
-            newvolumeDescriptor2.toString());
-        assertEquals(volumeDescriptor2.toString(),
-            newvolumeDescriptor1.toString());
-      }
-    } finally {
-      if (cBlockManager != null) {
-        cBlockManager.clean();
-      }
-      if (cBlockManager1 != null) {
-        cBlockManager1.close();
-        cBlockManager1.stop();
-        cBlockManager1.join();
-        cBlockManager1.clean();
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java
deleted file mode 100644
index 7749432..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java
+++ /dev/null
@@ -1,444 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-import com.google.common.primitives.Longs;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.cblock.jscsiHelper.CBlockIStorageImpl;
-import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics;
-import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher;
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.MiniOzoneClassicCluster;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.util.Time;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import static java.lang.Math.abs;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
-    DFS_CBLOCK_DISK_CACHE_PATH_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
-    DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
-    DFS_CBLOCK_TRACE_IO;
-
-/**
- * Tests for local cache.
- */
-public class TestLocalBlockCache {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(TestLocalBlockCache.class);
-  private final static long GB = 1024 * 1024 * 1024;
-  private final static int KB = 1024;
-  private static MiniOzoneCluster cluster;
-  private static OzoneConfiguration config;
-  private static StorageContainerLocationProtocolClientSideTranslatorPB
-      storageContainerLocationClient;
-  private static XceiverClientManager xceiverClientManager;
-
-  @BeforeClass
-  public static void init() throws IOException {
-    config = new OzoneConfiguration();
-    String path = GenericTestUtils
-        .getTempPath(TestLocalBlockCache.class.getSimpleName());
-    config.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
-    config.setBoolean(DFS_CBLOCK_TRACE_IO, true);
-    config.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
-    cluster = new MiniOzoneClassicCluster.Builder(config)
-        .numDataNodes(1)
-        .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
-    storageContainerLocationClient = cluster
-        .createStorageContainerLocationClient();
-    xceiverClientManager = new XceiverClientManager(config);
-  }
-
-  @AfterClass
-  public static void shutdown() throws InterruptedException {
-    if (cluster != null) {
-      cluster.shutdown();
-    }
-    IOUtils.cleanupWithLogger(null, storageContainerLocationClient, cluster);
-  }
-
-  /**
-   * getContainerPipelines creates a set of containers and returns the
-   * Pipelines that define those containers.
-   *
-   * @param count - Number of containers to create.
-   * @return - List of Pipelines.
-   * @throws IOException throws Exception
-   */
-  private List<Pipeline> getContainerPipeline(int count) throws IOException {
-    List<Pipeline> containerPipelines = new LinkedList<>();
-    for (int x = 0; x < count; x++) {
-      String traceID = "trace" + RandomStringUtils.randomNumeric(4);
-      String containerName = "container" + RandomStringUtils.randomNumeric(10);
-      Pipeline pipeline =
-          storageContainerLocationClient.allocateContainer(
-              xceiverClientManager.getType(),
-              xceiverClientManager.getFactor(), containerName, "CBLOCK");
-      XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline);
-      ContainerProtocolCalls.createContainer(client, traceID);
-      // This step is needed since we set private data on pipelines, when we
-      // read the list from CBlockServer. So we mimic that action here.
-      pipeline.setData(Longs.toByteArray(x));
-      containerPipelines.add(pipeline);
-      xceiverClientManager.releaseClient(client);
-    }
-    return containerPipelines;
-  }
-
-  /**
-   * This test creates a cache and performs a simple write / read.
-   * Due to the cache - we have Read-after-write consistency for cBlocks.
-   *
-   * @throws IOException throws Exception
-   */
-  @Test
-  public void testCacheWriteRead() throws IOException,
-      InterruptedException, TimeoutException {
-    final long blockID = 0;
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
-    String userName = "user" + RandomStringUtils.randomNumeric(4);
-    String data = RandomStringUtils.random(4 * KB);
-    String dataHash = DigestUtils.sha256Hex(data);
-    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
-        xceiverClientManager, metrics);
-    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
-        .setConfiguration(this.config)
-        .setVolumeName(volumeName)
-        .setUserName(userName)
-        .setPipelines(getContainerPipeline(10))
-        .setClientManager(xceiverClientManager)
-        .setBlockSize(4 * KB)
-        .setVolumeSize(50 * GB)
-        .setFlusher(flusher)
-        .setCBlockTargetMetrics(metrics)
-        .build();
-    cache.start();
-    cache.put(blockID, data.getBytes(StandardCharsets.UTF_8));
-    Assert.assertEquals(1, metrics.getNumWriteOps());
-    // Please note that this read is from the local cache.
-    LogicalBlock block = cache.get(blockID);
-    Assert.assertEquals(1, metrics.getNumReadOps());
-    Assert.assertEquals(1, metrics.getNumReadCacheHits());
-    Assert.assertEquals(0, metrics.getNumReadCacheMiss());
-    Assert.assertEquals(0, metrics.getNumReadLostBlocks());
-
-    cache.put(blockID + 1, data.getBytes(StandardCharsets.UTF_8));
-    Assert.assertEquals(2, metrics.getNumWriteOps());
-    // Please note that this read is from the local cache.
-    block = cache.get(blockID + 1);
-    Assert.assertEquals(2, metrics.getNumReadOps());
-    Assert.assertEquals(2, metrics.getNumReadCacheHits());
-    Assert.assertEquals(0, metrics.getNumReadCacheMiss());
-    Assert.assertEquals(0, metrics.getNumReadLostBlocks());
-    String readHash = DigestUtils.sha256Hex(block.getData().array());
-    Assert.assertEquals("File content does not match.", dataHash, readHash);
-    GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
-    cache.close();
-
-  }
-
-  @Test
-  public void testCacheWriteToRemoteContainer() throws IOException,
-      InterruptedException, TimeoutException {
-    final long blockID = 0;
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
-    String userName = "user" + RandomStringUtils.randomNumeric(4);
-    String data = RandomStringUtils.random(4 * KB);
-    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
-        xceiverClientManager, metrics);
-    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
-        .setConfiguration(this.config)
-        .setVolumeName(volumeName)
-        .setUserName(userName)
-        .setPipelines(getContainerPipeline(10))
-        .setClientManager(xceiverClientManager)
-        .setBlockSize(4 * KB)
-        .setVolumeSize(50 * GB)
-        .setFlusher(flusher)
-        .setCBlockTargetMetrics(metrics)
-        .build();
-    cache.start();
-    cache.put(blockID, data.getBytes(StandardCharsets.UTF_8));
-    GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
-    cache.close();
-  }
-
-  @Test
-  public void testCacheWriteToRemote50KBlocks() throws IOException,
-      InterruptedException, TimeoutException {
-    final long totalBlocks = 50 * 1000;
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
-    String userName = "user" + RandomStringUtils.randomNumeric(4);
-    String data = RandomStringUtils.random(4 * KB);
-    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
-        xceiverClientManager, metrics);
-    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
-        .setConfiguration(this.config)
-        .setVolumeName(volumeName)
-        .setUserName(userName)
-        .setPipelines(getContainerPipeline(10))
-        .setClientManager(xceiverClientManager)
-        .setBlockSize(4 * 1024)
-        .setVolumeSize(50 * GB)
-        .setFlusher(flusher)
-        .setCBlockTargetMetrics(metrics)
-        .build();
-    cache.start();
-    long startTime = Time.monotonicNow();
-    for (long blockid = 0; blockid < totalBlocks; blockid++) {
-      cache.put(blockid, data.getBytes(StandardCharsets.UTF_8));
-    }
-    Assert.assertEquals(totalBlocks, metrics.getNumWriteOps());
-    Assert.assertEquals(totalBlocks,  metrics.getNumBlockBufferUpdates());
-    LOG.info("Wrote 50K blocks, waiting for replication to finish.");
-    GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
-    long endTime = Time.monotonicNow();
-    LOG.info("Time taken for writing {} blocks is {} seconds", totalBlocks,
-        TimeUnit.MILLISECONDS.toSeconds(endTime - startTime));
-    // TODO: Read this data back.
-    cache.close();
-  }
-
-  @Test
-  public void testCacheInvalidBlock() throws IOException {
-    final int blockID = 1024;
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
-    String userName = "user" + RandomStringUtils.randomNumeric(4);
-    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
-        xceiverClientManager, metrics);
-    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
-        .setConfiguration(this.config)
-        .setVolumeName(volumeName)
-        .setUserName(userName)
-        .setPipelines(getContainerPipeline(10))
-        .setClientManager(xceiverClientManager)
-        .setBlockSize(4 * KB)
-        .setVolumeSize(50 * GB)
-        .setFlusher(flusher)
-        .setCBlockTargetMetrics(metrics)
-        .build();
-    cache.start();
-    // Read a non-existent block ID.
-    LogicalBlock block = cache.get(blockID);
-    Assert.assertNotNull(block);
-    Assert.assertEquals(4 * 1024, block.getData().array().length);
-    Assert.assertEquals(1, metrics.getNumReadOps());
-    Assert.assertEquals(1, metrics.getNumReadLostBlocks());
-    Assert.assertEquals(1, metrics.getNumReadCacheMiss());
-    cache.close();
-  }
-
-  @Test
-  public void testReadWriteCorrectness() throws IOException,
-      InterruptedException, TimeoutException {
-    Random r = new Random();
-    final int maxBlock = 12500000;
-    final int blockCount = 10 * 1000;
-    Map<Long, String> blockShaMap = new HashMap<>();
-    List<Pipeline> pipelines = getContainerPipeline(10);
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
-    String userName = "user" + RandomStringUtils.randomNumeric(4);
-    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
-        xceiverClientManager, metrics);
-    final CBlockLocalCache cache = CBlockLocalCache.newBuilder()
-        .setConfiguration(this.config)
-        .setVolumeName(volumeName)
-        .setUserName(userName)
-        .setPipelines(pipelines)
-        .setClientManager(xceiverClientManager)
-        .setBlockSize(4 * KB)
-        .setVolumeSize(50 * GB)
-        .setFlusher(flusher)
-        .setCBlockTargetMetrics(metrics)
-        .build();
-    cache.start();
-    for (int x = 0; x < blockCount; x++) {
-      String data = RandomStringUtils.random(4 * 1024);
-      String dataHash = DigestUtils.sha256Hex(data);
-      long blockId = abs(r.nextInt(maxBlock));
-      blockShaMap.put(blockId, dataHash);
-      cache.put(blockId, data.getBytes(StandardCharsets.UTF_8));
-    }
-    Assert.assertEquals(blockCount, metrics.getNumWriteOps());
-    GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
-    LOG.info("Finished with putting blocks ..starting reading blocks back. " +
-        "unique blocks : {}", blockShaMap.size());
-    // Test reading from local cache.
-    for (Map.Entry<Long, String> entry : blockShaMap.entrySet()) {
-      LogicalBlock block = cache.get(entry.getKey());
-      String blockSha = DigestUtils.sha256Hex(block.getData().array());
-      Assert.assertEquals("Block data is not equal", entry.getValue(),
-          blockSha);
-    }
-    Assert.assertEquals(blockShaMap.size(), metrics.getNumReadOps());
-    Assert.assertEquals(blockShaMap.size(), metrics.getNumReadCacheHits());
-    Assert.assertEquals(0, metrics.getNumReadCacheMiss());
-    Assert.assertEquals(0, metrics.getNumReadLostBlocks());
-
-    LOG.info("Finished with reading blocks, SUCCESS.");
-    // Close and discard local cache.
-    cache.close();
-    LOG.info("Closing the and destroying local cache");
-    CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher newflusher = new ContainerCacheFlusher(this.config,
-        xceiverClientManager, newMetrics);
-    Assert.assertEquals(0, newMetrics.getNumReadCacheHits());
-    CBlockLocalCache newCache = null;
-    try {
-      newCache = CBlockLocalCache.newBuilder()
-          .setConfiguration(this.config)
-          .setVolumeName(volumeName)
-          .setUserName(userName)
-          .setPipelines(pipelines)
-          .setClientManager(xceiverClientManager)
-          .setBlockSize(4 * KB)
-          .setVolumeSize(50 * GB)
-          .setFlusher(newflusher)
-          .setCBlockTargetMetrics(newMetrics)
-          .build();
-      newCache.start();
-      for (Map.Entry<Long, String> entry : blockShaMap.entrySet()) {
-        LogicalBlock block = newCache.get(entry.getKey());
-        String blockSha = DigestUtils.sha256Hex(block.getData().array());
-        Assert.assertEquals("Block data is not equal", entry.getValue(),
-            blockSha);
-      }
-
-      Assert.assertEquals(blockShaMap.size(), newMetrics.getNumReadOps());
-      Assert.assertEquals(blockShaMap.size(), newMetrics.getNumReadCacheHits());
-      Assert.assertEquals(0, newMetrics.getNumReadCacheMiss());
-      Assert.assertEquals(0, newMetrics.getNumReadLostBlocks());
-
-      LOG.info("Finished with reading blocks from remote cache, SUCCESS.");
-    } finally {
-      if (newCache != null) {
-        newCache.close();
-      }
-    }
-  }
-
-  @Test
-  public void testStorageImplReadWrite() throws IOException,
-      InterruptedException, TimeoutException {
-    String userName = "user" + RandomStringUtils.randomNumeric(5);
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
-    long volumeSize = 50L * (1024L * 1024L * 1024L);
-    int blockSize = 4096;
-    byte[] data =
-        RandomStringUtils.randomAlphanumeric(10 * (1024 * 1024))
-            .getBytes(StandardCharsets.UTF_8);
-    String hash = DigestUtils.sha256Hex(data);
-    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
-        xceiverClientManager, metrics);
-    CBlockIStorageImpl ozoneStore = CBlockIStorageImpl.newBuilder()
-        .setUserName(userName)
-        .setVolumeName(volumeName)
-        .setVolumeSize(volumeSize)
-        .setBlockSize(blockSize)
-        .setContainerList(getContainerPipeline(10))
-        .setClientManager(xceiverClientManager)
-        .setConf(this.config)
-        .setFlusher(flusher)
-        .setCBlockTargetMetrics(metrics)
-        .build();
-    ozoneStore.write(data, 0);
-
-    byte[] newData = new byte[10 * 1024 * 1024];
-    ozoneStore.read(newData, 0);
-    String newHash = DigestUtils.sha256Hex(newData);
-    Assert.assertEquals("hashes don't match.", hash, newHash);
-    GenericTestUtils.waitFor(() -> !ozoneStore.getCache().isDirtyCache(),
-        100, 20 * 1000);
-    ozoneStore.close();
-  }
-
-  //@Test
-  // Disabling this test for time being since the bug in JSCSI
-  // forces us always to have a local cache.
-  public void testStorageImplNoLocalCache() throws IOException,
-      InterruptedException, TimeoutException {
-    OzoneConfiguration oConfig = new OzoneConfiguration();
-    oConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, false);
-    oConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
-    String userName = "user" + RandomStringUtils.randomNumeric(5);
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
-    long volumeSize = 50L * (1024L * 1024L * 1024L);
-    int blockSize = 4096;
-    byte[] data =
-        RandomStringUtils.randomAlphanumeric(10 * (1024 * 1024))
-            .getBytes(StandardCharsets.UTF_8);
-    String hash = DigestUtils.sha256Hex(data);
-    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
-    ContainerCacheFlusher flusher = new ContainerCacheFlusher(oConfig,
-        xceiverClientManager, metrics);
-    CBlockIStorageImpl ozoneStore = CBlockIStorageImpl.newBuilder()
-        .setUserName(userName)
-        .setVolumeName(volumeName)
-        .setVolumeSize(volumeSize)
-        .setBlockSize(blockSize)
-        .setContainerList(getContainerPipeline(10))
-        .setClientManager(xceiverClientManager)
-        .setConf(oConfig)
-        .setFlusher(flusher)
-        .setCBlockTargetMetrics(metrics)
-        .build();
-    ozoneStore.write(data, 0);
-
-    byte[] newData = new byte[10 * 1024 * 1024];
-    ozoneStore.read(newData, 0);
-    String newHash = DigestUtils.sha256Hex(newData);
-    Assert.assertEquals("hashes don't match.", hash, newHash);
-    GenericTestUtils.waitFor(() -> !ozoneStore.getCache().isDirtyCache(),
-        100, 20 * 1000);
-    ozoneStore.close();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java
deleted file mode 100644
index 02c23ff..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.cblock.kubernetes;
-
-import io.kubernetes.client.JSON;
-import io.kubernetes.client.models.V1PersistentVolume;
-import io.kubernetes.client.models.V1PersistentVolumeClaim;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_ISCSI_ADVERTISED_IP;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.nio.file.Files;
-import java.nio.file.Paths;
-
-/**
- * Test the resource generation of Dynamic Provisioner.
- */
-public class TestDynamicProvisioner {
-
-  @Test
-  public void persitenceVolumeBuilder() throws Exception {
-
-    OzoneConfiguration conf = new OzoneConfiguration();
-    conf.setStrings(DFS_CBLOCK_ISCSI_ADVERTISED_IP, "1.2.3.4");
-
-    DynamicProvisioner provisioner =
-        new DynamicProvisioner(conf, null);
-
-    String pvc = new String(Files.readAllBytes(
-        Paths.get(getClass().getResource(
-                "/dynamicprovisioner/input1-pvc.json").toURI())));
-
-    String pv = new String(Files.readAllBytes(
-        Paths.get(getClass().getResource(
-            "/dynamicprovisioner/expected1-pv.json").toURI())));
-
-    JSON json = new io.kubernetes.client.JSON();
-
-    V1PersistentVolumeClaim claim =
-        json.getGson().fromJson(pvc, V1PersistentVolumeClaim.class);
-
-    String volumeName = provisioner.createVolumeName(claim);
-
-    V1PersistentVolume volume =
-        provisioner.persitenceVolumeBuilder(claim, volumeName);
-
-    //remove the data which should not been compared
-    V1PersistentVolume expectedVolume =
-        json.getGson().fromJson(pv, V1PersistentVolume.class);
-
-
-    Assert.assertEquals(expectedVolume, volume);
-  }
-
-}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[43/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
deleted file mode 100644
index 05a632e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
+++ /dev/null
@@ -1,492 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.client.io;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Result;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfoGroup;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.ksm.protocolPB.KeySpaceManagerProtocolClientSideTranslatorPB;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.container.common.helpers
-    .StorageContainerException;
-import org.apache.hadoop.scm.protocolPB
-    .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.storage.ChunkOutputStream;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Maintaining a list of ChunkInputStream. Write based on offset.
- *
- * Note that this may write to multiple containers in one write call. In case
- * that first container succeeded but later ones failed, the succeeded writes
- * are not rolled back.
- *
- * TODO : currently not support multi-thread access.
- */
-public class ChunkGroupOutputStream extends OutputStream {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ChunkGroupOutputStream.class);
-
-  // array list's get(index) is O(1)
-  private final ArrayList<ChunkOutputStreamEntry> streamEntries;
-  private int currentStreamIndex;
-  private long byteOffset;
-  private final KeySpaceManagerProtocolClientSideTranslatorPB ksmClient;
-  private final
-      StorageContainerLocationProtocolClientSideTranslatorPB scmClient;
-  private final KsmKeyArgs keyArgs;
-  private final int openID;
-  private final XceiverClientManager xceiverClientManager;
-  private final int chunkSize;
-  private final String requestID;
-  private boolean closed;
-
-  /**
-   * A constructor for testing purpose only.
-   */
-  @VisibleForTesting
-  public ChunkGroupOutputStream() {
-    streamEntries = new ArrayList<>();
-    ksmClient = null;
-    scmClient = null;
-    keyArgs = null;
-    openID = -1;
-    xceiverClientManager = null;
-    chunkSize = 0;
-    requestID = null;
-    closed = false;
-  }
-
-  /**
-   * For testing purpose only. Not building output stream from blocks, but
-   * taking from externally.
-   *
-   * @param outputStream
-   * @param length
-   */
-  @VisibleForTesting
-  public synchronized void addStream(OutputStream outputStream, long length) {
-    streamEntries.add(new ChunkOutputStreamEntry(outputStream, length));
-  }
-
-  @VisibleForTesting
-  public List<ChunkOutputStreamEntry> getStreamEntries() {
-    return streamEntries;
-  }
-
-  public ChunkGroupOutputStream(
-      OpenKeySession handler, XceiverClientManager xceiverClientManager,
-      StorageContainerLocationProtocolClientSideTranslatorPB scmClient,
-      KeySpaceManagerProtocolClientSideTranslatorPB ksmClient,
-      int chunkSize, String requestId, ReplicationFactor factor,
-      ReplicationType type) throws IOException {
-    this.streamEntries = new ArrayList<>();
-    this.currentStreamIndex = 0;
-    this.byteOffset = 0;
-    this.ksmClient = ksmClient;
-    this.scmClient = scmClient;
-    KsmKeyInfo info = handler.getKeyInfo();
-    this.keyArgs = new KsmKeyArgs.Builder()
-        .setVolumeName(info.getVolumeName())
-        .setBucketName(info.getBucketName())
-        .setKeyName(info.getKeyName())
-        .setType(type)
-        .setFactor(factor)
-        .setDataSize(info.getDataSize()).build();
-    this.openID = handler.getId();
-    this.xceiverClientManager = xceiverClientManager;
-    this.chunkSize = chunkSize;
-    this.requestID = requestId;
-    LOG.debug("Expecting open key with one block, but got" +
-        info.getKeyLocationVersions().size());
-  }
-
-  /**
-   * When a key is opened, it is possible that there are some blocks already
-   * allocated to it for this open session. In this case, to make use of these
-   * blocks, we need to add these blocks to stream entries. But, a key's version
-   * also includes blocks from previous versions, we need to avoid adding these
-   * old blocks to stream entries, because these old blocks should not be picked
-   * for write. To do this, the following method checks that, only those
-   * blocks created in this particular open version are added to stream entries.
-   *
-   * @param version the set of blocks that are pre-allocated.
-   * @param openVersion the version corresponding to the pre-allocation.
-   * @throws IOException
-   */
-  public void addPreallocateBlocks(KsmKeyLocationInfoGroup version,
-      long openVersion) throws IOException {
-    // server may return any number of blocks, (0 to any)
-    // only the blocks allocated in this open session (block createVersion
-    // equals to open session version)
-    for (KsmKeyLocationInfo subKeyInfo : version.getLocationList()) {
-      if (subKeyInfo.getCreateVersion() == openVersion) {
-        checkKeyLocationInfo(subKeyInfo);
-      }
-    }
-  }
-
-  private void checkKeyLocationInfo(KsmKeyLocationInfo subKeyInfo)
-      throws IOException {
-    String containerKey = subKeyInfo.getBlockID();
-    String containerName = subKeyInfo.getContainerName();
-    Pipeline pipeline = scmClient.getContainer(containerName);
-    XceiverClientSpi xceiverClient =
-        xceiverClientManager.acquireClient(pipeline);
-    // create container if needed
-    if (subKeyInfo.getShouldCreateContainer()) {
-      try {
-        ContainerProtocolCalls.createContainer(xceiverClient, requestID);
-        scmClient.notifyObjectStageChange(
-            ObjectStageChangeRequestProto.Type.container,
-            containerName, ObjectStageChangeRequestProto.Op.create,
-            ObjectStageChangeRequestProto.Stage.complete);
-      } catch (StorageContainerException ex) {
-        if (ex.getResult().equals(Result.CONTAINER_EXISTS)) {
-          //container already exist, this should never happen
-          LOG.debug("Container {} already exists.", containerName);
-        } else {
-          LOG.error("Container creation failed for {}.", containerName, ex);
-          throw ex;
-        }
-      }
-    }
-    streamEntries.add(new ChunkOutputStreamEntry(containerKey,
-        keyArgs.getKeyName(), xceiverClientManager, xceiverClient, requestID,
-        chunkSize, subKeyInfo.getLength()));
-  }
-
-
-  @VisibleForTesting
-  public long getByteOffset() {
-    return byteOffset;
-  }
-
-
-  @Override
-  public synchronized void write(int b) throws IOException {
-    checkNotClosed();
-
-    if (streamEntries.size() <= currentStreamIndex) {
-      Preconditions.checkNotNull(ksmClient);
-      // allocate a new block, if a exception happens, log an error and
-      // throw exception to the caller directly, and the write fails.
-      try {
-        allocateNewBlock(currentStreamIndex);
-      } catch (IOException ioe) {
-        LOG.error("Allocate block fail when writing.");
-        throw ioe;
-      }
-    }
-    ChunkOutputStreamEntry entry = streamEntries.get(currentStreamIndex);
-    entry.write(b);
-    if (entry.getRemaining() <= 0) {
-      currentStreamIndex += 1;
-    }
-    byteOffset += 1;
-  }
-
-  /**
-   * Try to write the bytes sequence b[off:off+len) to streams.
-   *
-   * NOTE: Throws exception if the data could not fit into the remaining space.
-   * In which case nothing will be written.
-   * TODO:May need to revisit this behaviour.
-   *
-   * @param b byte data
-   * @param off starting offset
-   * @param len length to write
-   * @throws IOException
-   */
-  @Override
-  public synchronized void write(byte[] b, int off, int len)
-      throws IOException {
-    checkNotClosed();
-
-    if (b == null) {
-      throw new NullPointerException();
-    }
-    if ((off < 0) || (off > b.length) || (len < 0) ||
-        ((off + len) > b.length) || ((off + len) < 0)) {
-      throw new IndexOutOfBoundsException();
-    }
-    if (len == 0) {
-      return;
-    }
-    int succeededAllocates = 0;
-    while (len > 0) {
-      if (streamEntries.size() <= currentStreamIndex) {
-        Preconditions.checkNotNull(ksmClient);
-        // allocate a new block, if a exception happens, log an error and
-        // throw exception to the caller directly, and the write fails.
-        try {
-          allocateNewBlock(currentStreamIndex);
-          succeededAllocates += 1;
-        } catch (IOException ioe) {
-          LOG.error("Try to allocate more blocks for write failed, already " +
-              "allocated " + succeededAllocates + " blocks for this write.");
-          throw ioe;
-        }
-      }
-      // in theory, this condition should never violate due the check above
-      // still do a sanity check.
-      Preconditions.checkArgument(currentStreamIndex < streamEntries.size());
-      ChunkOutputStreamEntry current = streamEntries.get(currentStreamIndex);
-      int writeLen = Math.min(len, (int)current.getRemaining());
-      current.write(b, off, writeLen);
-      if (current.getRemaining() <= 0) {
-        currentStreamIndex += 1;
-      }
-      len -= writeLen;
-      off += writeLen;
-      byteOffset += writeLen;
-    }
-  }
-
-  /**
-   * Contact KSM to get a new block. Set the new block with the index (e.g.
-   * first block has index = 0, second has index = 1 etc.)
-   *
-   * The returned block is made to new ChunkOutputStreamEntry to write.
-   *
-   * @param index the index of the block.
-   * @throws IOException
-   */
-  private void allocateNewBlock(int index) throws IOException {
-    KsmKeyLocationInfo subKeyInfo = ksmClient.allocateBlock(keyArgs, openID);
-    checkKeyLocationInfo(subKeyInfo);
-  }
-
-  @Override
-  public synchronized void flush() throws IOException {
-    checkNotClosed();
-    for (int i = 0; i <= currentStreamIndex; i++) {
-      streamEntries.get(i).flush();
-    }
-  }
-
-  /**
-   * Commit the key to KSM, this will add the blocks as the new key blocks.
-   *
-   * @throws IOException
-   */
-  @Override
-  public synchronized void close() throws IOException {
-    if (closed) {
-      return;
-    }
-    closed = true;
-    for (ChunkOutputStreamEntry entry : streamEntries) {
-      if (entry != null) {
-        entry.close();
-      }
-    }
-    if (keyArgs != null) {
-      // in test, this could be null
-      keyArgs.setDataSize(byteOffset);
-      ksmClient.commitKey(keyArgs, openID);
-    } else {
-      LOG.warn("Closing ChunkGroupOutputStream, but key args is null");
-    }
-  }
-
-  /**
-   * Builder class of ChunkGroupOutputStream.
-   */
-  public static class Builder {
-    private OpenKeySession openHandler;
-    private XceiverClientManager xceiverManager;
-    private StorageContainerLocationProtocolClientSideTranslatorPB scmClient;
-    private KeySpaceManagerProtocolClientSideTranslatorPB ksmClient;
-    private int chunkSize;
-    private String requestID;
-    private ReplicationType type;
-    private ReplicationFactor factor;
-
-    public Builder setHandler(OpenKeySession handler) {
-      this.openHandler = handler;
-      return this;
-    }
-
-    public Builder setXceiverClientManager(XceiverClientManager manager) {
-      this.xceiverManager = manager;
-      return this;
-    }
-
-    public Builder setScmClient(
-        StorageContainerLocationProtocolClientSideTranslatorPB client) {
-      this.scmClient = client;
-      return this;
-    }
-
-    public Builder setKsmClient(
-        KeySpaceManagerProtocolClientSideTranslatorPB client) {
-      this.ksmClient = client;
-      return this;
-    }
-
-    public Builder setChunkSize(int size) {
-      this.chunkSize = size;
-      return this;
-    }
-
-    public Builder setRequestID(String id) {
-      this.requestID = id;
-      return this;
-    }
-
-    public Builder setType(ReplicationType replicationType) {
-      this.type = replicationType;
-      return this;
-    }
-
-    public Builder setFactor(ReplicationFactor replicationFactor) {
-      this.factor = replicationFactor;
-      return this;
-    }
-
-    public ChunkGroupOutputStream build() throws IOException {
-      return new ChunkGroupOutputStream(openHandler, xceiverManager, scmClient,
-          ksmClient, chunkSize, requestID, factor, type);
-    }
-  }
-
-  private static class ChunkOutputStreamEntry extends OutputStream {
-    private OutputStream outputStream;
-    private final String containerKey;
-    private final String key;
-    private final XceiverClientManager xceiverClientManager;
-    private final XceiverClientSpi xceiverClient;
-    private final String requestId;
-    private final int chunkSize;
-    // total number of bytes that should be written to this stream
-    private final long length;
-    // the current position of this stream 0 <= currentPosition < length
-    private long currentPosition;
-
-    ChunkOutputStreamEntry(String containerKey, String key,
-        XceiverClientManager xceiverClientManager,
-        XceiverClientSpi xceiverClient, String requestId, int chunkSize,
-        long length) {
-      this.outputStream = null;
-      this.containerKey = containerKey;
-      this.key = key;
-      this.xceiverClientManager = xceiverClientManager;
-      this.xceiverClient = xceiverClient;
-      this.requestId = requestId;
-      this.chunkSize = chunkSize;
-
-      this.length = length;
-      this.currentPosition = 0;
-    }
-
-    /**
-     * For testing purpose, taking a some random created stream instance.
-     * @param  outputStream a existing writable output stream
-     * @param  length the length of data to write to the stream
-     */
-    ChunkOutputStreamEntry(OutputStream outputStream, long length) {
-      this.outputStream = outputStream;
-      this.containerKey = null;
-      this.key = null;
-      this.xceiverClientManager = null;
-      this.xceiverClient = null;
-      this.requestId = null;
-      this.chunkSize = -1;
-
-      this.length = length;
-      this.currentPosition = 0;
-    }
-
-    long getLength() {
-      return length;
-    }
-
-    long getRemaining() {
-      return length - currentPosition;
-    }
-
-    private synchronized void checkStream() {
-      if (this.outputStream == null) {
-        this.outputStream = new ChunkOutputStream(containerKey,
-            key, xceiverClientManager, xceiverClient,
-            requestId, chunkSize);
-      }
-    }
-
-    @Override
-    public void write(int b) throws IOException {
-      checkStream();
-      outputStream.write(b);
-      this.currentPosition += 1;
-    }
-
-    @Override
-    public void write(byte[] b, int off, int len) throws IOException {
-      checkStream();
-      outputStream.write(b, off, len);
-      this.currentPosition += len;
-    }
-
-    @Override
-    public void flush() throws IOException {
-      if (this.outputStream != null) {
-        this.outputStream.flush();
-      }
-    }
-
-    @Override
-    public void close() throws IOException {
-      if (this.outputStream != null) {
-        this.outputStream.close();
-      }
-    }
-  }
-
-  /**
-   * Verify that the output stream is open. Non blocking; this gives
-   * the last state of the volatile {@link #closed} field.
-   * @throws IOException if the connection is closed.
-   */
-  private void checkNotClosed() throws IOException {
-    if (closed) {
-      throw new IOException(
-          ": " + FSExceptionMessages.STREAM_IS_CLOSED + " Key: " + keyArgs
-              .getKeyName());
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/LengthInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/LengthInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/LengthInputStream.java
deleted file mode 100644
index baf1887..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/LengthInputStream.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.io;
-
-import java.io.FilterInputStream;
-import java.io.InputStream;
-
-/**
- * An input stream with length.
- */
-public class LengthInputStream extends FilterInputStream {
-
-  private final long length;
-
-  /**
-   * Create an stream.
-   * @param in the underlying input stream.
-   * @param length the length of the stream.
-   */
-  public LengthInputStream(InputStream in, long length) {
-    super(in);
-    this.length = length;
-  }
-
-  /** @return the length. */
-  public long getLength() {
-    return length;
-  }
-
-  public InputStream getWrappedStream() {
-    return in;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneContainerTranslation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneContainerTranslation.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneContainerTranslation.java
deleted file mode 100644
index ca6f7aa..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneContainerTranslation.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.io;
-
-
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyData;
-
-/**
- * This class contains methods that define the translation between the Ozone
- * domain model and the storage container domain model.
- */
-final class OzoneContainerTranslation {
-
-  /**
-   * Creates key data intended for reading a container key.
-   *
-   * @param containerName container name
-   * @param containerKey container key
-   * @return KeyData intended for reading the container key
-   */
-  public static KeyData containerKeyDataForRead(String containerName,
-      String containerKey) {
-    return KeyData
-        .newBuilder()
-        .setContainerName(containerName)
-        .setName(containerKey)
-        .build();
-  }
-
-  /**
-   * There is no need to instantiate this class.
-   */
-  private OzoneContainerTranslation() {
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneInputStream.java
deleted file mode 100644
index 3857bd0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneInputStream.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.client.io;
-
-import org.apache.hadoop.scm.storage.ChunkInputStream;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * OzoneInputStream is used to read data from Ozone.
- * It uses SCM's {@link ChunkInputStream} for reading the data.
- */
-public class OzoneInputStream extends InputStream {
-
-  private final InputStream inputStream;
-
-  /**
-   * Constructs OzoneInputStream with ChunkInputStream.
-   *
-   * @param inputStream
-   */
-  public OzoneInputStream(InputStream inputStream) {
-    this.inputStream = inputStream;
-  }
-
-  @Override
-  public int read() throws IOException {
-    return inputStream.read();
-  }
-
-  @Override
-  public synchronized void close() throws IOException {
-    inputStream.close();
-  }
-
-  @Override
-  public int available() throws IOException {
-    return inputStream.available();
-  }
-
-  public InputStream getInputStream() {
-    return inputStream;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneOutputStream.java
deleted file mode 100644
index 5369220..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneOutputStream.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.client.io;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * OzoneOutputStream is used to write data into Ozone.
- * It uses SCM's {@link ChunkGroupOutputStream} for writing the data.
- */
-public class OzoneOutputStream extends OutputStream {
-
-  private final OutputStream outputStream;
-
-  /**
-   * Constructs OzoneOutputStream with ChunkGroupOutputStream.
-   *
-   * @param outputStream
-   */
-  public OzoneOutputStream(OutputStream outputStream) {
-    this.outputStream = outputStream;
-  }
-
-  @Override
-  public void write(int b) throws IOException {
-    outputStream.write(b);
-  }
-
-  @Override
-  public void write(byte[] b, int off, int len) throws IOException {
-    outputStream.write(b, off, len);
-  }
-
-  @Override
-  public synchronized void flush() throws IOException {
-    outputStream.flush();
-  }
-
-  @Override
-  public synchronized void close() throws IOException {
-    //commitKey can be done here, if needed.
-    outputStream.close();
-  }
-
-  public OutputStream getOutputStream() {
-    return outputStream;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/package-info.java
deleted file mode 100644
index 493ece8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.io;
-
-/**
- * This package contains Ozone I/O classes.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/package-info.java
deleted file mode 100644
index 7e2591a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-/**
- * This package contains Ozone Client classes.
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
deleted file mode 100644
index 64a970e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ /dev/null
@@ -1,323 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.protocol;
-
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.BucketArgs;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneKey;
-import org.apache.hadoop.ozone.client.OzoneQuota;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.ReplicationFactor;
-import org.apache.hadoop.ozone.client.ReplicationType;
-import org.apache.hadoop.ozone.client.VolumeArgs;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * An implementer of this interface is capable of connecting to Ozone Cluster
- * and perform client operations. The protocol used for communication is
- * determined by the implementation class specified by
- * property <code>ozone.client.protocol</code>. The build-in implementation
- * includes: {@link org.apache.hadoop.ozone.client.rpc.RpcClient} for RPC and
- * {@link  org.apache.hadoop.ozone.client.rest.RestClient} for REST.
- */
-public interface ClientProtocol {
-
-  /**
-   * Creates a new Volume.
-   * @param volumeName Name of the Volume
-   * @throws IOException
-   */
-  void createVolume(String volumeName)
-      throws IOException;
-
-  /**
-   * Creates a new Volume with properties set in VolumeArgs.
-   * @param volumeName Name of the Volume
-   * @param args Properties to be set for the Volume
-   * @throws IOException
-   */
-  void createVolume(String volumeName, VolumeArgs args)
-      throws IOException;
-
-  /**
-   * Sets the owner of volume.
-   * @param volumeName Name of the Volume
-   * @param owner to be set for the Volume
-   * @throws IOException
-   */
-  void setVolumeOwner(String volumeName, String owner) throws IOException;
-
-  /**
-   * Set Volume Quota.
-   * @param volumeName Name of the Volume
-   * @param quota Quota to be set for the Volume
-   * @throws IOException
-   */
-  void setVolumeQuota(String volumeName, OzoneQuota quota)
-      throws IOException;
-
-  /**
-   * Returns {@link OzoneVolume}.
-   * @param volumeName Name of the Volume
-   * @return {@link OzoneVolume}
-   * @throws IOException
-   * */
-  OzoneVolume getVolumeDetails(String volumeName)
-      throws IOException;
-
-  /**
-   * Checks if a Volume exists and the user with a role specified has access
-   * to the Volume.
-   * @param volumeName Name of the Volume
-   * @param acl requested acls which needs to be checked for access
-   * @return Boolean - True if the user with a role can access the volume.
-   * This is possible for owners of the volume and admin users
-   * @throws IOException
-   */
-  boolean checkVolumeAccess(String volumeName, OzoneAcl acl)
-      throws IOException;
-
-  /**
-   * Deletes an empty Volume.
-   * @param volumeName Name of the Volume
-   * @throws IOException
-   */
-  void deleteVolume(String volumeName) throws IOException;
-
-  /**
-   * Lists all volumes in the cluster that matches the volumePrefix,
-   * size of the returned list depends on maxListResult. If volume prefix
-   * is null, returns all the volumes. The caller has to make multiple calls
-   * to read all volumes.
-   *
-   * @param volumePrefix Volume prefix to match
-   * @param prevVolume Starting point of the list, this volume is excluded
-   * @param maxListResult Max number of volumes to return.
-   * @return {@code List<OzoneVolume>}
-   * @throws IOException
-   */
-  List<OzoneVolume> listVolumes(String volumePrefix, String prevVolume,
-                                int maxListResult)
-      throws IOException;
-
-  /**
-   * Lists all volumes in the cluster that are owned by the specified
-   * user and matches the volumePrefix, size of the returned list depends on
-   * maxListResult. If the user is null, return volumes owned by current user.
-   * If volume prefix is null, returns all the volumes. The caller has to make
-   * multiple calls to read all volumes.
-   *
-   * @param user User Name
-   * @param volumePrefix Volume prefix to match
-   * @param prevVolume Starting point of the list, this volume is excluded
-   * @param maxListResult Max number of volumes to return.
-   * @return {@code List<OzoneVolume>}
-   * @throws IOException
-   */
-  List<OzoneVolume> listVolumes(String user, String volumePrefix,
-                                    String prevVolume, int maxListResult)
-      throws IOException;
-
-  /**
-   * Creates a new Bucket in the Volume.
-   * @param volumeName Name of the Volume
-   * @param bucketName Name of the Bucket
-   * @throws IOException
-   */
-  void createBucket(String volumeName, String bucketName)
-      throws IOException;
-
-  /**
-   * Creates a new Bucket in the Volume, with properties set in BucketArgs.
-   * @param volumeName Name of the Volume
-   * @param bucketName Name of the Bucket
-   * @param bucketArgs Bucket Arguments
-   * @throws IOException
-   */
-  void createBucket(String volumeName, String bucketName,
-                    BucketArgs bucketArgs)
-      throws IOException;
-
-  /**
-   * Adds ACLs to the Bucket.
-   * @param volumeName Name of the Volume
-   * @param bucketName Name of the Bucket
-   * @param addAcls ACLs to be added
-   * @throws IOException
-   */
-  void addBucketAcls(String volumeName, String bucketName,
-                     List<OzoneAcl> addAcls)
-      throws IOException;
-
-  /**
-   * Removes ACLs from a Bucket.
-   * @param volumeName Name of the Volume
-   * @param bucketName Name of the Bucket
-   * @param removeAcls ACLs to be removed
-   * @throws IOException
-   */
-  void removeBucketAcls(String volumeName, String bucketName,
-                        List<OzoneAcl> removeAcls)
-      throws IOException;
-
-
-  /**
-   * Enables or disables Bucket Versioning.
-   * @param volumeName Name of the Volume
-   * @param bucketName Name of the Bucket
-   * @param versioning True to enable Versioning, False to disable.
-   * @throws IOException
-   */
-  void setBucketVersioning(String volumeName, String bucketName,
-                           Boolean versioning)
-      throws IOException;
-
-  /**
-   * Sets the Storage Class of a Bucket.
-   * @param volumeName Name of the Volume
-   * @param bucketName Name of the Bucket
-   * @param storageType StorageType to be set
-   * @throws IOException
-   */
-  void setBucketStorageType(String volumeName, String bucketName,
-                            StorageType storageType)
-      throws IOException;
-
-  /**
-   * Deletes a bucket if it is empty.
-   * @param volumeName Name of the Volume
-   * @param bucketName Name of the Bucket
-   * @throws IOException
-   */
-  void deleteBucket(String volumeName, String bucketName)
-      throws IOException;
-
-  /**
-   * True if the bucket exists and user has read access
-   * to the bucket else throws Exception.
-   * @param volumeName Name of the Volume
-   * @param bucketName Name of the Bucket
-   * @throws IOException
-   */
-  void checkBucketAccess(String volumeName, String bucketName)
-      throws IOException;
-
-  /**
-   * Returns {@link OzoneBucket}.
-   * @param volumeName Name of the Volume
-   * @param bucketName Name of the Bucket
-   * @return {@link OzoneBucket}
-   * @throws IOException
-   */
-  OzoneBucket getBucketDetails(String volumeName, String bucketName)
-      throws IOException;
-
-  /**
-   * Returns the List of Buckets in the Volume that matches the bucketPrefix,
-   * size of the returned list depends on maxListResult. The caller has to make
-   * multiple calls to read all volumes.
-   * @param volumeName Name of the Volume
-   * @param bucketPrefix Bucket prefix to match
-   * @param prevBucket Starting point of the list, this bucket is excluded
-   * @param maxListResult Max number of buckets to return.
-   * @return {@code List<OzoneBucket>}
-   * @throws IOException
-   */
-  List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
-                                String prevBucket, int maxListResult)
-      throws IOException;
-
-  /**
-   * Writes a key in an existing bucket.
-   * @param volumeName Name of the Volume
-   * @param bucketName Name of the Bucket
-   * @param keyName Name of the Key
-   * @param size Size of the data
-   * @return {@link OzoneOutputStream}
-   *
-   */
-  OzoneOutputStream createKey(String volumeName, String bucketName,
-                              String keyName, long size, ReplicationType type,
-                              ReplicationFactor factor)
-      throws IOException;
-
-  /**
-   * Reads a key from an existing bucket.
-   * @param volumeName Name of the Volume
-   * @param bucketName Name of the Bucket
-   * @param keyName Name of the Key
-   * @return {@link OzoneInputStream}
-   * @throws IOException
-   */
-  OzoneInputStream getKey(String volumeName, String bucketName, String keyName)
-      throws IOException;
-
-
-  /**
-   * Deletes an existing key.
-   * @param volumeName Name of the Volume
-   * @param bucketName Name of the Bucket
-   * @param keyName Name of the Key
-   * @throws IOException
-   */
-  void deleteKey(String volumeName, String bucketName, String keyName)
-      throws IOException;
-
-
-  /**
-   * Returns list of Keys in {Volume/Bucket} that matches the keyPrefix,
-   * size of the returned list depends on maxListResult. The caller has
-   * to make multiple calls to read all keys.
-   * @param volumeName Name of the Volume
-   * @param bucketName Name of the Bucket
-   * @param keyPrefix Bucket prefix to match
-   * @param prevKey Starting point of the list, this key is excluded
-   * @param maxListResult Max number of buckets to return.
-   * @return {@code List<OzoneKey>}
-   * @throws IOException
-   */
-  List<OzoneKey> listKeys(String volumeName, String bucketName,
-                          String keyPrefix, String prevKey, int maxListResult)
-      throws IOException;
-
-
-  /**
-   * Get OzoneKey.
-   * @param volumeName Name of the Volume
-   * @param bucketName Name of the Bucket
-   * @param keyName Key name
-   * @return {@link OzoneKey}
-   * @throws IOException
-   */
-  OzoneKey getKeyDetails(String volumeName, String bucketName,
-                         String keyName)
-      throws IOException;
-
-  /**
-   * Close and release the resources.
-   */
-  void close() throws IOException;
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/package-info.java
deleted file mode 100644
index f4890a1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.protocol;
-
-/**
- * This package contains Ozone client protocol library classes.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/DefaultRestServerSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/DefaultRestServerSelector.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/DefaultRestServerSelector.java
deleted file mode 100644
index 93b3417..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/DefaultRestServerSelector.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest;
-
-import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
-
-import java.util.List;
-import java.util.Random;
-
-/**
- * Default selector randomly picks one of the REST Server from the list.
- */
-public class DefaultRestServerSelector implements RestServerSelector {
-
-  @Override
-  public ServiceInfo getRestServer(List<ServiceInfo> restServices) {
-    return restServices.get(
-        new Random().nextInt(restServices.size()));
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneException.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneException.java
deleted file mode 100644
index 953e399..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneException.java
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest;
-
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-
-/**
- * Class the represents various errors returned by the
- * Ozone Layer.
- */
-@InterfaceAudience.Private
-public class OzoneException extends Exception {
-
-  private static final ObjectReader READER =
-      new ObjectMapper().readerFor(OzoneException.class);
-  private static final ObjectMapper MAPPER;
-
-  static {
-    MAPPER = new ObjectMapper();
-    MAPPER.setVisibility(
-        MAPPER.getSerializationConfig().getDefaultVisibilityChecker()
-            .withCreatorVisibility(JsonAutoDetect.Visibility.NONE)
-            .withFieldVisibility(JsonAutoDetect.Visibility.NONE)
-            .withGetterVisibility(JsonAutoDetect.Visibility.NONE)
-            .withIsGetterVisibility(JsonAutoDetect.Visibility.NONE)
-            .withSetterVisibility(JsonAutoDetect.Visibility.NONE));
-  }
-
-  @JsonProperty("httpCode")
-  private long httpCode;
-  @JsonProperty("shortMessage")
-  private String shortMessage;
-  @JsonProperty("resource")
-  private String resource;
-  @JsonProperty("message")
-  private String message;
-  @JsonProperty("requestID")
-  private String requestId;
-  @JsonProperty("hostName")
-  private String hostID;
-
-  /**
-   * Constructs a new exception with {@code null} as its detail message. The
-   * cause is not initialized, and may subsequently be initialized by a call
-   * to {@link #initCause}.
-   *
-   * This constructor is needed by Json Serializer.
-   */
-  public OzoneException() {
-  }
-
-
-  /**
-   * Constructor that allows a shortMessage and exception.
-   *
-   * @param httpCode Error Code
-   * @param shortMessage Short Message
-   * @param ex Exception
-   */
-  public OzoneException(long httpCode, String shortMessage, Exception ex) {
-    super(ex);
-    this.message = ex.getMessage();
-    this.shortMessage = shortMessage;
-    this.httpCode = httpCode;
-  }
-
-
-  /**
-   * Constructor that allows a shortMessage.
-   *
-   * @param httpCode Error Code
-   * @param shortMessage Short Message
-   */
-  public OzoneException(long httpCode, String shortMessage) {
-    this.shortMessage = shortMessage;
-    this.httpCode = httpCode;
-  }
-
-  /**
-   * Constructor that allows a shortMessage and long message.
-   *
-   * @param httpCode Error Code
-   * @param shortMessage Short Message
-   * @param message long error message
-   */
-  public OzoneException(long httpCode, String shortMessage, String message) {
-    this.shortMessage = shortMessage;
-    this.message = message;
-    this.httpCode = httpCode;
-  }
-
-  /**
-   * Constructor that allows a shortMessage, a long message and an exception.
-   *
-   * @param httpCode Error code
-   * @param shortMessage Short message
-   * @param message Long error message
-   * @param ex Exception
-   */
-  public OzoneException(long httpCode, String shortMessage,
-      String message, Exception ex) {
-    super(ex);
-    this.shortMessage = shortMessage;
-    this.message = message;
-    this.httpCode = httpCode;
-  }
-
-  /**
-   * Returns the Resource that was involved in the stackTraceString.
-   *
-   * @return String
-   */
-  public String getResource() {
-    return resource;
-  }
-
-  /**
-   * Sets Resource.
-   *
-   * @param resourceName - Name of the Resource
-   */
-  public void setResource(String resourceName) {
-    this.resource = resourceName;
-  }
-
-  /**
-   * Gets a detailed message for the error.
-   *
-   * @return String
-   */
-  public String getMessage() {
-    return message;
-  }
-
-  /**
-   * Sets the error message.
-   *
-   * @param longMessage - Long message
-   */
-  public void setMessage(String longMessage) {
-    this.message = longMessage;
-  }
-
-  /**
-   * Returns request Id.
-   *
-   * @return String
-   */
-  public String getRequestId() {
-    return requestId;
-  }
-
-  /**
-   * Sets request ID.
-   *
-   * @param ozoneRequestId Request ID generated by the Server
-   */
-  public void setRequestId(String ozoneRequestId) {
-    this.requestId = ozoneRequestId;
-  }
-
-  /**
-   * Returns short error string.
-   *
-   * @return String
-   */
-  public String getShortMessage() {
-    return shortMessage;
-  }
-
-  /**
-   * Sets short error string.
-   *
-   * @param shortError Short Error Code
-   */
-  public void setShortMessage(String shortError) {
-    this.shortMessage = shortError;
-  }
-
-  /**
-   * Returns hostID.
-   *
-   * @return String
-   */
-  public String getHostID() {
-    return hostID;
-  }
-
-  /**
-   * Sets host ID.
-   *
-   * @param hostName host Name
-   */
-  public void setHostID(String hostName) {
-    this.hostID = hostName;
-  }
-
-  /**
-   * Returns http error code.
-   *
-   * @return long
-   */
-  public long getHttpCode() {
-    return httpCode;
-  }
-
-  /**
-   * Sets http status.
-   *
-   * @param httpStatus http error code.
-   */
-  public void setHttpCode(long httpStatus) {
-    this.httpCode = httpStatus;
-  }
-
-  /**
-   * Returns a Json String.
-   *
-   * @return JSON representation of the Error
-   */
-  public String toJsonString() {
-    try {
-      return MAPPER.writeValueAsString(this);
-    } catch (IOException ex) {
-      // TODO : Log this error on server side.
-    }
-    // TODO : Replace this with a JSON Object -- That represents this error.
-    return "500 Internal Server Error";
-  }
-
-  /**
-   * Parses an Exception record.
-   *
-   * @param jsonString - Exception in Json format.
-   *
-   * @return OzoneException Object
-   *
-   * @throws IOException
-   */
-  public static OzoneException parse(String jsonString) throws IOException {
-    return READER.readValue(jsonString);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneExceptionMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneExceptionMapper.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneExceptionMapper.java
deleted file mode 100644
index 6c479f7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneExceptionMapper.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest;
-
-
-import javax.ws.rs.core.Response;
-import javax.ws.rs.ext.ExceptionMapper;
-
-import org.slf4j.MDC;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *  Class the represents various errors returned by the
- *  Object Layer.
- */
-public class OzoneExceptionMapper implements ExceptionMapper<OzoneException> {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(OzoneExceptionMapper.class);
-
-  @Override
-  public Response toResponse(OzoneException exception) {
-    LOG.debug("Returning exception. ex: {}", exception.toJsonString());
-    MDC.clear();
-    return Response.status((int)exception.getHttpCode())
-      .entity(exception.toJsonString()).build();
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
deleted file mode 100644
index c420ce5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ /dev/null
@@ -1,799 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.BucketArgs;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.client.OzoneKey;
-import org.apache.hadoop.ozone.client.OzoneQuota;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.ReplicationFactor;
-import org.apache.hadoop.ozone.client.ReplicationType;
-import org.apache.hadoop.ozone.client.VolumeArgs;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.client.rest.response.BucketInfo;
-import org.apache.hadoop.ozone.client.rest.response.KeyInfo;
-import org.apache.hadoop.ozone.client.rest.response.VolumeInfo;
-import org.apache.hadoop.ozone.client.rpc.RpcClient;
-import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
-import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ServicePort;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Time;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpHeaders;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.HttpDelete;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.entity.InputStreamEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.PipedInputStream;
-import java.io.PipedOutputStream;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.text.ParseException;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.FutureTask;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import static java.net.HttpURLConnection.HTTP_CREATED;
-import static java.net.HttpURLConnection.HTTP_OK;
-
-/**
- * Ozone Client REST protocol implementation. It uses REST protocol to
- * connect to Ozone Handler that executes client calls.
- */
-public class RestClient implements ClientProtocol {
-
-  private static final String PATH_SEPARATOR = "/";
-  private static final Logger LOG = LoggerFactory.getLogger(RpcClient.class);
-
-  private final Configuration conf;
-  private final URI ozoneRestUri;
-  private final CloseableHttpClient httpClient;
-  private final UserGroupInformation ugi;
-  private final OzoneAcl.OzoneACLRights userRights;
-
-   /**
-    * Creates RestClient instance with the given configuration.
-    * @param conf Configuration
-    * @throws IOException
-    */
-  public RestClient(Configuration conf)
-      throws IOException {
-    try {
-      Preconditions.checkNotNull(conf);
-      this.conf = conf;
-
-      long socketTimeout = conf.getTimeDuration(
-          OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT,
-          OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT,
-          TimeUnit.MILLISECONDS);
-      long connectionTimeout = conf.getTimeDuration(
-          OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT,
-          OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT_DEFAULT,
-          TimeUnit.MILLISECONDS);
-      int maxConnection = conf.getInt(
-          OzoneConfigKeys.OZONE_REST_CLIENT_HTTP_CONNECTION_MAX,
-          OzoneConfigKeys.OZONE_REST_CLIENT_HTTP_CONNECTION_DEFAULT);
-
-      int maxConnectionPerRoute = conf.getInt(
-          OzoneConfigKeys.OZONE_REST_CLIENT_HTTP_CONNECTION_PER_ROUTE_MAX,
-          OzoneConfigKeys
-              .OZONE_REST_CLIENT_HTTP_CONNECTION_PER_ROUTE_MAX_DEFAULT
-      );
-
-      /*
-      To make RestClient Thread safe, creating the HttpClient with
-      ThreadSafeClientConnManager.
-      */
-      PoolingHttpClientConnectionManager connManager =
-          new PoolingHttpClientConnectionManager();
-      connManager.setMaxTotal(maxConnection);
-      connManager.setDefaultMaxPerRoute(maxConnectionPerRoute);
-
-      this.httpClient = HttpClients.custom()
-          .setConnectionManager(connManager)
-          .setDefaultRequestConfig(
-              RequestConfig.custom()
-              .setSocketTimeout(Math.toIntExact(socketTimeout))
-                  .setConnectTimeout(Math.toIntExact(connectionTimeout))
-                  .build())
-          .build();
-      this.ugi = UserGroupInformation.getCurrentUser();
-      this.userRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_USER_RIGHTS,
-          KSMConfigKeys.OZONE_KSM_USER_RIGHTS_DEFAULT);
-
-      // TODO: Add new configuration parameter to configure RestServerSelector.
-      RestServerSelector defaultSelector = new DefaultRestServerSelector();
-      InetSocketAddress restServer = getOzoneRestServerAddress(defaultSelector);
-      URIBuilder uriBuilder = new URIBuilder()
-          .setScheme("http")
-          .setHost(restServer.getHostName())
-          .setPort(restServer.getPort());
-      this.ozoneRestUri = uriBuilder.build();
-
-    } catch (URISyntaxException e) {
-      throw new IOException(e);
-    }
-  }
-
-  private InetSocketAddress getOzoneRestServerAddress(
-      RestServerSelector selector) throws IOException {
-    String httpAddress = conf.get(KSMConfigKeys.OZONE_KSM_HTTP_ADDRESS_KEY);
-
-    if (httpAddress == null) {
-      throw new IllegalArgumentException(
-          KSMConfigKeys.OZONE_KSM_HTTP_ADDRESS_KEY + " must be defined. See" +
-              " https://wiki.apache.org/hadoop/Ozone#Configuration for" +
-              " details on configuring Ozone.");
-    }
-
-    HttpGet httpGet = new HttpGet("http://" + httpAddress + "/serviceList");
-    HttpEntity entity = executeHttpRequest(httpGet);
-    try {
-      String serviceListJson = EntityUtils.toString(entity);
-
-      ObjectMapper objectMapper = new ObjectMapper();
-      TypeReference<List<ServiceInfo>> serviceInfoReference =
-          new TypeReference<List<ServiceInfo>>() {
-          };
-      List<ServiceInfo> services = objectMapper.readValue(
-          serviceListJson, serviceInfoReference);
-
-      List<ServiceInfo> dataNodeInfos = services.stream().filter(
-          a -> a.getNodeType().equals(OzoneProtos.NodeType.DATANODE))
-          .collect(Collectors.toList());
-
-      ServiceInfo restServer = selector.getRestServer(dataNodeInfos);
-
-      return NetUtils.createSocketAddr(restServer.getHostname() + ":" +
-          restServer.getPort(ServicePort.Type.HTTP));
-    } finally {
-      EntityUtils.consume(entity);
-    }
-  }
-
-  @Override
-  public void createVolume(String volumeName) throws IOException {
-    createVolume(volumeName, VolumeArgs.newBuilder().build());
-  }
-
-  @Override
-  public void createVolume(String volumeName, VolumeArgs volArgs)
-      throws IOException {
-    try {
-      Preconditions.checkNotNull(volumeName);
-      URIBuilder builder = new URIBuilder(ozoneRestUri);
-      String owner = volArgs.getOwner() == null ?
-          ugi.getUserName() : volArgs.getOwner();
-      //TODO: support for ACLs has to be done in OzoneHandler (rest server)
-      /**
-      List<OzoneAcl> listOfAcls = new ArrayList<>();
-      //User ACL
-      listOfAcls.add(new OzoneAcl(OzoneAcl.OzoneACLType.USER,
-          owner, userRights));
-      //ACLs from VolumeArgs
-      if(volArgs.getAcls() != null) {
-        listOfAcls.addAll(volArgs.getAcls());
-      }
-       */
-      builder.setPath(PATH_SEPARATOR + volumeName);
-
-      String quota = volArgs.getQuota();
-      if(quota != null) {
-        builder.setParameter(Header.OZONE_QUOTA_QUERY_TAG, quota);
-      }
-
-      HttpPost httpPost = new HttpPost(builder.build());
-      addOzoneHeaders(httpPost);
-      //use admin from VolumeArgs, if it's present
-      if(volArgs.getAdmin() != null) {
-        httpPost.removeHeaders(HttpHeaders.AUTHORIZATION);
-        httpPost.addHeader(HttpHeaders.AUTHORIZATION,
-            Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
-                volArgs.getAdmin());
-      }
-      httpPost.addHeader(Header.OZONE_USER, owner);
-      LOG.info("Creating Volume: {}, with {} as owner and quota set to {}.",
-          volumeName, owner, quota == null ? "default" : quota);
-      EntityUtils.consume(executeHttpRequest(httpPost));
-    } catch (URISyntaxException e) {
-      throw new IOException(e);
-    }
-  }
-
-
-  @Override
-  public void setVolumeOwner(String volumeName, String owner)
-      throws IOException {
-    try {
-      Preconditions.checkNotNull(volumeName);
-      Preconditions.checkNotNull(owner);
-      URIBuilder builder = new URIBuilder(ozoneRestUri);
-      builder.setPath(PATH_SEPARATOR + volumeName);
-      HttpPut httpPut = new HttpPut(builder.build());
-      addOzoneHeaders(httpPut);
-      httpPut.addHeader(Header.OZONE_USER, owner);
-      EntityUtils.consume(executeHttpRequest(httpPut));
-    } catch (URISyntaxException e) {
-      throw new IOException(e);
-    }
-  }
-
-  @Override
-  public void setVolumeQuota(String volumeName, OzoneQuota quota)
-      throws IOException {
-    try {
-      Preconditions.checkNotNull(volumeName);
-      Preconditions.checkNotNull(quota);
-      String quotaString = quota.toString();
-      URIBuilder builder = new URIBuilder(ozoneRestUri);
-      builder.setPath(PATH_SEPARATOR + volumeName);
-      builder.setParameter(Header.OZONE_QUOTA_QUERY_TAG, quotaString);
-      HttpPut httpPut = new HttpPut(builder.build());
-      addOzoneHeaders(httpPut);
-      EntityUtils.consume(executeHttpRequest(httpPut));
-    } catch (URISyntaxException e) {
-      throw new IOException(e);
-    }
-  }
-
-  @Override
-  public OzoneVolume getVolumeDetails(String volumeName)
-      throws IOException {
-    try {
-      Preconditions.checkNotNull(volumeName);
-      URIBuilder builder = new URIBuilder(ozoneRestUri);
-      builder.setPath(PATH_SEPARATOR + volumeName);
-      builder.setParameter(Header.OZONE_INFO_QUERY_TAG,
-          Header.OZONE_INFO_QUERY_VOLUME);
-      HttpGet httpGet = new HttpGet(builder.build());
-      addOzoneHeaders(httpGet);
-      HttpEntity response = executeHttpRequest(httpGet);
-      VolumeInfo volInfo =
-          VolumeInfo.parse(EntityUtils.toString(response));
-      //TODO: OzoneHandler in datanode has to be modified to send ACLs
-      OzoneVolume volume = new OzoneVolume(conf,
-          this,
-          volInfo.getVolumeName(),
-          volInfo.getCreatedBy(),
-          volInfo.getOwner().getName(),
-          volInfo.getQuota().sizeInBytes(),
-          OzoneClientUtils.formatDateTime(volInfo.getCreatedOn()),
-          null);
-      EntityUtils.consume(response);
-      return volume;
-    } catch (URISyntaxException | ParseException e) {
-      throw new IOException(e);
-    }
-  }
-
-  @Override
-  public boolean checkVolumeAccess(String volumeName, OzoneAcl acl)
-      throws IOException {
-    throw new UnsupportedOperationException("Not yet implemented.");
-  }
-
-  @Override
-  public void deleteVolume(String volumeName) throws IOException {
-    try {
-      Preconditions.checkNotNull(volumeName);
-      URIBuilder builder = new URIBuilder(ozoneRestUri);
-      builder.setPath(PATH_SEPARATOR + volumeName);
-      HttpDelete httpDelete = new HttpDelete(builder.build());
-      addOzoneHeaders(httpDelete);
-      EntityUtils.consume(executeHttpRequest(httpDelete));
-    } catch (URISyntaxException e) {
-      throw new IOException(e);
-    }
-  }
-
-  @Override
-  public List<OzoneVolume> listVolumes(String volumePrefix, String prevKey,
-                                       int maxListResult)
-      throws IOException {
-    throw new UnsupportedOperationException("Not yet implemented.");
-  }
-
-  @Override
-  public List<OzoneVolume> listVolumes(String user, String volumePrefix,
-                                       String prevKey, int maxListResult)
-      throws IOException {
-    throw new UnsupportedOperationException("Not yet implemented.");
-  }
-
-  @Override
-  public void createBucket(String volumeName, String bucketName)
-      throws IOException {
-    createBucket(volumeName, bucketName, BucketArgs.newBuilder().build());
-  }
-
-  @Override
-  public void createBucket(
-      String volumeName, String bucketName, BucketArgs bucketArgs)
-      throws IOException {
-    try {
-      Preconditions.checkNotNull(volumeName);
-      Preconditions.checkNotNull(bucketName);
-      Preconditions.checkNotNull(bucketArgs);
-      URIBuilder builder = new URIBuilder(ozoneRestUri);
-      OzoneConsts.Versioning versioning = OzoneConsts.Versioning.DISABLED;
-      if(bucketArgs.getVersioning() != null &&
-          bucketArgs.getVersioning()) {
-        versioning = OzoneConsts.Versioning.ENABLED;
-      }
-      StorageType storageType = bucketArgs.getStorageType() == null ?
-          StorageType.DEFAULT : bucketArgs.getStorageType();
-
-      builder.setPath(PATH_SEPARATOR + volumeName +
-          PATH_SEPARATOR + bucketName);
-      HttpPost httpPost = new HttpPost(builder.build());
-      addOzoneHeaders(httpPost);
-
-      //ACLs from BucketArgs
-      if(bucketArgs.getAcls() != null) {
-        for (OzoneAcl acl : bucketArgs.getAcls()) {
-          httpPost.addHeader(
-              Header.OZONE_ACLS, Header.OZONE_ACL_ADD + " " + acl.toString());
-        }
-      }
-      httpPost.addHeader(Header.OZONE_STORAGE_TYPE, storageType.toString());
-      httpPost.addHeader(Header.OZONE_BUCKET_VERSIONING,
-          versioning.toString());
-      LOG.info("Creating Bucket: {}/{}, with Versioning {} and Storage Type" +
-              " set to {}", volumeName, bucketName, versioning,
-          storageType);
-
-      EntityUtils.consume(executeHttpRequest(httpPost));
-    } catch (URISyntaxException e) {
-      throw new IOException(e);
-    }
-  }
-
-  @Override
-  public void addBucketAcls(
-      String volumeName, String bucketName, List<OzoneAcl> addAcls)
-      throws IOException {
-    try {
-      Preconditions.checkNotNull(volumeName);
-      Preconditions.checkNotNull(bucketName);
-      Preconditions.checkNotNull(addAcls);
-      URIBuilder builder = new URIBuilder(ozoneRestUri);
-
-      builder.setPath(PATH_SEPARATOR + volumeName +
-          PATH_SEPARATOR + bucketName);
-      HttpPut httpPut = new HttpPut(builder.build());
-      addOzoneHeaders(httpPut);
-
-      for (OzoneAcl acl : addAcls) {
-        httpPut.addHeader(
-            Header.OZONE_ACLS, Header.OZONE_ACL_ADD + " " + acl.toString());
-      }
-      EntityUtils.consume(executeHttpRequest(httpPut));
-    } catch (URISyntaxException e) {
-      throw new IOException(e);
-    }
-  }
-
-  @Override
-  public void removeBucketAcls(
-      String volumeName, String bucketName, List<OzoneAcl> removeAcls)
-      throws IOException {
-    try {
-      Preconditions.checkNotNull(volumeName);
-      Preconditions.checkNotNull(bucketName);
-      Preconditions.checkNotNull(removeAcls);
-      URIBuilder builder = new URIBuilder(ozoneRestUri);
-
-      builder.setPath(PATH_SEPARATOR + volumeName +
-          PATH_SEPARATOR + bucketName);
-      HttpPut httpPut = new HttpPut(builder.build());
-      addOzoneHeaders(httpPut);
-
-      for (OzoneAcl acl : removeAcls) {
-        httpPut.addHeader(
-            Header.OZONE_ACLS, Header.OZONE_ACL_REMOVE + " " + acl.toString());
-      }
-      EntityUtils.consume(executeHttpRequest(httpPut));
-    } catch (URISyntaxException e) {
-      throw new IOException(e);
-    }
-  }
-
-  @Override
-  public void setBucketVersioning(
-      String volumeName, String bucketName, Boolean versioning)
-      throws IOException {
-    try {
-      Preconditions.checkNotNull(volumeName);
-      Preconditions.checkNotNull(bucketName);
-      Preconditions.checkNotNull(versioning);
-      URIBuilder builder = new URIBuilder(ozoneRestUri);
-
-      builder.setPath(PATH_SEPARATOR + volumeName +
-          PATH_SEPARATOR + bucketName);
-      HttpPut httpPut = new HttpPut(builder.build());
-      addOzoneHeaders(httpPut);
-
-      httpPut.addHeader(Header.OZONE_BUCKET_VERSIONING,
-          getBucketVersioning(versioning).toString());
-      EntityUtils.consume(executeHttpRequest(httpPut));
-    } catch (URISyntaxException e) {
-      throw new IOException(e);
-    }
-  }
-
-  @Override
-  public void setBucketStorageType(
-      String volumeName, String bucketName, StorageType storageType)
-      throws IOException {
-    try {
-      Preconditions.checkNotNull(volumeName);
-      Preconditions.checkNotNull(bucketName);
-      Preconditions.checkNotNull(storageType);
-      URIBuilder builder = new URIBuilder(ozoneRestUri);
-
-      builder.setPath(PATH_SEPARATOR + volumeName +
-          PATH_SEPARATOR + bucketName);
-      HttpPut httpPut = new HttpPut(builder.build());
-      addOzoneHeaders(httpPut);
-
-      httpPut.addHeader(Header.OZONE_STORAGE_TYPE, storageType.toString());
-      EntityUtils.consume(executeHttpRequest(httpPut));
-    } catch (URISyntaxException e) {
-      throw new IOException(e);
-    }
-  }
-
-  @Override
-  public void deleteBucket(String volumeName, String bucketName)
-      throws IOException {
-    try {
-      Preconditions.checkNotNull(volumeName);
-      Preconditions.checkNotNull(bucketName);
-      URIBuilder builder = new URIBuilder(ozoneRestUri);
-      builder.setPath(PATH_SEPARATOR + volumeName +
-          PATH_SEPARATOR + bucketName);
-      HttpDelete httpDelete = new HttpDelete(builder.build());
-      addOzoneHeaders(httpDelete);
-      EntityUtils.consume(executeHttpRequest(httpDelete));
-    } catch (URISyntaxException e) {
-      throw new IOException(e);
-    }
-  }
-
-  @Override
-  public void checkBucketAccess(String volumeName, String bucketName)
-      throws IOException {
-    throw new UnsupportedOperationException("Not yet implemented.");
-  }
-
-  @Override
-  public OzoneBucket getBucketDetails(String volumeName, String bucketName)
-      throws IOException {
-    try {
-      Preconditions.checkNotNull(volumeName);
-      Preconditions.checkNotNull(bucketName);
-      URIBuilder builder = new URIBuilder(ozoneRestUri);
-      builder.setPath(PATH_SEPARATOR + volumeName +
-          PATH_SEPARATOR + bucketName);
-      builder.setParameter(Header.OZONE_INFO_QUERY_TAG,
-          Header.OZONE_INFO_QUERY_BUCKET);
-      HttpGet httpGet = new HttpGet(builder.build());
-      addOzoneHeaders(httpGet);
-      HttpEntity response = executeHttpRequest(httpGet);
-      BucketInfo bucketInfo =
-          BucketInfo.parse(EntityUtils.toString(response));
-      OzoneBucket bucket = new OzoneBucket(conf,
-          this,
-          bucketInfo.getVolumeName(),
-          bucketInfo.getBucketName(),
-          bucketInfo.getAcls(),
-          bucketInfo.getStorageType(),
-          getBucketVersioningFlag(bucketInfo.getVersioning()),
-          OzoneClientUtils.formatDateTime(bucketInfo.getCreatedOn()));
-      EntityUtils.consume(response);
-      return bucket;
-    } catch (URISyntaxException | ParseException e) {
-      throw new IOException(e);
-    }
-  }
-
-  @Override
-  public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
-                                       String prevBucket, int maxListResult)
-      throws IOException {
-    throw new UnsupportedOperationException("Not yet implemented.");
-  }
-
-  /**
-   * Writes a key in an existing bucket.
-   *
-   * @param volumeName Name of the Volume
-   * @param bucketName Name of the Bucket
-   * @param keyName Name of the Key
-   * @param size Size of the data
-   * @param type
-   * @param factor @return {@link OzoneOutputStream}
-   */
-  @Override
-  public OzoneOutputStream createKey(
-      String volumeName, String bucketName, String keyName, long size,
-      ReplicationType type, ReplicationFactor factor)
-      throws IOException {
-    // TODO: Once ReplicationType and ReplicationFactor are supported in
-    // OzoneHandler (in Datanode), set them in header.
-    try {
-      Preconditions.checkNotNull(volumeName);
-      Preconditions.checkNotNull(bucketName);
-      Preconditions.checkNotNull(keyName);
-      URIBuilder builder = new URIBuilder(ozoneRestUri);
-      builder.setPath(PATH_SEPARATOR + volumeName +
-          PATH_SEPARATOR + bucketName +
-          PATH_SEPARATOR + keyName);
-      HttpPut putRequest = new HttpPut(builder.build());
-      addOzoneHeaders(putRequest);
-      PipedInputStream in = new PipedInputStream();
-      OutputStream out = new PipedOutputStream(in);
-      putRequest.setEntity(new InputStreamEntity(in, size));
-      FutureTask<HttpEntity> futureTask =
-          new FutureTask<>(() -> executeHttpRequest(putRequest));
-      new Thread(futureTask).start();
-      OzoneOutputStream outputStream = new OzoneOutputStream(
-          new OutputStream() {
-            @Override
-            public void write(int b) throws IOException {
-              out.write(b);
-            }
-
-            @Override
-            public void close() throws IOException {
-              try {
-                out.close();
-                EntityUtils.consume(futureTask.get());
-              } catch (ExecutionException | InterruptedException e) {
-                throw new IOException(e);
-              }
-            }
-          });
-
-      return outputStream;
-    } catch (URISyntaxException e) {
-      throw new IOException(e);
-    }
-  }
-
-  @Override
-  public OzoneInputStream getKey(
-      String volumeName, String bucketName, String keyName)
-      throws IOException {
-    try {
-      Preconditions.checkNotNull(volumeName);
-      Preconditions.checkNotNull(bucketName);
-      Preconditions.checkNotNull(keyName);
-      URIBuilder builder = new URIBuilder(ozoneRestUri);
-      builder.setPath(PATH_SEPARATOR + volumeName +
-          PATH_SEPARATOR + bucketName +
-          PATH_SEPARATOR + keyName);
-      HttpGet getRequest = new HttpGet(builder.build());
-      addOzoneHeaders(getRequest);
-      HttpEntity entity = executeHttpRequest(getRequest);
-      PipedInputStream in = new PipedInputStream();
-      OutputStream out = new PipedOutputStream(in);
-      FutureTask<Void> futureTask =
-          new FutureTask<>(() -> {
-            entity.writeTo(out);
-            out.close();
-            return null;
-          });
-      new Thread(futureTask).start();
-      OzoneInputStream inputStream = new OzoneInputStream(
-          new InputStream() {
-
-            @Override
-            public int read() throws IOException {
-              return in.read();
-            }
-
-            @Override
-            public void close() throws IOException {
-              in.close();
-              EntityUtils.consume(entity);
-            }
-          });
-
-      return inputStream;
-    } catch (URISyntaxException e) {
-      throw new IOException(e);
-    }
-  }
-
-  @Override
-  public void deleteKey(String volumeName, String bucketName, String keyName)
-      throws IOException {
-    try {
-      Preconditions.checkNotNull(volumeName);
-      Preconditions.checkNotNull(bucketName);
-      Preconditions.checkNotNull(keyName);
-      URIBuilder builder = new URIBuilder(ozoneRestUri);
-      builder.setPath(PATH_SEPARATOR + volumeName +
-          PATH_SEPARATOR + bucketName + PATH_SEPARATOR + keyName);
-      HttpDelete httpDelete = new HttpDelete(builder.build());
-      addOzoneHeaders(httpDelete);
-      EntityUtils.consume(executeHttpRequest(httpDelete));
-    } catch (URISyntaxException e) {
-      throw new IOException(e);
-    }
-  }
-
-  @Override
-  public List<OzoneKey> listKeys(String volumeName, String bucketName,
-                                 String keyPrefix, String prevKey,
-                                 int maxListResult)
-      throws IOException {
-    throw new UnsupportedOperationException("Not yet implemented.");
-  }
-
-  @Override
-  public OzoneKey getKeyDetails(
-      String volumeName, String bucketName, String keyName)
-      throws IOException {
-    try {
-      Preconditions.checkNotNull(volumeName);
-      Preconditions.checkNotNull(bucketName);
-      Preconditions.checkNotNull(keyName);
-      URIBuilder builder = new URIBuilder(ozoneRestUri);
-      builder.setPath(PATH_SEPARATOR + volumeName +
-          PATH_SEPARATOR + bucketName + PATH_SEPARATOR + keyName);
-      builder.setParameter(Header.OZONE_INFO_QUERY_TAG,
-          Header.OZONE_INFO_QUERY_KEY);
-      HttpGet httpGet = new HttpGet(builder.build());
-      addOzoneHeaders(httpGet);
-      HttpEntity response = executeHttpRequest(httpGet);
-      KeyInfo keyInfo =
-          KeyInfo.parse(EntityUtils.toString(response));
-      OzoneKey key = new OzoneKey(volumeName,
-          bucketName,
-          keyInfo.getKeyName(),
-          keyInfo.getSize(),
-          OzoneClientUtils.formatDateTime(keyInfo.getCreatedOn()),
-          OzoneClientUtils.formatDateTime(keyInfo.getModifiedOn()));
-      EntityUtils.consume(response);
-      return key;
-    } catch (URISyntaxException | ParseException e) {
-      throw new IOException(e);
-    }
-  }
-
-  /**
-   * Adds Ozone headers to http request.
-   *
-   * @param httpRequest Http Request
-   */
-  private void addOzoneHeaders(HttpUriRequest httpRequest) {
-    httpRequest.addHeader(HttpHeaders.AUTHORIZATION,
-        Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
-            ugi.getUserName());
-    httpRequest.addHeader(HttpHeaders.DATE,
-        OzoneClientUtils.formatDateTime(Time.monotonicNow()));
-    httpRequest.addHeader(Header.OZONE_VERSION_HEADER,
-        Header.OZONE_V1_VERSION_HEADER);
-  }
-
-  /**
-   * Sends the http request to server and returns the response HttpEntity.
-   * It's responsibility of the caller to consume and close response HttpEntity
-   * by calling {@code EntityUtils.consume}
-   *
-   * @param httpUriRequest http request
-   * @throws IOException
-   */
-  private HttpEntity executeHttpRequest(HttpUriRequest httpUriRequest)
-      throws IOException {
-    HttpResponse response = httpClient.execute(httpUriRequest);
-    int errorCode = response.getStatusLine().getStatusCode();
-    HttpEntity entity = response.getEntity();
-    if ((errorCode == HTTP_OK) || (errorCode == HTTP_CREATED)) {
-      return entity;
-    }
-    if (entity != null) {
-      throw new IOException(
-          OzoneException.parse(EntityUtils.toString(entity)));
-    } else {
-      throw new IOException("Unexpected null in http payload," +
-          " while processing request");
-    }
-  }
-
-  /**
-   * Converts OzoneConts.Versioning to boolean.
-   *
-   * @param version
-   * @return corresponding boolean value
-   */
-  private Boolean getBucketVersioningFlag(
-      OzoneConsts.Versioning version) {
-    if(version != null) {
-      switch(version) {
-      case ENABLED:
-        return true;
-      case NOT_DEFINED:
-      case DISABLED:
-      default:
-        return false;
-      }
-    }
-    return false;
-  }
-
-  /**
-   * Converts Bucket versioning flag into OzoneConts.Versioning.
-   *
-   * @param flag versioning flag
-   * @return corresponding OzoneConts.Versionin
-   */
-  private OzoneConsts.Versioning getBucketVersioning(Boolean flag) {
-    if(flag != null) {
-      if(flag) {
-        return OzoneConsts.Versioning.ENABLED;
-      } else {
-        return OzoneConsts.Versioning.DISABLED;
-      }
-    }
-    return OzoneConsts.Versioning.NOT_DEFINED;
-  }
-
-  @Override
-  public void close() throws IOException {
-    httpClient.close();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java
deleted file mode 100644
index 54e219b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest;
-
-import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
-
-import java.util.List;
-
-/**
- * The implementor of this interface should select the REST server which will
- * be used by the client to connect to Ozone Cluster, given list of
- * REST Servers/DataNodes (DataNodes are the ones which hosts REST Service).
- */
-public interface RestServerSelector {
-
-  /**
-   * Returns the REST Service which will be used by the client for connection.
-   *
-   * @param restServices list of available REST servers
-   * @return ServiceInfo
-   */
-  ServiceInfo getRestServer(List<ServiceInfo> restServices);
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/exceptions/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/exceptions/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/exceptions/package-info.java
deleted file mode 100644
index 233e788..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/exceptions/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.client.rest.exceptions;
-
-/**
- * This package contains ozone rest client libraries.
- */


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[45/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 63f3376..0e349d3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -18,24 +18,6 @@
 
 package org.apache.hadoop.fs;
 
-import com.google.common.base.Preconditions;
-import org.apache.commons.collections.map.CaseInsensitiveMap;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.Shell.ShellCommandExecutor;
-import org.apache.hadoop.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
@@ -71,8 +53,26 @@ import java.util.zip.CheckedOutputStream;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
-import java.util.zip.ZipOutputStream;
 import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.collections.map.CaseInsensitiveMap;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.Shell.ShellCommandExecutor;
+import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * A collection of file-processing util methods.
@@ -91,9 +91,9 @@ public class FileUtil {
   public static final int SYMLINK_NO_PRIVILEGE = 2;
 
   /**
-   * Buffer size used while zipping and unzipping zip-ed archives.
+   * Buffer size for copy the content of compressed file to new file.
    */
-  private static final int BUFFER_SIZE = 8192;
+  private static final int BUFFER_SIZE = 8_192;
 
   /**
    * convert an array of FileStatus to an array of Path
@@ -613,7 +613,6 @@ public class FileUtil {
   }
 
   /**
-<<<<<<< HEAD
    * creates zip archieve of the source dir and writes a zip file.
    *
    * @param sourceDir - The directory to zip.
@@ -673,7 +672,7 @@ public class FileUtil {
   }
 
   /**
-   * Given a File input it will unzip the file in a the unzip directory
+   * Given a stream input it will unzip the it in the unzip directory.
    * passed as the second parameter
    * @param inputStream The zip file as input
    * @param toDir The unzip directory where to unzip the zip file.
@@ -731,12 +730,12 @@ public class FileUtil {
             if (!file.getParentFile().mkdirs()) {
               if (!file.getParentFile().isDirectory()) {
                 throw new IOException("Mkdirs failed to create " +
-                    file.getParentFile().toString());
+                                      file.getParentFile().toString());
               }
             }
             OutputStream out = new FileOutputStream(file);
             try {
-              byte[] buffer = new byte[BUFFER_SIZE];
+              byte[] buffer = new byte[8192];
               int i;
               while ((i = in.read(buffer)) != -1) {
                 out.write(buffer, 0, i);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
index 7e76099..3c31a8c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
@@ -814,7 +814,7 @@ public class RPC {
 
     static String serverNameFromClass(Class<?> clazz) {
       //The basic idea here is to handle names like
-      //org.apache.hadoop.ozone.protocol.proto.
+      //org.apache.hadoop.hdsl.protocol.proto.
       //
       // StorageDatanodeProtocolProtos$StorageContainerDatanodeProtocolService$2
       //where the getSimpleName is also empty

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index 7b885c9..4f68ace 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -68,6 +68,44 @@
       <artifactId>hadoop-client-integration-tests</artifactId>
       <scope>provided</scope>
     </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-ozone-ozone-manager</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdsl-server-scm</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdsl-tools</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-cblock-server</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdsl-container-service</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-ozone-objectstore-service</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdsl-tools</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-ozone-tools</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-cblock-tools</artifactId>
+    </dependency>
+
   </dependencies>
 
   <build>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-dist/src/main/compose/cblock/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/cblock/docker-compose.yaml b/hadoop-dist/src/main/compose/cblock/docker-compose.yaml
index fefb37f..b88514e 100644
--- a/hadoop-dist/src/main/compose/cblock/docker-compose.yaml
+++ b/hadoop-dist/src/main/compose/cblock/docker-compose.yaml
@@ -17,7 +17,7 @@
 version: "3"
 services:
    namenode:
-      image: elek/hadoop-runner:latest
+      image: elek/hadoop-runner:o3-refactor
       hostname: namenode
       volumes:
          - ../..//hadoop-${VERSION}:/opt/hadoop
@@ -29,36 +29,38 @@ services:
          - ./docker-config
       command: ["/opt/hadoop/bin/hdfs","namenode"]
    datanode:
-      image: elek/hadoop-runner:latest
+      image: elek/hadoop-runner:o3-refactor
       volumes:
         - ../..//hadoop-${VERSION}:/opt/hadoop
       ports:
         - 9864
-      command: ["/opt/hadoop/bin/hdfs","datanode"]
+      command: ["/opt/hadoop/bin/oz","datanode"]
+      env_file:
+         - ./docker-config
    jscsi:
-      image: elek/hadoop-runner:latest
+      image: elek/hadoop-runner:o3-refactor
       ports:
         - 3260:3260
       volumes:
          - ../..//hadoop-${VERSION}:/opt/hadoop
       env_file:
           - ./docker-config
-      command: ["/opt/hadoop/bin/hdfs","jscsi"]
+      command: ["/opt/hadoop/bin/oz","jscsi"]
    cblock:
-      image: elek/hadoop-runner:latest
+      image: elek/hadoop-runner:o3-refactor
       volumes:
          - ../..//hadoop-${VERSION}:/opt/hadoop
       env_file:
           - ./docker-config
-      command: ["/opt/hadoop/bin/hdfs","cblockserver"]
+      command: ["/opt/hadoop/bin/oz","cblockserver"]
    scm:
-      image: elek/hadoop-runner:latest
+      image: elek/hadoop-runner:o3-refactor
       volumes:
          - ../..//hadoop-${VERSION}:/opt/hadoop
       ports:
          - 9876:9876
       env_file:
           - ./docker-config
-      command: ["/opt/hadoop/bin/hdfs","scm"]
+      command: ["/opt/hadoop/bin/oz","scm"]
       environment:
           ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-dist/src/main/compose/cblock/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/cblock/docker-config b/hadoop-dist/src/main/compose/cblock/docker-config
index 6851bc6..95917fd 100644
--- a/hadoop-dist/src/main/compose/cblock/docker-config
+++ b/hadoop-dist/src/main/compose/cblock/docker-config
@@ -27,7 +27,7 @@ OZONE-SITE.XML_ozone.scm.client.address=scm
 OZONE-SITE.XML_dfs.cblock.jscsi.cblock.server.address=cblock
 OZONE-SITE.XML_dfs.cblock.scm.ipaddress=scm
 OZONE-SITE.XML_dfs.cblock.service.leveldb.path=/tmp
-
+HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HdslServerPlugin,org.apache.hadoop.ozone.web.ObjectStoreRestPlugin
 HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
 HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
index 02713c7..f2b263c 100644
--- a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
+++ b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
@@ -17,7 +17,7 @@
 version: "3"
 services:
    namenode:
-      image: elek/hadoop-runner:latest
+      image: elek/hadoop-runner:o3-refactor
       hostname: namenode
       volumes:
          - ../..//hadoop-${VERSION}:/opt/hadoop
@@ -29,14 +29,16 @@ services:
          - ./docker-config
       command: ["/opt/hadoop/bin/hdfs","namenode"]
    datanode:
-      image: elek/hadoop-runner:latest
+      image: elek/hadoop-runner:o3-refactor
       volumes:
         - ../..//hadoop-${VERSION}:/opt/hadoop
       ports:
         - 9864
-      command: ["/opt/hadoop/bin/hdfs","datanode"]
+      command: ["/opt/hadoop/bin/oz","datanode"]
+      env_file:
+        - ./docker-config
    ksm:
-      image: elek/hadoop-runner:latest
+      image: elek/hadoop-runner:o3-refactor
       volumes:
          - ../..//hadoop-${VERSION}:/opt/hadoop
       ports:
@@ -45,9 +47,9 @@ services:
          ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION
       env_file:
           - ./docker-config
-      command: ["/opt/hadoop/bin/hdfs","ksm"]
+      command: ["/opt/hadoop/bin/oz","ksm"]
    scm:
-      image: elek/hadoop-runner:latest
+      image: elek/hadoop-runner:o3-refactor
       volumes:
          - ../..//hadoop-${VERSION}:/opt/hadoop
       ports:
@@ -56,4 +58,4 @@ services:
           - ./docker-config
       environment:
           ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
-      command: ["/opt/hadoop/bin/hdfs","scm"]
+      command: ["/opt/hadoop/bin/oz","scm"]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-dist/src/main/compose/ozone/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone/docker-config b/hadoop-dist/src/main/compose/ozone/docker-config
index c3f3663..20c1e30 100644
--- a/hadoop-dist/src/main/compose/ozone/docker-config
+++ b/hadoop-dist/src/main/compose/ozone/docker-config
@@ -27,6 +27,7 @@ HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
 HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HdslServerPlugin,org.apache.hadoop.ozone.web.ObjectStoreRestPlugin
 LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
 LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
index 5503cb2..8e2bc94 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
@@ -63,13 +63,6 @@
     <Bug pattern="IS2_INCONSISTENT_SYNC" />
   </Match>
 
-  <Match>
-    <Package name="org.apache.hadoop.ozone.protocol.proto" />
-  </Match>
-  <Match>
-    <Package name="org.apache.hadoop.hdfs.ozone.protocol.proto" />
-  </Match>
-
   <!-- BlockLocations are user-facing, but LocatedBlocks are not. -->
   <Match>
     <Class name="org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus" />
@@ -98,4 +91,5 @@
     <Method name="getSymlinkInBytes" />
     <Bug pattern="EI_EXPOSE_REP" />
   </Match>
+
 </FindBugsFilter>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
index 19d8113..a5ed7a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
@@ -64,6 +64,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <scope>test</scope>
     </dependency>
     <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-all</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
       <groupId>org.mock-server</groupId>
       <artifactId>mockserver-netty</artifactId>
       <scope>test</scope>
@@ -100,37 +105,14 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <scope>test</scope>
       <type>test-jar</type>
     </dependency>
-    <dependency>
-      <groupId>com.fasterxml.jackson.core</groupId>
-      <artifactId>jackson-annotations</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty-all</artifactId>
-    </dependency>
+      <dependency>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-annotations</artifactId>
+      </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
     </dependency>
-
-    <dependency>
-      <artifactId>ratis-server</artifactId>
-      <groupId>org.apache.ratis</groupId>
-      <exclusions>
-        <exclusion>
-          <groupId>org.slf4j</groupId>
-          <artifactId>slf4j-log4j12</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-    <dependency>
-      <artifactId>ratis-netty</artifactId>
-      <groupId>org.apache.ratis</groupId>
-    </dependency>
-    <dependency>
-      <artifactId>ratis-grpc</artifactId>
-      <groupId>org.apache.ratis</groupId>
-    </dependency>
   </dependencies>
 
   <build>
@@ -177,11 +159,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
                   <include>inotify.proto</include>
                   <include>erasurecoding.proto</include>
                   <include>ReconfigurationProtocol.proto</include>
-                  <include>StorageContainerLocationProtocol.proto</include>
-                  <include>DatanodeContainerProtocol.proto</include>
-                  <include>Ozone.proto</include>
-                  <include>KeySpaceManagerProtocol.proto</include>
-                  <include>ScmBlockLocationProtocol.proto</include>
                 </includes>
               </source>
             </configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
deleted file mode 100644
index fde728c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-import static java.lang.Thread.NORM_PRIORITY;
-
-/**
- * This class contains constants for configuration keys used in CBlock.
- */
-public final class CBlockConfigKeys {
-  public static final String DFS_CBLOCK_SERVICERPC_ADDRESS_KEY =
-      "dfs.cblock.servicerpc-address";
-  public static final int DFS_CBLOCK_SERVICERPC_PORT_DEFAULT =
-      9810;
-  public static final String DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT =
-      "0.0.0.0";
-
-  public static final String DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY =
-      "dfs.cblock.jscsi-address";
-
-  //The port on CBlockManager node for jSCSI to ask
-  public static final String DFS_CBLOCK_JSCSI_PORT_KEY =
-      "dfs.cblock.jscsi.port";
-  public static final int DFS_CBLOCK_JSCSI_PORT_DEFAULT =
-      9811;
-
-  public static final String DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY =
-      "dfs.cblock.service.rpc-bind-host";
-  public static final String DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY =
-      "dfs.cblock.jscsi.rpc-bind-host";
-
-  // default block size is 4KB
-  public static final int DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT =
-      4096;
-
-  public static final String DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY =
-      "dfs.cblock.service.handler.count";
-  public static final int DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT = 10;
-
-  public static final String DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY =
-      "dfs.cblock.service.leveldb.path";
-  //TODO : find a better place
-  public static final String DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT =
-      "/tmp/cblock_levelDB.dat";
-
-
-  public static final String DFS_CBLOCK_DISK_CACHE_PATH_KEY =
-      "dfs.cblock.disk.cache.path";
-  public static final String DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT =
-      "/tmp/cblockCacheDB";
-  /**
-   * Setting this flag to true makes the block layer compute a sha256 hash of
-   * the data and log that information along with block ID. This is very
-   * useful for doing trace based simulation of various workloads. Since it is
-   * computing a hash for each block this could be expensive, hence default
-   * is false.
-   */
-  public static final String DFS_CBLOCK_TRACE_IO = "dfs.cblock.trace.io";
-  public static final boolean DFS_CBLOCK_TRACE_IO_DEFAULT = false;
-
-  public static final String DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO =
-      "dfs.cblock.short.circuit.io";
-  public static final boolean DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO_DEFAULT =
-      false;
-
-  /**
-   * Cache size in 1000s of entries. 256 indicates 256 * 1024.
-   */
-  public static final String DFS_CBLOCK_CACHE_QUEUE_SIZE_KB =
-      "dfs.cblock.cache.queue.size.in.kb";
-  public static final int DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT = 256;
-
-  /**
-   *  Minimum Number of threads that cache pool will use for background I/O.
-   */
-  public static final String DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE =
-      "dfs.cblock.cache.core.min.pool.size";
-  public static final int DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT = 16;
-
-  /**
-   *  Maximum Number of threads that cache pool will use for background I/O.
-   */
-
-  public static final String DFS_CBLOCK_CACHE_MAX_POOL_SIZE =
-      "dfs.cblock.cache.max.pool.size";
-  public static final int DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT = 256;
-
-  /**
-   * Number of seconds to keep the Thread alive when it is idle.
-   */
-  public static final String DFS_CBLOCK_CACHE_KEEP_ALIVE =
-      "dfs.cblock.cache.keep.alive";
-  public static final String DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT = "60s";
-
-  /**
-   * Priority of cache flusher thread, affecting the relative performance of
-   * write and read.
-   */
-  public static final String DFS_CBLOCK_CACHE_THREAD_PRIORITY =
-      "dfs.cblock.cache.thread.priority";
-  public static final int DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT =
-      NORM_PRIORITY;
-
-  /**
-   * Block Buffer size in terms of blockID entries, 512 means 512 blockIDs.
-   */
-  public static final String DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE =
-      "dfs.cblock.cache.block.buffer.size";
-  public static final int DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT = 512;
-
-  public static final String DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL =
-      "dfs.cblock.block.buffer.flush.interval";
-  public static final String DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL_DEFAULT =
-      "60s";
-
-  // jscsi server settings
-  public static final String DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY =
-      "dfs.cblock.jscsi.server.address";
-  public static final String DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT =
-      "0.0.0.0";
-  public static final String DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY =
-      "dfs.cblock.jscsi.cblock.server.address";
-  public static final String DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT =
-      "127.0.0.1";
-
-  // to what address cblock server should talk to scm?
-  public static final String DFS_CBLOCK_SCM_IPADDRESS_KEY =
-      "dfs.cblock.scm.ipaddress";
-  public static final String DFS_CBLOCK_SCM_IPADDRESS_DEFAULT =
-      "127.0.0.1";
-  public static final String DFS_CBLOCK_SCM_PORT_KEY =
-      "dfs.cblock.scm.port";
-  public static final int DFS_CBLOCK_SCM_PORT_DEFAULT = 9860;
-
-  public static final String DFS_CBLOCK_CONTAINER_SIZE_GB_KEY =
-      "dfs.cblock.container.size.gb";
-  public static final int DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT =
-      5;
-
-  // LevelDB cache file uses an off-heap cache in LevelDB of 256 MB.
-  public static final String DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_KEY =
-      "dfs.cblock.cache.leveldb.cache.size.mb";
-  public static final int DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_DEFAULT = 256;
-
-  /**
-   * Cache does an best case attempt to write a block to a container.
-   * At some point of time, we will need to handle the case where we did try
-   * 64K times and is till not able to write to the container.
-   *
-   * TODO: We will need cBlock Server to allow us to do a remapping of the
-   * block location in case of failures, at that point we should reduce the
-   * retry count to a more normal number. This is approximately 18 hours of
-   * retry.
-   */
-  public static final String DFS_CBLOCK_CACHE_MAX_RETRY_KEY =
-      "dfs.cblock.cache.max.retry";
-  public static final int DFS_CBLOCK_CACHE_MAX_RETRY_DEFAULT =
-      64 * 1024;
-
-  /**
-   * Cblock CLI configs.
-   */
-  public static final String DFS_CBLOCK_MANAGER_POOL_SIZE =
-      "dfs.cblock.manager.pool.size";
-  public static final int DFS_CBLOCK_MANAGER_POOL_SIZE_DEFAULT = 16;
-
-  /**
-   * currently the largest supported volume is about 8TB, which might take
-   * > 20 seconds to finish creating containers. thus set timeout to 30 sec.
-   */
-  public static final String DFS_CBLOCK_RPC_TIMEOUT =
-      "dfs.cblock.rpc.timeout";
-  public static final String DFS_CBLOCK_RPC_TIMEOUT_DEFAULT = "300s";
-
-  public static final String DFS_CBLOCK_ISCSI_ADVERTISED_IP =
-      "dfs.cblock.iscsi.advertised.ip";
-
-  public static final String DFS_CBLOCK_ISCSI_ADVERTISED_PORT =
-      "dfs.cblock.iscsi.advertised.port";
-
-  public static final int DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT = 3260;
-
-
-  public static final String
-      DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED
-        = "dfs.cblock.kubernetes.dynamic-provisioner.enabled";
-
-  public static final boolean
-      DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED_DEFAULT = false;
-
-  public static final String
-      DFS_CBLOCK_KUBERNETES_CBLOCK_USER =
-         "dfs.cblock.kubernetes.cblock-user";
-
-  public static final String
-      DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT =
-         "iqn.2001-04.org.apache.hadoop";
-
-  public static final String
-      DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY =
-         "dfs.cblock.kubernetes.configfile";
-
-  private CBlockConfigKeys() {
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/package-info.java
deleted file mode 100644
index 5f5d3cd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-/**
- This package contains ozone client side libraries.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
index 517e474..96f22ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
@@ -7,7 +7,7 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- * http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -54,6 +54,7 @@ public class DatanodeID implements Comparable<DatanodeID> {
   private String xferAddr;
   private int containerPort; // container Stand_alone Rpc port.
   private int ratisPort; // Container Ratis RPC Port.
+  private int ozoneRestPort;
 
   /**
    * UUID identifying a given datanode. For upgraded Datanodes this is the
@@ -75,6 +76,7 @@ public class DatanodeID implements Comparable<DatanodeID> {
         from.getInfoPort(),
         from.getInfoSecurePort(),
         from.getIpcPort());
+    this.ozoneRestPort = from.getOzoneRestPort();
     this.peerHostName = from.getPeerHostName();
   }
 
@@ -267,6 +269,8 @@ public class DatanodeID implements Comparable<DatanodeID> {
     infoPort = nodeReg.getInfoPort();
     infoSecurePort = nodeReg.getInfoSecurePort();
     ipcPort = nodeReg.getIpcPort();
+    ratisPort = nodeReg.getRatisPort();
+    ozoneRestPort = nodeReg.getOzoneRestPort();
   }
 
   /**
@@ -313,6 +317,24 @@ public class DatanodeID implements Comparable<DatanodeID> {
   }
 
   /**
+   * Ozone rest port.
+   *
+   * @return rest port.
+   */
+  public int getOzoneRestPort() {
+    return ozoneRestPort;
+  }
+
+  /**
+   * Set the ozone rest port.
+   *
+   * @param ozoneRestPort
+   */
+  public void setOzoneRestPort(int ozoneRestPort) {
+    this.ozoneRestPort = ozoneRestPort;
+  }
+
+  /**
    * Returns a DataNode ID from the protocol buffers.
    *
    * @param datanodeIDProto - protoBuf Message
@@ -326,6 +348,7 @@ public class DatanodeID implements Comparable<DatanodeID> {
         datanodeIDProto.getInfoSecurePort(), datanodeIDProto.getIpcPort());
     id.setContainerPort(datanodeIDProto.getContainerPort());
     id.setRatisPort(datanodeIDProto.getRatisPort());
+    id.setOzoneRestPort(datanodeIDProto.getOzoneRestPort());
     return id;
   }
 
@@ -345,6 +368,7 @@ public class DatanodeID implements Comparable<DatanodeID> {
         .setIpcPort(this.getIpcPort())
         .setContainerPort(this.getContainerPort())
         .setRatisPort(this.getRatisPort())
+        .setOzoneRestPort(this.getOzoneRestPort())
         .build();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index d9e7aa0..e94b094 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -292,7 +292,9 @@ public class PBHelperClient {
             dn.getDatanodeUuid() : "")
         .setInfoPort(dn.getInfoPort())
         .setInfoSecurePort(dn.getInfoSecurePort())
-        .setIpcPort(dn.getIpcPort()).build();
+        .setIpcPort(dn.getIpcPort())
+        .setOzoneRestPort(dn.getOzoneRestPort())
+        .build();
   }
 
   public static DatanodeInfoProto.AdminState convert(
@@ -742,9 +744,13 @@ public class PBHelperClient {
 
   // DatanodeId
   public static DatanodeID convert(DatanodeIDProto dn) {
-    return new DatanodeID(dn.getIpAddr(), dn.getHostName(),
-        dn.getDatanodeUuid(), dn.getXferPort(), dn.getInfoPort(),
-        dn.hasInfoSecurePort() ? dn.getInfoSecurePort() : 0, dn.getIpcPort());
+    DatanodeID datanodeID =
+        new DatanodeID(dn.getIpAddr(), dn.getHostName(), dn.getDatanodeUuid(),
+            dn.getXferPort(), dn.getInfoPort(),
+            dn.hasInfoSecurePort() ? dn.getInfoSecurePort() : 0,
+            dn.getIpcPort());
+    datanodeID.setOzoneRestPort(dn.getOzoneRestPort());
+    return datanodeID;
   }
 
   public static AdminStates convert(AdminState adminState) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneAcl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneAcl.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneAcl.java
deleted file mode 100644
index ff0ac4e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneAcl.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-
-package org.apache.hadoop.ozone;
-
-import java.util.Objects;
-
-/**
- * OzoneACL classes define bucket ACLs used in OZONE.
- *
- * ACLs in Ozone follow this pattern.
- * • user:name:rw
- * • group:name:rw
- * • world::rw
- */
-public class OzoneAcl {
-  private OzoneACLType type;
-  private String name;
-  private OzoneACLRights rights;
-
-  /**
-   * Constructor for OzoneAcl.
-   */
-  public OzoneAcl() {
-  }
-
-  /**
-   * Constructor for OzoneAcl.
-   *
-   * @param type - Type
-   * @param name - Name of user
-   * @param rights - Rights
-   */
-  public OzoneAcl(OzoneACLType type, String name, OzoneACLRights rights) {
-    this.name = name;
-    this.rights = rights;
-    this.type = type;
-    if (type == OzoneACLType.WORLD && name.length() != 0) {
-      throw new IllegalArgumentException("Unexpected name part in world type");
-    }
-    if (((type == OzoneACLType.USER) || (type == OzoneACLType.GROUP))
-        && (name.length() == 0)) {
-      throw new IllegalArgumentException("User or group name is required");
-    }
-  }
-
-  /**
-   * Parses an ACL string and returns the ACL object.
-   *
-   * @param acl - Acl String , Ex. user:anu:rw
-   *
-   * @return - Ozone ACLs
-   */
-  public static OzoneAcl parseAcl(String acl) throws IllegalArgumentException {
-    if ((acl == null) || acl.isEmpty()) {
-      throw new IllegalArgumentException("ACLs cannot be null or empty");
-    }
-    String[] parts = acl.trim().split(":");
-    if (parts.length < 3) {
-      throw new IllegalArgumentException("ACLs are not in expected format");
-    }
-
-    OzoneACLType aclType = OzoneACLType.valueOf(parts[0].toUpperCase());
-    OzoneACLRights rights = OzoneACLRights.getACLRight(parts[2].toLowerCase());
-
-    // TODO : Support sanitation of these user names by calling into
-    // userAuth Interface.
-    return new OzoneAcl(aclType, parts[1], rights);
-  }
-
-  @Override
-  public String toString() {
-    return type + ":" + name + ":" + OzoneACLRights.getACLRightsString(rights);
-  }
-
-  /**
-   * Returns a hash code value for the object. This method is
-   * supported for the benefit of hash tables.
-   *
-   * @return a hash code value for this object.
-   *
-   * @see Object#equals(Object)
-   * @see System#identityHashCode
-   */
-  @Override
-  public int hashCode() {
-    return Objects.hash(this.getName(), this.getRights().toString(),
-                        this.getType().toString());
-  }
-
-  /**
-   * Returns name.
-   *
-   * @return name
-   */
-  public String getName() {
-    return name;
-  }
-
-  /**
-   * Returns Rights.
-   *
-   * @return - Rights
-   */
-  public OzoneACLRights getRights() {
-    return rights;
-  }
-
-  /**
-   * Returns Type.
-   *
-   * @return type
-   */
-  public OzoneACLType getType() {
-    return type;
-  }
-
-  /**
-   * Indicates whether some other object is "equal to" this one.
-   *
-   * @param obj the reference object with which to compare.
-   *
-   * @return {@code true} if this object is the same as the obj
-   * argument; {@code false} otherwise.
-   */
-  @Override
-  public boolean equals(Object obj) {
-    if (obj == null) {
-      return false;
-    }
-    if (getClass() != obj.getClass()) {
-      return false;
-    }
-    OzoneAcl otherAcl = (OzoneAcl) obj;
-    return otherAcl.getName().equals(this.getName()) &&
-        otherAcl.getRights() == this.getRights() &&
-        otherAcl.getType() == this.getType();
-  }
-
-  /**
-   * ACL types.
-   */
-  public enum OzoneACLType {
-    USER(OzoneConsts.OZONE_ACL_USER_TYPE),
-    GROUP(OzoneConsts.OZONE_ACL_GROUP_TYPE),
-    WORLD(OzoneConsts.OZONE_ACL_WORLD_TYPE);
-
-    /**
-     * String value for this Enum.
-     */
-    private final String value;
-
-    /**
-     * Init OzoneACLtypes enum.
-     *
-     * @param val String type for this enum.
-     */
-    OzoneACLType(String val) {
-      value = val;
-    }
-  }
-
-  /**
-   * ACL rights.
-   */
-  public enum OzoneACLRights {
-    READ, WRITE, READ_WRITE;
-
-    /**
-     * Returns the ACL rights based on passed in String.
-     *
-     * @param type ACL right string
-     *
-     * @return OzoneACLRights
-     */
-    public static OzoneACLRights getACLRight(String type) {
-      if (type == null || type.isEmpty()) {
-        throw new IllegalArgumentException("ACL right cannot be empty");
-      }
-
-      switch (type) {
-      case OzoneConsts.OZONE_ACL_READ:
-        return OzoneACLRights.READ;
-      case OzoneConsts.OZONE_ACL_WRITE:
-        return OzoneACLRights.WRITE;
-      case OzoneConsts.OZONE_ACL_READ_WRITE:
-      case OzoneConsts.OZONE_ACL_WRITE_READ:
-        return OzoneACLRights.READ_WRITE;
-      default:
-        throw new IllegalArgumentException("ACL right is not recognized");
-      }
-
-    }
-
-    /**
-     * Returns String representation of ACL rights.
-     * @param acl OzoneACLRights
-     * @return String representation of acl
-     */
-    public static String getACLRightsString(OzoneACLRights acl) {
-      switch(acl) {
-      case READ:
-        return OzoneConsts.OZONE_ACL_READ;
-      case WRITE:
-        return OzoneConsts.OZONE_ACL_WRITE;
-      case READ_WRITE:
-        return OzoneConsts.OZONE_ACL_READ_WRITE;
-      default:
-        throw new IllegalArgumentException("ACL right is not recognized");
-      }
-    }
-
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
deleted file mode 100644
index b3b35f1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.ozone.client.ReplicationFactor;
-import org.apache.hadoop.ozone.client.ReplicationType;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.apache.hadoop.ozone.client.rest.RestClient;
-import org.apache.hadoop.ozone.client.rpc.RpcClient;
-import org.apache.hadoop.scm.ScmConfigKeys;
-
-/**
- * This class contains constants for configuration keys used in Ozone.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public final class OzoneConfigKeys {
-  public static final String DFS_CONTAINER_IPC_PORT =
-      "dfs.container.ipc";
-  public static final int DFS_CONTAINER_IPC_PORT_DEFAULT = 9859;
-
-  /**
-   *
-   * When set to true, allocate a random free port for ozone container,
-   * so that a mini cluster is able to launch multiple containers on a node.
-   *
-   * When set to false (default), container port is fixed as specified by
-   * DFS_CONTAINER_IPC_PORT_DEFAULT.
-   */
-  public static final String DFS_CONTAINER_IPC_RANDOM_PORT =
-      "dfs.container.ipc.random.port";
-  public static final boolean DFS_CONTAINER_IPC_RANDOM_PORT_DEFAULT =
-      false;
-
-  /**
-   * Ratis Port where containers listen to.
-   */
-  public static final String DFS_CONTAINER_RATIS_IPC_PORT =
-      "dfs.container.ratis.ipc";
-  public static final int DFS_CONTAINER_RATIS_IPC_PORT_DEFAULT = 9858;
-
-  /**
-   * When set to true, allocate a random free port for ozone container, so that
-   * a mini cluster is able to launch multiple containers on a node.
-   */
-  public static final String DFS_CONTAINER_RATIS_IPC_RANDOM_PORT =
-      "dfs.container.ratis.ipc.random.port";
-  public static final boolean DFS_CONTAINER_RATIS_IPC_RANDOM_PORT_DEFAULT =
-      false;
-
-  public static final String OZONE_LOCALSTORAGE_ROOT =
-      "ozone.localstorage.root";
-  public static final String OZONE_LOCALSTORAGE_ROOT_DEFAULT = "/tmp/ozone";
-  public static final String OZONE_ENABLED =
-      "ozone.enabled";
-  public static final boolean OZONE_ENABLED_DEFAULT = false;
-  public static final String OZONE_HANDLER_TYPE_KEY =
-      "ozone.handler.type";
-  public static final String OZONE_HANDLER_TYPE_DEFAULT = "distributed";
-  public static final String OZONE_TRACE_ENABLED_KEY =
-      "ozone.trace.enabled";
-  public static final boolean OZONE_TRACE_ENABLED_DEFAULT = false;
-
-  public static final String OZONE_METADATA_DIRS =
-      "ozone.metadata.dirs";
-
-  public static final String OZONE_METADATA_STORE_IMPL =
-      "ozone.metastore.impl";
-  public static final String OZONE_METADATA_STORE_IMPL_LEVELDB =
-      "LevelDB";
-  public static final String OZONE_METADATA_STORE_IMPL_ROCKSDB =
-      "RocksDB";
-  public static final String OZONE_METADATA_STORE_IMPL_DEFAULT =
-      OZONE_METADATA_STORE_IMPL_ROCKSDB;
-
-  public static final String OZONE_METADATA_STORE_ROCKSDB_STATISTICS =
-      "ozone.metastore.rocksdb.statistics";
-
-  public static final String  OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT =
-      "ALL";
-  public static final String OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF =
-      "OFF";
-
-  public static final String OZONE_CONTAINER_CACHE_SIZE =
-      "ozone.container.cache.size";
-  public static final int OZONE_CONTAINER_CACHE_DEFAULT = 1024;
-
-  public static final String OZONE_SCM_BLOCK_SIZE_IN_MB =
-      "ozone.scm.block.size.in.mb";
-  public static final long OZONE_SCM_BLOCK_SIZE_DEFAULT = 256;
-
-  /**
-   * Ozone administrator users delimited by comma.
-   * If not set, only the user who launches an ozone service will be the
-   * admin user. This property must be set if ozone services are started by
-   * different users. Otherwise the RPC layer will reject calls from
-   * other servers which are started by users not in the list.
-   * */
-  public static final String OZONE_ADMINISTRATORS =
-      "ozone.administrators";
-
-  public static final String OZONE_CLIENT_PROTOCOL =
-      "ozone.client.protocol";
-  public static final Class<? extends ClientProtocol>
-      OZONE_CLIENT_PROTOCOL_RPC = RpcClient.class;
-  public static final Class<? extends ClientProtocol>
-      OZONE_CLIENT_PROTOCOL_REST = RestClient.class;
-
-  // This defines the overall connection limit for the connection pool used in
-  // RestClient.
-  public static final String OZONE_REST_CLIENT_HTTP_CONNECTION_MAX =
-      "ozone.rest.client.http.connection.max";
-  public static final int OZONE_REST_CLIENT_HTTP_CONNECTION_DEFAULT = 100;
-
-  // This defines the connection limit per one HTTP route/host.
-  public static final String OZONE_REST_CLIENT_HTTP_CONNECTION_PER_ROUTE_MAX =
-      "ozone.rest.client.http.connection.per-route.max";
-
-  public static final int
-      OZONE_REST_CLIENT_HTTP_CONNECTION_PER_ROUTE_MAX_DEFAULT = 20;
-
-  public static final String OZONE_CLIENT_SOCKET_TIMEOUT =
-      "ozone.client.socket.timeout";
-  public static final int OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT = 5000;
-  public static final String OZONE_CLIENT_CONNECTION_TIMEOUT =
-      "ozone.client.connection.timeout";
-  public static final int OZONE_CLIENT_CONNECTION_TIMEOUT_DEFAULT = 5000;
-
-  public static final String OZONE_REPLICATION = "ozone.replication";
-  public static final int OZONE_REPLICATION_DEFAULT =
-      ReplicationFactor.THREE.getValue();
-
-  public static final String OZONE_REPLICATION_TYPE = "ozone.replication.type";
-  public static final String OZONE_REPLICATION_TYPE_DEFAULT =
-      ReplicationType.RATIS.toString();
-
-  /**
-   * Configuration property to configure the cache size of client list calls.
-   */
-  public static final String OZONE_CLIENT_LIST_CACHE_SIZE =
-      "ozone.client.list.cache";
-  public static final int OZONE_CLIENT_LIST_CACHE_SIZE_DEFAULT = 1000;
-
-  /**
-   * Configuration properties for Ozone Block Deleting Service.
-   */
-  public static final String OZONE_BLOCK_DELETING_SERVICE_INTERVAL =
-      "ozone.block.deleting.service.interval";
-  public static final String OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT
-      = "60s";
-
-  /**
-   * The interval of open key clean service.
-   */
-  public static final String OZONE_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_SECONDS =
-      "ozone.open.key.cleanup.service.interval.seconds";
-  public static final int
-      OZONE_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_SECONDS_DEFAULT
-      = 24 * 3600; // a total of 24 hour
-
-  /**
-   * An open key gets cleaned up when it is being in open state for too long.
-   */
-  public static final String OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS =
-      "ozone.open.key.expire.threshold";
-  public static final int OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT =
-      24 * 3600;
-
-  public static final String OZONE_BLOCK_DELETING_SERVICE_TIMEOUT =
-      "ozone.block.deleting.service.timeout";
-  public static final String OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT
-      = "300s"; // 300s for default
-
-  public static final String OZONE_KEY_PREALLOCATION_MAXSIZE =
-      "ozone.key.preallocation.maxsize";
-  public static final long OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT
-      = 128 * OzoneConsts.MB;
-
-  public static final String OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER =
-      "ozone.block.deleting.limit.per.task";
-  public static final int OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER_DEFAULT
-      = 1000;
-
-  public static final String OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL
-      = "ozone.block.deleting.container.limit.per.interval";
-  public static final int
-      OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT = 10;
-
-  public static final String OZONE_CONTAINER_REPORT_INTERVAL =
-      "ozone.container.report.interval";
-  public static final String OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT =
-      "60s";
-
-  public static final String DFS_CONTAINER_RATIS_ENABLED_KEY
-      = ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY;
-  public static final boolean DFS_CONTAINER_RATIS_ENABLED_DEFAULT
-      = ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT;
-  public static final String DFS_CONTAINER_RATIS_RPC_TYPE_KEY
-      = ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY;
-  public static final String DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT
-      = ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT;
-  public static final String DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY
-      = ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY;
-  public static final int DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT
-      = ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT;
-  public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY
-      = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY;
-  public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
-      = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT;
-  public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY
-      = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY;
-  public static final int DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT
-      = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;
-  public static final int DFS_CONTAINER_CHUNK_MAX_SIZE
-      = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
-  public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
-      "dfs.container.ratis.datanode.storage.dir";
-
-  public static final String OZONE_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
-      "ozone.web.authentication.kerberos.principal";
-
-  /**
-   * There is no need to instantiate this class.
-   */
-  private OzoneConfigKeys() {
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
deleted file mode 100644
index 3cc4697..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * Set of constants used in Ozone implementation.
- */
-@InterfaceAudience.Private
-public final class OzoneConsts {
-  public static final String OZONE_SIMPLE_ROOT_USER = "root";
-  public static final String OZONE_SIMPLE_HDFS_USER = "hdfs";
-
-  /*
-   * BucketName length is used for both buckets and volume lengths
-   */
-  public static final int OZONE_MIN_BUCKET_NAME_LENGTH = 3;
-  public static final int OZONE_MAX_BUCKET_NAME_LENGTH = 63;
-
-  public static final String OZONE_ACL_USER_TYPE = "user";
-  public static final String OZONE_ACL_GROUP_TYPE = "group";
-  public static final String OZONE_ACL_WORLD_TYPE = "world";
-
-  public static final String OZONE_ACL_READ = "r";
-  public static final String OZONE_ACL_WRITE = "w";
-  public static final String OZONE_ACL_READ_WRITE = "rw";
-  public static final String OZONE_ACL_WRITE_READ = "wr";
-
-  public static final String OZONE_DATE_FORMAT =
-      "EEE, dd MMM yyyy HH:mm:ss zzz";
-  public static final String OZONE_TIME_ZONE = "GMT";
-
-  public static final String OZONE_COMPONENT = "component";
-  public static final String OZONE_FUNCTION  = "function";
-  public static final String OZONE_RESOURCE = "resource";
-  public static final String OZONE_USER = "user";
-  public static final String OZONE_REQUEST = "request";
-
-  public static final String CONTAINER_EXTENSION = ".container";
-  public static final String CONTAINER_META = ".meta";
-
-  //  container storage is in the following format.
-  //  Data Volume basePath/containers/<containerName>/metadata and
-  //  Data Volume basePath/containers/<containerName>/data/...
-  public static final String CONTAINER_PREFIX  = "containers";
-  public static final String CONTAINER_META_PATH = "metadata";
-  public static final String CONTAINER_DATA_PATH = "data";
-  public static final String CONTAINER_TEMPORARY_CHUNK_PREFIX = "tmp";
-  public static final String CONTAINER_CHUNK_NAME_DELIMITER = ".";
-  public static final String CONTAINER_ROOT_PREFIX = "repository";
-
-  public static final String FILE_HASH = "SHA-256";
-  public final static String CHUNK_OVERWRITE = "OverWriteRequested";
-
-  public static final int CHUNK_SIZE = 1 * 1024 * 1024; // 1 MB
-  public static final long KB = 1024L;
-  public static final long MB = KB * 1024L;
-  public static final long GB = MB * 1024L;
-  public static final long TB = GB * 1024L;
-
-  /**
-   * level DB names used by SCM and data nodes.
-   */
-  public static final String CONTAINER_DB_SUFFIX = "container.db";
-  public static final String SCM_CONTAINER_DB = "scm-" + CONTAINER_DB_SUFFIX;
-  public static final String DN_CONTAINER_DB = "-dn-"+ CONTAINER_DB_SUFFIX;
-  public static final String BLOCK_DB = "block.db";
-  public static final String NODEPOOL_DB = "nodepool.db";
-  public static final String OPEN_CONTAINERS_DB = "openContainers.db";
-  public static final String DELETED_BLOCK_DB = "deletedBlock.db";
-  public static final String KSM_DB_NAME = "ksm.db";
-
-  /**
-   * Supports Bucket Versioning.
-   */
-  public enum Versioning {NOT_DEFINED, ENABLED, DISABLED}
-
-  /**
-   * Ozone handler types.
-   */
-  public static final String OZONE_HANDLER_DISTRIBUTED = "distributed";
-  public static final String OZONE_HANDLER_LOCAL = "local";
-
-  public static final String DELETING_KEY_PREFIX = "#deleting#";
-  public static final String OPEN_KEY_PREFIX = "#open#";
-  public static final String OPEN_KEY_ID_DELIMINATOR = "#";
-
-  /**
-   * KSM LevelDB prefixes.
-   *
-   * KSM DB stores metadata as KV pairs with certain prefixes,
-   * prefix is used to improve the performance to get related
-   * metadata.
-   *
-   * KSM DB Schema:
-   *  ----------------------------------------------------------
-   *  |  KEY                                     |     VALUE   |
-   *  ----------------------------------------------------------
-   *  | $userName                                |  VolumeList |
-   *  ----------------------------------------------------------
-   *  | /#volumeName                             |  VolumeInfo |
-   *  ----------------------------------------------------------
-   *  | /#volumeName/#bucketName                 |  BucketInfo |
-   *  ----------------------------------------------------------
-   *  | /volumeName/bucketName/keyName           |  KeyInfo    |
-   *  ----------------------------------------------------------
-   *  | #deleting#/volumeName/bucketName/keyName |  KeyInfo    |
-   *  ----------------------------------------------------------
-   */
-  public static final String KSM_VOLUME_PREFIX = "/#";
-  public static final String KSM_BUCKET_PREFIX = "/#";
-  public static final String KSM_KEY_PREFIX = "/";
-  public static final String KSM_USER_PREFIX = "$";
-
-  /**
-   * Max KSM Quota size of 1024 PB.
-   */
-  public static final long MAX_QUOTA_IN_BYTES = 1024L * 1024 * TB;
-
-  /**
-   * Max number of keys returned per list buckets operation.
-   */
-  public static final int MAX_LISTBUCKETS_SIZE  = 1024;
-
-  /**
-   * Max number of keys returned per list keys operation.
-   */
-  public static final int MAX_LISTKEYS_SIZE  = 1024;
-
-  /**
-   * Max number of volumes returned per list volumes operation.
-   */
-  public static final int MAX_LISTVOLUMES_SIZE = 1024;
-
-  public static final int INVALID_PORT = -1;
-
-
-  // The ServiceListJSONServlet context attribute where KeySpaceManager
-  // instance gets stored.
-  public static final String KSM_CONTEXT_ATTRIBUTE = "ozone.ksm";
-
-  private OzoneConsts() {
-    // Never Constructed
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java
deleted file mode 100644
index 39b7bb8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.OzoneAcl;
-
-import java.util.List;
-
-/**
- * This class encapsulates the arguments that are
- * required for creating a bucket.
- */
-public final class BucketArgs {
-
-  /**
-   * ACL Information.
-   */
-  private List<OzoneAcl> acls;
-  /**
-   * Bucket Version flag.
-   */
-  private Boolean versioning;
-  /**
-   * Type of storage to be used for this bucket.
-   * [RAM_DISK, SSD, DISK, ARCHIVE]
-   */
-  private StorageType storageType;
-
-  /**
-   * Private constructor, constructed via builder.
-   * @param versioning Bucket version flag.
-   * @param storageType Storage type to be used.
-   * @param acls list of ACLs.
-   */
-  private BucketArgs(Boolean versioning, StorageType storageType,
-                     List<OzoneAcl> acls) {
-    this.acls = acls;
-    this.versioning = versioning;
-    this.storageType = storageType;
-  }
-
-  /**
-   * Returns true if bucket version is enabled, else false.
-   * @return isVersionEnabled
-   */
-  public Boolean getVersioning() {
-    return versioning;
-  }
-
-  /**
-   * Returns the type of storage to be used.
-   * @return StorageType
-   */
-  public StorageType getStorageType() {
-    return storageType;
-  }
-
-  /**
-   * Returns the ACL's associated with this bucket.
-   * @return List<OzoneAcl>
-   */
-  public List<OzoneAcl> getAcls() {
-    return acls;
-  }
-
-  /**
-   * Returns new builder class that builds a KsmBucketInfo.
-   *
-   * @return Builder
-   */
-  public static BucketArgs.Builder newBuilder() {
-    return new BucketArgs.Builder();
-  }
-
-  /**
-   * Builder for KsmBucketInfo.
-   */
-  public static class Builder {
-    private Boolean versioning;
-    private StorageType storageType;
-    private List<OzoneAcl> acls;
-
-    public BucketArgs.Builder setVersioning(Boolean versionFlag) {
-      this.versioning = versionFlag;
-      return this;
-    }
-
-    public BucketArgs.Builder setStorageType(StorageType storage) {
-      this.storageType = storage;
-      return this;
-    }
-
-    public BucketArgs.Builder setAcls(List<OzoneAcl> listOfAcls) {
-      this.acls = listOfAcls;
-      return this;
-    }
-
-    /**
-     * Constructs the BucketArgs.
-     * @return instance of BucketArgs.
-     */
-    public BucketArgs build() {
-      return new BucketArgs(versioning, storageType, acls);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
deleted file mode 100644
index 1ed41f5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.apache.hadoop.security.UserGroupInformation;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-/**
- * ObjectStore class is responsible for the client operations that can be
- * performed on Ozone Object Store.
- */
-public class ObjectStore {
-
-  /**
-   * The proxy used for connecting to the cluster and perform
-   * client operations.
-   */
-  private final ClientProtocol proxy;
-
-  /**
-   * Cache size to be used for listVolume calls.
-   */
-  private int listCacheSize;
-
-  /**
-   * Creates an instance of ObjectStore.
-   * @param conf Configuration object.
-   * @param proxy ClientProtocol proxy.
-   */
-  public ObjectStore(Configuration conf, ClientProtocol proxy) {
-    this.proxy = proxy;
-    this.listCacheSize = OzoneClientUtils.getListCacheSize(conf);
-  }
-
-  /**
-   * Creates the volume with default values.
-   * @param volumeName Name of the volume to be created.
-   * @throws IOException
-   */
-  public void createVolume(String volumeName) throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    OzoneClientUtils.verifyResourceName(volumeName);
-    proxy.createVolume(volumeName);
-  }
-
-  /**
-   * Creates the volume.
-   * @param volumeName Name of the volume to be created.
-   * @param volumeArgs Volume properties.
-   * @throws IOException
-   */
-  public void createVolume(String volumeName, VolumeArgs volumeArgs)
-      throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(volumeArgs);
-    OzoneClientUtils.verifyResourceName(volumeName);
-    proxy.createVolume(volumeName, volumeArgs);
-  }
-
-  /**
-   * Returns the volume information.
-   * @param volumeName Name of the volume.
-   * @return OzoneVolume
-   * @throws IOException
-   */
-  public OzoneVolume getVolume(String volumeName) throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    OzoneClientUtils.verifyResourceName(volumeName);
-    OzoneVolume volume = proxy.getVolumeDetails(volumeName);
-    return volume;
-  }
-
-
-  /**
-   * Returns Iterator to iterate over all the volumes in object store.
-   * The result can be restricted using volume prefix, will return all
-   * volumes if volume prefix is null.
-   *
-   * @param volumePrefix Volume prefix to match
-   * @return {@code Iterator<OzoneVolume>}
-   */
-  public Iterator<OzoneVolume> listVolumes(String volumePrefix)
-      throws IOException {
-    return new VolumeIterator(volumePrefix);
-  }
-
-  /**
-   * Returns Iterator to iterate over the List of volumes owned by a specific
-   * user. The result can be restricted using volume prefix, will return all
-   * volumes if volume prefix is null. If user is null, returns the volume of
-   * current user.
-   *
-   * @param user User Name
-   * @param volumePrefix Volume prefix to match
-   * @return {@code Iterator<OzoneVolume>}
-   */
-  public Iterator<OzoneVolume> listVolumes(String user, String volumePrefix)
-      throws IOException {
-    if(Strings.isNullOrEmpty(user)) {
-      user = UserGroupInformation.getCurrentUser().getShortUserName();
-    }
-    return new VolumeIterator(user, volumePrefix);
-  }
-
-  /**
-   * Deletes the volume.
-   * @param volumeName Name of the volume.
-   * @throws IOException
-   */
-  public void deleteVolume(String volumeName) throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    OzoneClientUtils.verifyResourceName(volumeName);
-    proxy.deleteVolume(volumeName);
-  }
-
-  /**
-   * An Iterator to iterate over {@link OzoneVolume} list.
-   */
-  private class VolumeIterator implements Iterator<OzoneVolume> {
-
-    private String user = null;
-    private String volPrefix = null;
-
-    private Iterator<OzoneVolume> currentIterator;
-    private OzoneVolume currentValue;
-
-    /**
-     * Creates an Iterator to iterate over all volumes in the cluster,
-     * which matches the volume prefix.
-     * @param volPrefix prefix to match
-     */
-    VolumeIterator(String volPrefix) {
-      this(null, volPrefix);
-    }
-
-    /**
-     * Creates an Iterator to iterate over all volumes of the user,
-     * which matches volume prefix.
-     * @param user user name
-     * @param volPrefix volume prefix to match
-     */
-    VolumeIterator(String user, String volPrefix) {
-      this.user = user;
-      this.volPrefix = volPrefix;
-      this.currentValue = null;
-      this.currentIterator = getNextListOfVolumes(null).iterator();
-    }
-
-    @Override
-    public boolean hasNext() {
-      if(!currentIterator.hasNext()) {
-        currentIterator = getNextListOfVolumes(
-            currentValue != null ? currentValue.getName() : null)
-            .iterator();
-      }
-      return currentIterator.hasNext();
-    }
-
-    @Override
-    public OzoneVolume next() {
-      if(hasNext()) {
-        currentValue = currentIterator.next();
-        return currentValue;
-      }
-      throw new NoSuchElementException();
-    }
-
-    /**
-     * Returns the next set of volume list using proxy.
-     * @param prevVolume previous volume, this will be excluded from the result
-     * @return {@code List<OzoneVolume>}
-     */
-    private List<OzoneVolume> getNextListOfVolumes(String prevVolume) {
-      try {
-        //if user is null, we do list of all volumes.
-        if(user != null) {
-          return proxy.listVolumes(user, volPrefix, prevVolume, listCacheSize);
-        }
-        return proxy.listVolumes(volPrefix, prevVolume, listCacheSize);
-      } catch (IOException e) {
-        throw new RuntimeException(e);
-      }
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
deleted file mode 100644
index b94e0f7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++ /dev/null
@@ -1,360 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.apache.hadoop.ozone.OzoneAcl;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-/**
- * A class that encapsulates OzoneBucket.
- */
-public class OzoneBucket {
-
-  /**
-   * The proxy used for connecting to the cluster and perform
-   * client operations.
-   */
-  private final ClientProtocol proxy;
-  /**
-   * Name of the volume in which the bucket belongs to.
-   */
-  private final String volumeName;
-  /**
-   * Name of the bucket.
-   */
-  private final String name;
-  /**
-   * Default replication factor to be used while creating keys.
-   */
-  private final ReplicationFactor defaultReplication;
-
-  /**
-   * Default replication type to be used while creating keys.
-   */
-  private final ReplicationType defaultReplicationType;
-  /**
-   * Bucket ACLs.
-   */
-  private List<OzoneAcl> acls;
-
-  /**
-   * Type of storage to be used for this bucket.
-   * [RAM_DISK, SSD, DISK, ARCHIVE]
-   */
-  private StorageType storageType;
-
-  /**
-   * Bucket Version flag.
-   */
-  private Boolean versioning;
-
-  /**
-   * Cache size to be used for listKey calls.
-   */
-  private int listCacheSize;
-
-  /**
-   * Creation time of the bucket.
-   */
-  private long creationTime;
-
-  /**
-   * Constructs OzoneBucket instance.
-   * @param conf Configuration object.
-   * @param proxy ClientProtocol proxy.
-   * @param volumeName Name of the volume the bucket belongs to.
-   * @param bucketName Name of the bucket.
-   * @param acls ACLs associated with the bucket.
-   * @param storageType StorageType of the bucket.
-   * @param versioning versioning status of the bucket.
-   * @param creationTime creation time of the bucket.
-   */
-  public OzoneBucket(Configuration conf, ClientProtocol proxy,
-                     String volumeName, String bucketName,
-                     List<OzoneAcl> acls, StorageType storageType,
-                     Boolean versioning, long creationTime) {
-    this.proxy = proxy;
-    this.volumeName = volumeName;
-    this.name = bucketName;
-    this.acls = acls;
-    this.storageType = storageType;
-    this.versioning = versioning;
-    this.listCacheSize = OzoneClientUtils.getListCacheSize(conf);
-    this.creationTime = creationTime;
-    this.defaultReplication = ReplicationFactor.valueOf(conf.getInt(
-        OzoneConfigKeys.OZONE_REPLICATION,
-        OzoneConfigKeys.OZONE_REPLICATION_DEFAULT));
-    this.defaultReplicationType = ReplicationType.valueOf(conf.get(
-        OzoneConfigKeys.OZONE_REPLICATION_TYPE,
-        OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT));
-  }
-
-  /**
-   * Returns Volume Name.
-   *
-   * @return volumeName
-   */
-  public String getVolumeName() {
-    return volumeName;
-  }
-
-  /**
-   * Returns Bucket Name.
-   *
-   * @return bucketName
-   */
-  public String getName() {
-    return name;
-  }
-
-  /**
-   * Returns ACL's associated with the Bucket.
-   *
-   * @return acls
-   */
-  public List<OzoneAcl> getAcls() {
-    return acls;
-  }
-
-  /**
-   * Returns StorageType of the Bucket.
-   *
-   * @return storageType
-   */
-  public StorageType getStorageType() {
-    return storageType;
-  }
-
-  /**
-   * Returns Versioning associated with the Bucket.
-   *
-   * @return versioning
-   */
-  public Boolean getVersioning() {
-    return versioning;
-  }
-
-  /**
-   * Returns creation time of the Bucket.
-   *
-   * @return creation time of the bucket
-   */
-  public long getCreationTime() {
-    return creationTime;
-  }
-
-  /**
-   * Adds ACLs to the Bucket.
-   * @param addAcls ACLs to be added
-   * @throws IOException
-   */
-  public void addAcls(List<OzoneAcl> addAcls) throws IOException {
-    Preconditions.checkNotNull(proxy, "Client proxy is not set.");
-    Preconditions.checkNotNull(addAcls);
-    proxy.addBucketAcls(volumeName, name, addAcls);
-    addAcls.stream().filter(acl -> !acls.contains(acl)).forEach(
-        acls::add);
-  }
-
-  /**
-   * Removes ACLs from the bucket.
-   * @param removeAcls ACLs to be removed
-   * @throws IOException
-   */
-  public void removeAcls(List<OzoneAcl> removeAcls) throws IOException {
-    Preconditions.checkNotNull(proxy, "Client proxy is not set.");
-    Preconditions.checkNotNull(removeAcls);
-    proxy.removeBucketAcls(volumeName, name, removeAcls);
-    acls.removeAll(removeAcls);
-  }
-
-  /**
-   * Sets/Changes the storage type of the bucket.
-   * @param newStorageType Storage type to be set
-   * @throws IOException
-   */
-  public void setStorageType(StorageType newStorageType) throws IOException {
-    Preconditions.checkNotNull(proxy, "Client proxy is not set.");
-    Preconditions.checkNotNull(newStorageType);
-    proxy.setBucketStorageType(volumeName, name, newStorageType);
-    storageType = newStorageType;
-  }
-
-  /**
-   * Enable/Disable versioning of the bucket.
-   * @param newVersioning
-   * @throws IOException
-   */
-  public void setVersioning(Boolean newVersioning) throws IOException {
-    Preconditions.checkNotNull(proxy, "Client proxy is not set.");
-    Preconditions.checkNotNull(newVersioning);
-    proxy.setBucketVersioning(volumeName, name, newVersioning);
-    versioning = newVersioning;
-  }
-
-  /**
-   * Creates a new key in the bucket, with default replication type RATIS and
-   * with replication factor THREE.
-   * @param key Name of the key to be created.
-   * @param size Size of the data the key will point to.
-   * @return OzoneOutputStream to which the data has to be written.
-   * @throws IOException
-   */
-  public OzoneOutputStream createKey(String key, long size)
-      throws IOException {
-    Preconditions.checkNotNull(proxy, "Client proxy is not set.");
-    Preconditions.checkNotNull(key);
-    return createKey(key, size, defaultReplicationType, defaultReplication);
-  }
-
-  /**
-   * Creates a new key in the bucket.
-   * @param key Name of the key to be created.
-   * @param size Size of the data the key will point to.
-   * @param type Replication type to be used.
-   * @param factor Replication factor of the key.
-   * @return OzoneOutputStream to which the data has to be written.
-   * @throws IOException
-   */
-  public OzoneOutputStream createKey(String key, long size,
-                                     ReplicationType type,
-                                     ReplicationFactor factor)
-      throws IOException {
-    Preconditions.checkNotNull(proxy, "Client proxy is not set.");
-    Preconditions.checkNotNull(key);
-    Preconditions.checkNotNull(type);
-    Preconditions.checkNotNull(factor);
-    return proxy.createKey(volumeName, name, key, size, type, factor);
-  }
-
-  /**
-   * Reads an existing key from the bucket.
-   * @param key Name of the key to be read.
-   * @return OzoneInputStream the stream using which the data can be read.
-   * @throws IOException
-   */
-  public OzoneInputStream readKey(String key) throws IOException {
-    Preconditions.checkNotNull(proxy, "Client proxy is not set.");
-    Preconditions.checkNotNull(key);
-    return proxy.getKey(volumeName, name, key);
-  }
-
-  /**
-   * Returns information about the key.
-   * @param key Name of the key.
-   * @return OzoneKey Information about the key.
-   * @throws IOException
-   */
-  public OzoneKey getKey(String key) throws IOException {
-    Preconditions.checkNotNull(proxy, "Client proxy is not set.");
-    Preconditions.checkNotNull(key);
-    return proxy.getKeyDetails(volumeName, name, key);
-  }
-
-  /**
-   * Returns Iterator to iterate over all keys in the bucket.
-   * The result can be restricted using key prefix, will return all
-   * keys if key prefix is null.
-   *
-   * @param keyPrefix Bucket prefix to match
-   * @return {@code Iterator<OzoneKey>}
-   */
-  public Iterator<OzoneKey> listKeys(String keyPrefix) {
-    return new KeyIterator(keyPrefix);
-  }
-
-  /**
-   * Deletes key from the bucket.
-   * @param key Name of the key to be deleted.
-   * @throws IOException
-   */
-  public void deleteKey(String key) throws IOException {
-    Preconditions.checkNotNull(proxy, "Client proxy is not set.");
-    Preconditions.checkNotNull(key);
-    proxy.deleteKey(volumeName, name, key);
-  }
-
-  /**
-   * An Iterator to iterate over {@link OzoneKey} list.
-   */
-  private class KeyIterator implements Iterator<OzoneKey> {
-
-    private String keyPrefix = null;
-
-    private Iterator<OzoneKey> currentIterator;
-    private OzoneKey currentValue;
-
-
-    /**
-     * Creates an Iterator to iterate over all keys in the bucket,
-     * which matches volume prefix.
-     * @param keyPrefix
-     */
-    KeyIterator(String keyPrefix) {
-      this.keyPrefix = keyPrefix;
-      this.currentValue = null;
-      this.currentIterator = getNextListOfKeys(null).iterator();
-    }
-
-    @Override
-    public boolean hasNext() {
-      if(!currentIterator.hasNext()) {
-        currentIterator = getNextListOfKeys(
-            currentValue != null ? currentValue.getName() : null)
-            .iterator();
-      }
-      return currentIterator.hasNext();
-    }
-
-    @Override
-    public OzoneKey next() {
-      if(hasNext()) {
-        currentValue = currentIterator.next();
-        return currentValue;
-      }
-      throw new NoSuchElementException();
-    }
-
-    /**
-     * Gets the next set of key list using proxy.
-     * @param prevKey
-     * @return {@code List<OzoneVolume>}
-     */
-    private List<OzoneKey> getNextListOfKeys(String prevKey) {
-      try {
-        return proxy.listKeys(volumeName, name, keyPrefix, prevKey,
-            listCacheSize);
-      } catch (IOException e) {
-        throw new RuntimeException(e);
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java
deleted file mode 100644
index f191507..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-/**
- * OzoneClient connects to Ozone Cluster and
- * perform basic operations.
- */
-public class OzoneClient implements Closeable {
-
-  /*
-   * OzoneClient connects to Ozone Cluster and
-   * perform basic operations.
-   *
-   * +-------------+     +---+   +-------------------------------------+
-   * | OzoneClient | --> | C |   | Object Store                        |
-   * |_____________|     | l |   |  +-------------------------------+  |
-   *                     | i |   |  | Volume(s)                     |  |
-   *                     | e |   |  |   +------------------------+  |  |
-   *                     | n |   |  |   | Bucket(s)              |  |  |
-   *                     | t |   |  |   |   +------------------+ |  |  |
-   *                     |   |   |  |   |   | Key -> Value (s) | |  |  |
-   *                     | P |-->|  |   |   |                  | |  |  |
-   *                     | r |   |  |   |   |__________________| |  |  |
-   *                     | o |   |  |   |                        |  |  |
-   *                     | t |   |  |   |________________________|  |  |
-   *                     | o |   |  |                               |  |
-   *                     | c |   |  |_______________________________|  |
-   *                     | o |   |                                     |
-   *                     | l |   |_____________________________________|
-   *                     |___|
-   * Example:
-   * ObjectStore store = client.getObjectStore();
-   * store.createVolume(“volume one”, VolumeArgs);
-   * volume.setQuota(“10 GB”);
-   * OzoneVolume volume = store.getVolume(“volume one”);
-   * volume.createBucket(“bucket one”, BucketArgs);
-   * bucket.setVersioning(true);
-   * OzoneOutputStream os = bucket.createKey(“key one”, 1024);
-   * os.write(byte[]);
-   * os.close();
-   * OzoneInputStream is = bucket.readKey(“key one”);
-   * is.read();
-   * is.close();
-   * bucket.deleteKey(“key one”);
-   * volume.deleteBucket(“bucket one”);
-   * store.deleteVolume(“volume one”);
-   * client.close();
-   */
-
-  private final ClientProtocol proxy;
-  private final ObjectStore objectStore;
-
-  /**
-   * Creates a new OzoneClient object, generally constructed
-   * using {@link OzoneClientFactory}.
-   * @param conf Configuration object
-   * @param proxy ClientProtocol proxy instance
-   */
-  public OzoneClient(Configuration conf, ClientProtocol proxy) {
-    this.proxy = proxy;
-    this.objectStore = new ObjectStore(conf, this.proxy);
-  }
-
-  /**
-   * Returns the object store associated with the Ozone Cluster.
-   * @return ObjectStore
-   */
-  public ObjectStore getObjectStore() {
-    return objectStore;
-  }
-
-  /**
-   * Closes the client and all the underlying resources.
-   * @throws IOException
-   */
-  @Override
-  public void close() throws IOException {
-    proxy.close();
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[35/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java
deleted file mode 100644
index 292662e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java
+++ /dev/null
@@ -1,599 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper;
-
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.Longs;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.cblock.CBlockConfigKeys;
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import org.apache.hadoop.cblock.jscsiHelper.cache.impl.AsyncBlockWriter;
-import org.apache.hadoop.cblock.jscsiHelper.cache.impl.DiskBlock;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.LevelDBStore;
-import org.iq80.leveldb.Options;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.nio.channels.ReadableByteChannel;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_CACHE_KEEP_ALIVE;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_CACHE_MAX_POOL_SIZE;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_CACHE_QUEUE_SIZE_KB;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_CACHE_THREAD_PRIORITY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_DEFAULT;
-
-/**
- * Class that writes to remote containers.
- */
-public class ContainerCacheFlusher implements Runnable {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ContainerCacheFlusher.class);
-  private final LinkedBlockingQueue<Message> messageQueue;
-  private final ThreadPoolExecutor threadPoolExecutor;
-  private final ArrayBlockingQueue<Runnable> workQueue;
-  private final ConcurrentMap<String, RefCountedDB> dbMap;
-  private final ByteBuffer blockIDBuffer;
-  private final ConcurrentMap<String, Pipeline[]> pipelineMap;
-  private final AtomicLong remoteIO;
-  private final XceiverClientManager xceiverClientManager;
-  private final CBlockTargetMetrics metrics;
-  private AtomicBoolean shutdown;
-  private final long levelDBCacheSize;
-  private final int maxRetryCount;
-  private final String tracePrefix;
-
-  private final ConcurrentMap<String, FinishCounter> finishCountMap;
-
-  /**
-   * Constructs the writers to remote queue.
-   */
-  public ContainerCacheFlusher(Configuration config,
-      XceiverClientManager xceiverClientManager,
-      CBlockTargetMetrics metrics) {
-    int queueSize = config.getInt(DFS_CBLOCK_CACHE_QUEUE_SIZE_KB,
-        DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT) * 1024;
-    int corePoolSize = config.getInt(DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE,
-        DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT);
-    int maxPoolSize = config.getInt(DFS_CBLOCK_CACHE_MAX_POOL_SIZE,
-        DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT);
-    long keepAlive = config.getTimeDuration(DFS_CBLOCK_CACHE_KEEP_ALIVE,
-        DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT, TimeUnit.SECONDS);
-    int threadPri = config.getInt(DFS_CBLOCK_CACHE_THREAD_PRIORITY,
-        DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT);
-    int blockBufferSize = config.getInt(DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE,
-        DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT) * (Long.SIZE / Byte.SIZE);
-    levelDBCacheSize = config.getInt(DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_KEY,
-        DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_DEFAULT) * OzoneConsts.MB;
-
-    LOG.info("Cache: Core Pool Size: {}", corePoolSize);
-    LOG.info("Cache: Keep Alive: {}", keepAlive);
-    LOG.info("Cache: Max Pool Size: {}", maxPoolSize);
-    LOG.info("Cache: Thread Pri: {}", threadPri);
-    LOG.info("Cache: BlockBuffer Size: {}", blockBufferSize);
-
-    shutdown = new AtomicBoolean(false);
-    messageQueue = new LinkedBlockingQueue<>();
-    workQueue = new ArrayBlockingQueue<>(queueSize, true);
-
-    ThreadFactory workerThreadFactory = new ThreadFactoryBuilder()
-        .setNameFormat("Cache Block Writer Thread #%d")
-        .setDaemon(true)
-        .setPriority(threadPri)
-        .build();
-    threadPoolExecutor = new ThreadPoolExecutor(corePoolSize, maxPoolSize,
-        keepAlive, TimeUnit.SECONDS, workQueue, workerThreadFactory,
-        new ThreadPoolExecutor.AbortPolicy());
-    threadPoolExecutor.prestartAllCoreThreads();
-
-    dbMap = new ConcurrentHashMap<>();
-    pipelineMap = new ConcurrentHashMap<>();
-    blockIDBuffer = ByteBuffer.allocateDirect(blockBufferSize);
-    this.xceiverClientManager = xceiverClientManager;
-    this.metrics = metrics;
-    this.remoteIO = new AtomicLong();
-
-    this.finishCountMap = new ConcurrentHashMap<>();
-    this.maxRetryCount =
-        config.getInt(CBlockConfigKeys.DFS_CBLOCK_CACHE_MAX_RETRY_KEY,
-            CBlockConfigKeys.DFS_CBLOCK_CACHE_MAX_RETRY_DEFAULT);
-    this.tracePrefix = getTracePrefix();
-  }
-
-  private void checkExistingLog(String prefixFileName, File dbPath) {
-    if (!dbPath.exists()) {
-      LOG.debug("No existing dirty log found at {}", dbPath);
-      return;
-    }
-    LOG.debug("Need to check and requeue existing dirty log {}", dbPath);
-    HashMap<String, ArrayList<String>> allFiles = new HashMap<>();
-    traverse(prefixFileName, dbPath, allFiles);
-    for (Map.Entry<String, ArrayList<String>> entry : allFiles.entrySet()) {
-      String parentPath = entry.getKey();
-      for (String fileName : entry.getValue()) {
-        LOG.info("found {} {} with prefix {}",
-            parentPath, fileName, prefixFileName);
-        processDirtyBlocks(parentPath, fileName);
-      }
-    }
-  }
-
-  private void traverse(String prefixFileName, File path,
-                        HashMap<String, ArrayList<String>> files) {
-    if (path.isFile()) {
-      if (path.getName().startsWith(prefixFileName)) {
-        LOG.debug("found this {} with {}", path.getParent(), path.getName());
-        if (!files.containsKey(path.getParent())) {
-          files.put(path.getParent(), new ArrayList<>());
-        }
-        files.get(path.getParent()).add(path.getName());
-      }
-    } else {
-      File[] listFiles = path.listFiles();
-      if (listFiles != null) {
-        for (File subPath : listFiles) {
-          traverse(prefixFileName, subPath, files);
-        }
-      }
-    }
-  }
-
-  /**
-   * Gets the CBlockTargetMetrics.
-   *
-   * @return CBlockTargetMetrics
-   */
-  public CBlockTargetMetrics getTargetMetrics() {
-    return metrics;
-  }
-
-  /**
-   * Gets the  getXceiverClientManager.
-   *
-   * @return XceiverClientManager
-   */
-  public XceiverClientManager getXceiverClientManager() {
-    return xceiverClientManager;
-  }
-
-  /**
-   * Shutdown this instance.
-   */
-  public void shutdown() {
-    this.shutdown.set(true);
-    threadPoolExecutor.shutdown();
-  }
-
-  public long incrementRemoteIO() {
-    return remoteIO.incrementAndGet();
-  }
-
-  /**
-   * Processes a block cache file and queues those blocks for the remote I/O.
-   *
-   * @param dbPath - Location where the DB can be found.
-   * @param fileName - Block Cache File Name
-   */
-  public void processDirtyBlocks(String dbPath, String fileName) {
-    LOG.info("Adding {}/{} to queue. Queue Length: {}", dbPath, fileName,
-        messageQueue.size());
-    this.messageQueue.add(new Message(dbPath, fileName));
-  }
-
-  public Logger getLOG() {
-    return LOG;
-  }
-
-  /**
-   * Opens a DB if needed or returns a handle to an already open DB.
-   *
-   * @param dbPath -- dbPath
-   * @return the levelDB on the given path.
-   * @throws IOException
-   */
-  public synchronized LevelDBStore openDB(String dbPath)
-      throws IOException {
-    if (dbMap.containsKey(dbPath)) {
-      RefCountedDB refDB = dbMap.get(dbPath);
-      refDB.open();
-      return refDB.db;
-    } else {
-      Options options = new Options();
-      options.cacheSize(levelDBCacheSize);
-      options.createIfMissing(true);
-      LevelDBStore cacheDB = new LevelDBStore(
-          new File(getDBFileName(dbPath)), options);
-      RefCountedDB refDB = new RefCountedDB(dbPath, cacheDB);
-      dbMap.put(dbPath, refDB);
-      return cacheDB;
-    }
-  }
-
-  /**
-   * Updates the container map. This data never changes so we will update this
-   * during restarts and it should not hurt us.
-   *
-   * Once a CBlockLocalCache cache is registered, requeue dirty/retry log files
-   * for the volume
-   *
-   * @param dbPath - DbPath
-   * @param containerList - Container List.
-   */
-  public void register(String dbPath, Pipeline[] containerList) {
-    File dbFile = Paths.get(dbPath).toFile();
-    pipelineMap.put(dbPath, containerList);
-    checkExistingLog(AsyncBlockWriter.DIRTY_LOG_PREFIX, dbFile);
-    checkExistingLog(AsyncBlockWriter.RETRY_LOG_PREFIX, dbFile);
-  }
-
-  private String getDBFileName(String dbPath) {
-    return dbPath + ".db";
-  }
-
-  public LevelDBStore getCacheDB(String dbPath) throws IOException {
-    return openDB(dbPath);
-  }
-
-  public void releaseCacheDB(String dbPath) {
-    try {
-      closeDB(dbPath);
-    } catch (Exception e) {
-      metrics.incNumFailedReleaseLevelDB();
-      LOG.error("LevelDB close failed, dbPath:" + dbPath, e);
-    }
-  }
-  /**
-   * Close the DB if we don't have any outstanding references.
-   *
-   * @param dbPath - dbPath
-   * @throws IOException
-   */
-  public synchronized void closeDB(String dbPath) throws IOException {
-    if (dbMap.containsKey(dbPath)) {
-      RefCountedDB refDB = dbMap.get(dbPath);
-      int count = refDB.close();
-      if (count == 0) {
-        dbMap.remove(dbPath);
-      }
-    }
-  }
-
-  Pipeline getPipeline(String dbPath, long blockId) {
-    Pipeline[] containerList = pipelineMap.get(dbPath);
-    Preconditions.checkNotNull(containerList);
-    int containerIdx = (int) blockId % containerList.length;
-    long cBlockIndex =
-        Longs.fromByteArray(containerList[containerIdx].getData());
-    if (cBlockIndex > 0) {
-      // This catches the case when we get a wrong container in the ordering
-      // of the containers.
-      Preconditions.checkState(containerIdx % cBlockIndex == 0,
-          "The container ID computed should match with the container index " +
-              "returned from cBlock Server.");
-    }
-    return containerList[containerIdx];
-  }
-
-  public void incFinishCount(String fileName) {
-    if (!finishCountMap.containsKey(fileName)) {
-      LOG.error("No record for such file:" + fileName);
-      return;
-    }
-    finishCountMap.get(fileName).incCount();
-    if (finishCountMap.get(fileName).isFileDeleted()) {
-      finishCountMap.remove(fileName);
-    }
-  }
-
-  /**
-   * When an object implementing interface <code>Runnable</code> is used
-   * to create a thread, starting the thread causes the object's
-   * <code>run</code> method to be called in that separately executing
-   * thread.
-   * <p>
-   * The general contract of the method <code>run</code> is that it may
-   * take any action whatsoever.
-   *
-   * @see Thread#run()
-   */
-  @Override
-  public void run() {
-    while (!this.shutdown.get()) {
-      try {
-        Message message = messageQueue.take();
-        LOG.debug("Got message to process -- DB Path : {} , FileName; {}",
-            message.getDbPath(), message.getFileName());
-        String fullPath = Paths.get(message.getDbPath(),
-            message.getFileName()).toString();
-        String[] fileNameParts = message.getFileName().split("\\.");
-        Preconditions.checkState(fileNameParts.length > 1);
-        String fileType = fileNameParts[0];
-        boolean isDirtyLogFile =
-            fileType.equalsIgnoreCase(AsyncBlockWriter.DIRTY_LOG_PREFIX);
-        ReadableByteChannel fileChannel = new FileInputStream(fullPath)
-            .getChannel();
-        // TODO: We can batch and unique the IOs here. First getting the code
-        // to work, we will add those later.
-        int bytesRead = fileChannel.read(blockIDBuffer);
-        fileChannel.close();
-        LOG.debug("Read blockID log of size: {} position {} remaining {}",
-            bytesRead, blockIDBuffer.position(), blockIDBuffer.remaining());
-        // current position of in the buffer in bytes, divided by number of
-        // bytes per long (which is calculated by number of bits per long
-        // divided by number of bits per byte) gives the number of blocks
-        int blockCount = blockIDBuffer.position()/(Long.SIZE / Byte.SIZE);
-        if (isDirtyLogFile) {
-          getTargetMetrics().incNumBytesDirtyLogRead(bytesRead);
-        } else {
-          getTargetMetrics().incNumBytesRetryLogRead(bytesRead);
-        }
-        if (finishCountMap.containsKey(message.getFileName())) {
-          // In theory this should never happen. But if it happened,
-          // we need to know it...
-          getTargetMetrics().incNumIllegalDirtyLogFiles();
-          LOG.error("Adding DirtyLog file again {} current count {} new {}",
-              message.getFileName(),
-              finishCountMap.get(message.getFileName()).expectedCount,
-              blockCount);
-        }
-        finishCountMap.put(message.getFileName(),
-            new FinishCounter(blockCount, message.getDbPath(),
-                message.getFileName(), this));
-        // should be flip instead of rewind, because we also need to make sure
-        // the end position is correct.
-        blockIDBuffer.flip();
-        LOG.debug("Remaining blocks count {} and {}", blockIDBuffer.remaining(),
-            blockCount);
-        while (blockIDBuffer.remaining() >= (Long.SIZE / Byte.SIZE)) {
-          long blockID = blockIDBuffer.getLong();
-          int retryCount = 0;
-          if (isDirtyLogFile) {
-            getTargetMetrics().incNumDirtyLogBlockRead();
-          } else {
-            getTargetMetrics().incNumRetryLogBlockRead();
-            Preconditions.checkState(fileNameParts.length == 4);
-            retryCount = Integer.parseInt(fileNameParts[3]);
-          }
-          LogicalBlock block = new DiskBlock(blockID, null, false);
-          BlockWriterTask blockWriterTask = new BlockWriterTask(block, this,
-              message.getDbPath(), retryCount, message.getFileName(),
-              maxRetryCount);
-          threadPoolExecutor.submit(blockWriterTask);
-        }
-        blockIDBuffer.clear();
-      } catch (InterruptedException e) {
-        LOG.info("ContainerCacheFlusher is interrupted.", e);
-      } catch (FileNotFoundException e) {
-        LOG.error("Unable to find the dirty blocks file. This will cause " +
-            "data errors. Please stop using this volume.", e);
-      } catch (IOException e) {
-        LOG.error("Unable to read the dirty blocks file. This will cause " +
-            "data errors. Please stop using this volume.", e);
-      } catch (Exception e) {
-        LOG.error("Generic exception.", e);
-      }
-    }
-    LOG.info("Exiting flusher");
-  }
-
-  /**
-   * Tries to get the local host IP Address as trace prefix
-   * for creating trace IDs, otherwise uses a random UUID for it.
-   */
-  private static String getTracePrefix() {
-    String tmp;
-    try {
-      tmp = InetAddress.getLocalHost().getHostAddress();
-    } catch (UnknownHostException ex) {
-      tmp = UUID.randomUUID().toString();
-      LOG.error("Unable to read the host address. Using a GUID for " +
-          "hostname:{} ", tmp, ex);
-    }
-    return tmp;
-  }
-
-  /**
-   * We create a trace ID to make it easy to debug issues.
-   * A trace ID is in IPAddress:UserName:VolumeName:blockID:second format.
-   *
-   * This will get written down on the data node if we get any failures, so
-   * with this trace ID we can correlate cBlock failures across machines.
-   *
-   * @param blockID - Block ID
-   * @return trace ID
-   */
-  public String getTraceID(File dbPath, long blockID) {
-    String volumeName = dbPath.getName();
-    String userName = dbPath.getParentFile().getName();
-    // mapping to seconds to make the string smaller.
-    return tracePrefix + ":" + userName + ":" + volumeName
-        + ":" + blockID + ":" + Time.monotonicNow() / 1000;
-  }
-
-  /**
-   * Keeps a Reference counted DB that we close only when the total Reference
-   * has gone to zero.
-   */
-  private static class RefCountedDB {
-    private LevelDBStore db;
-    private AtomicInteger refcount;
-    private String dbPath;
-
-    /**
-     * RefCountedDB DB ctor.
-     *
-     * @param dbPath - DB path.
-     * @param db - LevelDBStore db
-     */
-    RefCountedDB(String dbPath, LevelDBStore db) {
-      this.db = db;
-      this.refcount = new AtomicInteger(1);
-      this.dbPath = dbPath;
-    }
-
-    /**
-     * close the DB if possible.
-     */
-    public int close() throws IOException {
-      int count = this.refcount.decrementAndGet();
-      if (count == 0) {
-        LOG.info("Closing the LevelDB. {} ", this.dbPath);
-        db.close();
-      }
-      return count;
-    }
-
-    public void open() {
-      this.refcount.incrementAndGet();
-    }
-  }
-
-  /**
-   * The message held in processing queue.
-   */
-  private static class Message {
-    private String dbPath;
-    private String fileName;
-
-    /**
-     * A message that holds the info about which path dirty blocks log and
-     * which path contains db.
-     *
-     * @param dbPath
-     * @param fileName
-     */
-    Message(String dbPath, String fileName) {
-      this.dbPath = dbPath;
-      this.fileName = fileName;
-    }
-
-    public String getDbPath() {
-      return dbPath;
-    }
-
-    public void setDbPath(String dbPath) {
-      this.dbPath = dbPath;
-    }
-
-    public String getFileName() {
-      return fileName;
-    }
-
-    public void setFileName(String fileName) {
-      this.fileName = fileName;
-    }
-  }
-
-  private static class FinishCounter {
-    private final long expectedCount;
-    private final String dbPath;
-    private final String dirtyLogPath;
-    private final AtomicLong currentCount;
-    private AtomicBoolean fileDeleted;
-    private final ContainerCacheFlusher flusher;
-
-    FinishCounter(long expectedCount, String dbPath,
-        String dirtyLogPath, ContainerCacheFlusher flusher) throws IOException {
-      this.expectedCount = expectedCount;
-      this.dbPath = dbPath;
-      this.dirtyLogPath = dirtyLogPath;
-      this.currentCount = new AtomicLong(0);
-      this.fileDeleted = new AtomicBoolean(false);
-      this.flusher = flusher;
-    }
-
-    public boolean isFileDeleted() {
-      return fileDeleted.get();
-    }
-
-    public void incCount() {
-      long count = this.currentCount.incrementAndGet();
-      if (count >= expectedCount) {
-        String filePath = String.format("%s/%s", dbPath, dirtyLogPath);
-        LOG.debug(
-            "Deleting {} with count {} {}", filePath, count, expectedCount);
-        try {
-          Path path = Paths.get(filePath);
-          Files.delete(path);
-          // the following part tries to remove the directory if it is empty
-          // but not sufficient, because the .db directory still exists....
-          // TODO how to handle the .db directory?
-          /*Path parent = path.getParent();
-          if (parent.toFile().listFiles().length == 0) {
-            Files.delete(parent);
-          }*/
-          fileDeleted.set(true);
-        } catch (Exception e) {
-          flusher.getTargetMetrics().incNumFailedDirtyLogFileDeletes();
-          LOG.error("Error deleting dirty log file:" + filePath, e);
-        }
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java
deleted file mode 100644
index ce7717b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_ISCSI_ADVERTISED_IP;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_ISCSI_ADVERTISED_PORT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT;
-import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB;
-import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.scm.client.ContainerOperationClient;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.jscsi.target.Configuration;
-
-import java.net.InetSocketAddress;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_PORT_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_PORT_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY;
-
-/**
- * This class runs the target server process.
- */
-public final class SCSITargetDaemon {
-  public static void main(String[] args) throws Exception {
-    OzoneConfiguration ozoneConf = new OzoneConfiguration();
-
-    RPC.setProtocolEngine(ozoneConf, CBlockClientServerProtocolPB.class,
-        ProtobufRpcEngine.class);
-    long containerSizeGB = ozoneConf.getInt(DFS_CBLOCK_CONTAINER_SIZE_GB_KEY,
-        DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT);
-    ContainerOperationClient.setContainerSizeB(
-        containerSizeGB * OzoneConsts.GB);
-    String jscsiServerAddress = ozoneConf.get(
-        DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY,
-        DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT);
-    String cbmIPAddress = ozoneConf.get(
-        DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY,
-        DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT
-    );
-    int cbmPort = ozoneConf.getInt(
-        DFS_CBLOCK_JSCSI_PORT_KEY,
-        DFS_CBLOCK_JSCSI_PORT_DEFAULT
-    );
-
-    String scmAddress = ozoneConf.get(OZONE_SCM_CLIENT_BIND_HOST_KEY,
-        OZONE_SCM_CLIENT_BIND_HOST_DEFAULT);
-    int scmClientPort = ozoneConf.getInt(OZONE_SCM_CLIENT_PORT_KEY,
-        OZONE_SCM_CLIENT_PORT_DEFAULT);
-    int scmDatanodePort = ozoneConf.getInt(OZONE_SCM_DATANODE_PORT_KEY,
-        OZONE_SCM_DATANODE_PORT_DEFAULT);
-
-    String scmClientAddress = scmAddress + ":" + scmClientPort;
-    String scmDataodeAddress = scmAddress + ":" + scmDatanodePort;
-
-    ozoneConf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scmClientAddress);
-    ozoneConf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, scmDataodeAddress);
-
-    InetSocketAddress cbmAddress = new InetSocketAddress(
-        cbmIPAddress, cbmPort);
-    long version = RPC.getProtocolVersion(
-        CBlockServiceProtocolPB.class);
-    CBlockClientProtocolClientSideTranslatorPB cbmClient =
-        new CBlockClientProtocolClientSideTranslatorPB(
-            RPC.getProxy(CBlockClientServerProtocolPB.class, version,
-                cbmAddress, UserGroupInformation.getCurrentUser(), ozoneConf,
-                NetUtils.getDefaultSocketFactory(ozoneConf), 5000)
-        );
-    CBlockManagerHandler cbmHandler = new CBlockManagerHandler(cbmClient);
-
-    String advertisedAddress = ozoneConf.
-        getTrimmed(DFS_CBLOCK_ISCSI_ADVERTISED_IP, jscsiServerAddress);
-
-    int advertisedPort = ozoneConf.
-        getInt(DFS_CBLOCK_ISCSI_ADVERTISED_PORT,
-            DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT);
-
-    Configuration jscsiConfig =
-        new Configuration(jscsiServerAddress,
-            advertisedAddress,
-            advertisedPort);
-    DefaultMetricsSystem.initialize("CBlockMetrics");
-    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
-    CBlockTargetServer targetServer = new CBlockTargetServer(
-        ozoneConf, jscsiConfig, cbmHandler, metrics);
-
-    targetServer.call();
-  }
-
-  private SCSITargetDaemon() {
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java
deleted file mode 100644
index 300b2ae..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper.cache;
-
-import java.io.IOException;
-
-/**
- * Defines the interface for cache implementations. The cache will be called
- * by cblock storage module when it performs IO operations.
- */
-public interface CacheModule {
-  /**
-   * check if the key is cached, if yes, returned the cached object.
-   * otherwise, load from data source. Then put it into cache.
-   *
-   * @param blockID
-   * @return the target block.
-   */
-  LogicalBlock get(long blockID) throws IOException;
-
-  /**
-   * put the value of the key into cache.
-   * @param blockID
-   * @param value
-   */
-  void put(long blockID, byte[] value) throws IOException;
-
-  void flush() throws IOException;
-
-  void start() throws IOException;
-
-  void stop() throws IOException;
-
-  void close() throws IOException;
-
-  boolean isDirtyCache();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java
deleted file mode 100644
index 470826f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper.cache;
-
-import java.nio.ByteBuffer;
-
-/**
- * Logical Block is the data structure that we write to the cache,
- * the key and data gets written to remote contianers. Rest is used for
- * book keeping for the cache.
- */
-public interface LogicalBlock {
-  /**
-   * Returns the data stream of this block.
-   * @return - ByteBuffer
-   */
-  ByteBuffer getData();
-
-  /**
-   * Frees the byte buffer since we don't need it any more.
-   */
-  void clearData();
-
-  /**
-   * Returns the Block ID for this Block.
-   * @return long - BlockID
-   */
-  long getBlockID();
-
-  /**
-   * Flag that tells us if this block has been persisted to container.
-   * @return whether this block is now persistent
-   */
-  boolean isPersisted();
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java
deleted file mode 100644
index 992578f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
-
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.Longs;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.LevelDBStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-/**
- * A Queue that is used to write blocks asynchronously to the container.
- */
-public class AsyncBlockWriter {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(AsyncBlockWriter.class);
-
-  /**
-   * XceiverClientManager is used to get client connections to a set of
-   * machines.
-   */
-  private final XceiverClientManager xceiverClientManager;
-
-  /**
-   * This lock is used as a signal to re-queuing thread. The requeue thread
-   * wakes up as soon as it is signaled some blocks are in the retry queue.
-   * We try really aggressively since this new block will automatically move
-   * to the end of the queue.
-   * <p>
-   * In the event a container is unavailable for a long time, we can either
-   * fail all writes or remap and let the writes succeed. The easier
-   * semantics is to fail the volume until the container is recovered by SCM.
-   */
-  private final Lock lock;
-  private final Condition notEmpty;
-  /**
-   * The cache this writer is operating against.
-   */
-  private final CBlockLocalCache parentCache;
-  private final BlockBufferManager blockBufferManager;
-  public final static String DIRTY_LOG_PREFIX = "DirtyLog";
-  public static final String RETRY_LOG_PREFIX = "RetryLog";
-  private AtomicLong localIoCount;
-
-  /**
-   * Constructs an Async Block Writer.
-   *
-   * @param config - Config
-   * @param cache - Parent Cache for this writer
-   */
-  public AsyncBlockWriter(Configuration config, CBlockLocalCache cache) {
-
-    Preconditions.checkNotNull(cache, "Cache cannot be null.");
-    Preconditions.checkNotNull(cache.getCacheDB(), "DB cannot be null.");
-    localIoCount = new AtomicLong();
-    lock = new ReentrantLock();
-    notEmpty = lock.newCondition();
-    parentCache = cache;
-    xceiverClientManager = cache.getClientManager();
-    blockBufferManager = new BlockBufferManager(config, parentCache);
-  }
-
-  public void start() throws IOException {
-    File logDir = new File(parentCache.getDbPath().toString());
-    if (!logDir.exists() && !logDir.mkdirs()) {
-      LOG.error("Unable to create the log directory, Critical error cannot " +
-          "continue. Log Dir : {}", logDir);
-      throw new IllegalStateException("Cache Directory create failed, Cannot " +
-          "continue. Log Dir: {}" + logDir);
-    }
-    blockBufferManager.start();
-  }
-
-  /**
-   * Return the log to write to.
-   *
-   * @return Logger.
-   */
-  public static Logger getLOG() {
-    return LOG;
-  }
-
-  /**
-   * Get the CacheDB.
-   *
-   * @return LevelDB Handle
-   */
-  LevelDBStore getCacheDB() {
-    return parentCache.getCacheDB();
-  }
-
-  /**
-   * Returns the client manager.
-   *
-   * @return XceiverClientManager
-   */
-  XceiverClientManager getXceiverClientManager() {
-    return xceiverClientManager;
-  }
-
-  /**
-   * Incs the localIoPacket Count that has gone into this device.
-   */
-  public long incrementLocalIO() {
-    return localIoCount.incrementAndGet();
-  }
-
-  /**
-   * Return the local io counts to this device.
-   * @return the count of io
-   */
-  public long getLocalIOCount() {
-    return localIoCount.get();
-  }
-
-  /**
-   * Writes a block to LevelDB store and queues a work item for the system to
-   * sync the block to containers.
-   *
-   * @param block - Logical Block
-   */
-  public void writeBlock(LogicalBlock block) throws IOException {
-    byte[] keybuf = Longs.toByteArray(block.getBlockID());
-    String traceID = parentCache.getTraceID(block.getBlockID());
-    if (parentCache.isShortCircuitIOEnabled()) {
-      long startTime = Time.monotonicNow();
-      getCacheDB().put(keybuf, block.getData().array());
-      incrementLocalIO();
-      long endTime = Time.monotonicNow();
-      parentCache.getTargetMetrics().updateDBWriteLatency(
-                                                    endTime - startTime);
-      if (parentCache.isTraceEnabled()) {
-        String datahash = DigestUtils.sha256Hex(block.getData().array());
-        parentCache.getTracer().info(
-            "Task=WriterTaskDBPut,BlockID={},Time={},SHA={}",
-            block.getBlockID(), endTime - startTime, datahash);
-      }
-      block.clearData();
-      blockBufferManager.addToBlockBuffer(block.getBlockID());
-    } else {
-      Pipeline pipeline = parentCache.getPipeline(block.getBlockID());
-      String containerName = pipeline.getContainerName();
-      XceiverClientSpi client = null;
-      try {
-        long startTime = Time.monotonicNow();
-        client = parentCache.getClientManager()
-            .acquireClient(parentCache.getPipeline(block.getBlockID()));
-        ContainerProtocolCalls.writeSmallFile(client, containerName,
-            Long.toString(block.getBlockID()), block.getData().array(),
-            traceID);
-        long endTime = Time.monotonicNow();
-        if (parentCache.isTraceEnabled()) {
-          String datahash = DigestUtils.sha256Hex(block.getData().array());
-          parentCache.getTracer().info(
-              "Task=DirectWriterPut,BlockID={},Time={},SHA={}",
-              block.getBlockID(), endTime - startTime, datahash);
-        }
-        parentCache.getTargetMetrics().
-            updateDirectBlockWriteLatency(endTime - startTime);
-        parentCache.getTargetMetrics().incNumDirectBlockWrites();
-      } catch (Exception ex) {
-        parentCache.getTargetMetrics().incNumFailedDirectBlockWrites();
-        LOG.error("Direct I/O writing of block:{} traceID:{} to "
-            + "container {} failed", block.getBlockID(), traceID,
-            containerName, ex);
-        throw ex;
-      } finally {
-        if (client != null) {
-          parentCache.getClientManager().releaseClient(client);
-        }
-        block.clearData();
-      }
-    }
-  }
-
-  /**
-   * Shutdown by writing any pending I/O to dirtylog buffer.
-   */
-  public void shutdown() {
-    blockBufferManager.shutdown();
-  }
-  /**
-   * Returns tracer.
-   *
-   * @return Tracer
-   */
-  Logger getTracer() {
-    return parentCache.getTracer();
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java
deleted file mode 100644
index c61a7a4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
-
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.Paths;
-
-/**
- * This task is responsible for flushing the BlockIDBuffer
- * to Dirty Log File. This Dirty Log file is used later by
- * ContainerCacheFlusher when the data is written to container
- */
-public class BlockBufferFlushTask implements Runnable {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(BlockBufferFlushTask.class);
-  private final CBlockLocalCache parentCache;
-  private final BlockBufferManager bufferManager;
-  private final ByteBuffer blockIDBuffer;
-
-  BlockBufferFlushTask(ByteBuffer blockIDBuffer, CBlockLocalCache parentCache,
-                       BlockBufferManager manager) {
-    this.parentCache = parentCache;
-    this.bufferManager = manager;
-    this.blockIDBuffer = blockIDBuffer;
-  }
-
-  /**
-   * When an object implementing interface <code>Runnable</code> is used
-   * to create a thread, starting the thread causes the object's
-   * <code>run</code> method to be called in that separately executing
-   * thread.
-   * <p>
-   * The general contract of the method <code>run</code> is that it may
-   * take any action whatsoever.
-   *
-   * @see Thread#run()
-   */
-  @Override
-  public void run() {
-    try {
-      writeBlockBufferToFile(blockIDBuffer);
-    } catch (Exception e) {
-      parentCache.getTargetMetrics().incNumFailedBlockBufferFlushes();
-      LOG.error("Unable to sync the Block map to disk with "
-          + (blockIDBuffer.position() / Long.SIZE) + "entries "
-          + "-- NOTE: This might cause a data loss or corruption", e);
-    } finally {
-      bufferManager.releaseBuffer(blockIDBuffer);
-    }
-  }
-
-  /**
-   * Write Block Buffer to file.
-   *
-   * @param buffer - ByteBuffer
-   * @throws IOException
-   */
-  private void writeBlockBufferToFile(ByteBuffer buffer)
-      throws IOException {
-    long startTime = Time.monotonicNow();
-    boolean append = false;
-
-    // If there is nothing written to blockId buffer,
-    // then skip flushing of blockId buffer
-    if (buffer.position() == 0) {
-      return;
-    }
-
-    buffer.flip();
-    String fileName =
-        String.format("%s.%s", AsyncBlockWriter.DIRTY_LOG_PREFIX,
-                      Time.monotonicNow());
-    String log = Paths.get(parentCache.getDbPath().toString(), fileName)
-        .toString();
-
-    FileChannel channel = new FileOutputStream(log, append).getChannel();
-    int bytesWritten = channel.write(buffer);
-    channel.close();
-    buffer.clear();
-    parentCache.processDirtyMessage(fileName);
-    long endTime = Time.monotonicNow();
-    if (parentCache.isTraceEnabled()) {
-      parentCache.getTracer().info(
-          "Task=DirtyBlockLogWrite,Time={} bytesWritten={}",
-          endTime - startTime, bytesWritten);
-    }
-
-    parentCache.getTargetMetrics().incNumBlockBufferFlushCompleted();
-    parentCache.getTargetMetrics().incNumBytesDirtyLogWritten(bytesWritten);
-    parentCache.getTargetMetrics().
-        updateBlockBufferFlushLatency(endTime - startTime);
-    LOG.debug("Block buffer writer bytesWritten:{} Time:{}",
-        bytesWritten, endTime - startTime);
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java
deleted file mode 100644
index 5d3209c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.nio.ByteBuffer;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.ThreadFactory;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
-    DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
-    DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
-    DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
-    DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
-    DFS_CBLOCK_CACHE_KEEP_ALIVE;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
-    DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
-    DFS_CBLOCK_CACHE_THREAD_PRIORITY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
-    DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT;
-
-/**
- * This class manages the block ID buffer.
- * Block ID Buffer keeps a list of blocks which are in leveldb cache
- * This buffer is used later when the blocks are flushed to container
- *
- * Two blockIDBuffers are maintained so that write are not blocked when
- * DirtyLog is being written. Once a blockIDBuffer is full, it will be
- * enqueued for DirtyLog write while the other buffer accepts new write.
- * Once the DirtyLog write is done, the buffer is returned back to the pool.
- *
- * There are three triggers for blockIDBuffer flush
- * 1) BlockIDBuffer is full,
- * 2) Time period defined for blockIDBuffer flush has elapsed.
- * 3) Shutdown
- */
-public class BlockBufferManager {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(BlockBufferManager.class);
-
-  private enum FlushReason {
-    BUFFER_FULL,
-    SHUTDOWN,
-    TIMER
-  };
-
-  private final int blockBufferSize;
-  private final CBlockLocalCache parentCache;
-  private final ScheduledThreadPoolExecutor scheduledExecutor;
-  private final ThreadPoolExecutor threadPoolExecutor;
-  private final long intervalSeconds;
-  private final ArrayBlockingQueue<ByteBuffer> acquireQueue;
-  private final ArrayBlockingQueue<Runnable> workQueue;
-  private ByteBuffer currentBuffer;
-
-  BlockBufferManager(Configuration config, CBlockLocalCache parentCache) {
-    this.parentCache = parentCache;
-    this.scheduledExecutor = new ScheduledThreadPoolExecutor(1);
-
-    this.intervalSeconds =
-        config.getTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL,
-            DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL_DEFAULT,
-            TimeUnit.SECONDS);
-
-    long keepAlive = config.getTimeDuration(DFS_CBLOCK_CACHE_KEEP_ALIVE,
-        DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT,
-        TimeUnit.SECONDS);
-    this.workQueue = new ArrayBlockingQueue<>(2, true);
-    int threadPri = config.getInt(DFS_CBLOCK_CACHE_THREAD_PRIORITY,
-        DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT);
-    ThreadFactory workerThreadFactory = new ThreadFactoryBuilder()
-        .setNameFormat("Cache Block Buffer Manager Thread #%d")
-        .setDaemon(true)
-        .setPriority(threadPri)
-        .build();
-    /*
-     * starting a thread pool with core pool size of 1 and maximum of 2 threads
-     * as there are maximum of 2 buffers which can be flushed at the same time.
-     */
-    this.threadPoolExecutor = new ThreadPoolExecutor(1, 2,
-        keepAlive, TimeUnit.SECONDS, workQueue, workerThreadFactory,
-        new ThreadPoolExecutor.AbortPolicy());
-
-    this.blockBufferSize = config.getInt(DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE,
-        DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT) * (Long.SIZE / Byte.SIZE);
-    this.acquireQueue = new ArrayBlockingQueue<>(2, true);
-
-    for (int i = 0; i < 2; i++) {
-      acquireQueue.add(ByteBuffer.allocate(blockBufferSize));
-    }
-    // get the first buffer to be used
-    this.currentBuffer = acquireQueue.remove();
-
-    LOG.info("BufferManager: Buffer Size:{} FlushIntervalSeconds:{}",
-        blockBufferSize, intervalSeconds);
-  }
-
-  // triggerBlockBufferFlush enqueues current ByteBuffer for flush and returns.
-  // This enqueue is asynchronous and hence triggerBlockBufferFlush will
-  // only block when there are no available buffers in acquireQueue
-  // Once the DirtyLog write is done, buffer is returned back to
-  // BlockBufferManager using releaseBuffer
-  private synchronized void triggerBlockBufferFlush(FlushReason reason) {
-    LOG.debug("Flush triggered because: " + reason.toString() +
-        " Num entries in buffer: " +
-        currentBuffer.position() / (Long.SIZE / Byte.SIZE) +
-        " Acquire Queue Size: " + acquireQueue.size());
-
-    parentCache.getTargetMetrics().incNumBlockBufferFlushTriggered();
-    BlockBufferFlushTask flushTask =
-        new BlockBufferFlushTask(currentBuffer, parentCache, this);
-    threadPoolExecutor.submit(flushTask);
-    try {
-      currentBuffer = acquireQueue.take();
-    } catch (InterruptedException ex) {
-      currentBuffer = null;
-      parentCache.getTargetMetrics().incNumInterruptedBufferWaits();
-      LOG.error("wait on take operation on acquire queue interrupted", ex);
-      Thread.currentThread().interrupt();
-    }
-  }
-
-  public synchronized void addToBlockBuffer(long blockId)  {
-    parentCache.getTargetMetrics().incNumBlockBufferUpdates();
-    currentBuffer.putLong(blockId);
-    // if no space left, flush this buffer
-    if (currentBuffer.remaining() == 0) {
-      triggerBlockBufferFlush(FlushReason.BUFFER_FULL);
-    }
-  }
-
-  public void releaseBuffer(ByteBuffer buffer) {
-    if (buffer.position() != 0) {
-      LOG.error("requeuing a non empty buffer with:{}",
-          "elements enqueued in the acquire queue",
-          buffer.position() / (Long.SIZE / Byte.SIZE));
-      buffer.reset();
-    }
-    // There should always be space in the queue to add an element
-    acquireQueue.add(buffer);
-  }
-
-  // Start a scheduled task to flush blockIDBuffer
-  public void start() {
-    Runnable scheduledTask = () -> triggerBlockBufferFlush(FlushReason.TIMER);
-    scheduledExecutor.scheduleWithFixedDelay(scheduledTask, intervalSeconds,
-                                        intervalSeconds, TimeUnit.SECONDS);
-    threadPoolExecutor.prestartAllCoreThreads();
-  }
-
-  public void shutdown() {
-    triggerBlockBufferFlush(FlushReason.SHUTDOWN);
-    scheduledExecutor.shutdown();
-    threadPoolExecutor.shutdown();
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java
deleted file mode 100644
index 1149164..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java
+++ /dev/null
@@ -1,577 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
-
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.Longs;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher;
-import org.apache.hadoop.cblock.jscsiHelper.cache.CacheModule;
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics;
-import org.apache.hadoop.utils.LevelDBStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.FileStore;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.List;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_DISK_CACHE_PATH_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_TRACE_IO;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_TRACE_IO_DEFAULT;
-
-/**
- * A local cache used by the CBlock ISCSI server. This class is enabled or
- * disabled via config settings.
- */
-public class CBlockLocalCache implements CacheModule {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(CBlockLocalCache.class);
-  private static final Logger TRACER =
-      LoggerFactory.getLogger("TraceIO");
-
-  private final Configuration conf;
-  /**
-   * LevelDB cache file.
-   */
-  private final LevelDBStore cacheDB;
-
-  /**
-   * AsyncBlock writer updates the cacheDB and writes the blocks async to
-   * remote containers.
-   */
-  private final AsyncBlockWriter blockWriter;
-
-  /**
-   * Sync block reader tries to read from the cache and if we get a cache
-   * miss we will fetch the block from remote location. It will asynchronously
-   * update the cacheDB.
-   */
-  private final SyncBlockReader blockReader;
-  private final String userName;
-  private final String volumeName;
-
-  /**
-   * From a block ID we are able to get the pipeline by indexing this array.
-   */
-  private final Pipeline[] containerList;
-  private final int blockSize;
-  private XceiverClientManager clientManager;
-  /**
-   * If this flag is enabled then cache traces all I/O, all reads and writes
-   * are visible in the log with sha of the block written. Makes the system
-   * slower use it only for debugging or creating trace simulations.
-   */
-  private final boolean traceEnabled;
-  private final boolean enableShortCircuitIO;
-  private final long volumeSize;
-  private long currentCacheSize;
-  private File dbPath;
-  private final ContainerCacheFlusher flusher;
-  private CBlockTargetMetrics cblockTargetMetrics;
-
-  /**
-   * Get Db Path.
-   * @return the file instance of the db.
-   */
-  public File getDbPath() {
-    return dbPath;
-  }
-
-  /**
-   * Constructor for CBlockLocalCache invoked via the builder.
-   *
-   * @param conf -  Configuration
-   * @param volumeName - volume Name
-   * @param userName - user name
-   * @param containerPipelines - Pipelines that make up this contianer
-   * @param blockSize - blockSize
-   * @param flusher - flusher to flush data to container
-   * @throws IOException
-   */
-  CBlockLocalCache(
-      Configuration conf, String volumeName,
-      String userName, List<Pipeline> containerPipelines, int blockSize,
-      long volumeSize, ContainerCacheFlusher flusher) throws IOException {
-    this.conf = conf;
-    this.userName = userName;
-    this.volumeName = volumeName;
-    this.blockSize = blockSize;
-    this.flusher = flusher;
-    this.traceEnabled = conf.getBoolean(DFS_CBLOCK_TRACE_IO,
-        DFS_CBLOCK_TRACE_IO_DEFAULT);
-    this.enableShortCircuitIO = conf.getBoolean(
-        DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO,
-        DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO_DEFAULT);
-    dbPath = Paths.get(conf.get(DFS_CBLOCK_DISK_CACHE_PATH_KEY,
-        DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT), userName, volumeName).toFile();
-
-    if (!dbPath.exists() && !dbPath.mkdirs()) {
-      LOG.error("Unable to create the cache paths. Path: {}", dbPath);
-      throw new IllegalArgumentException("Unable to create paths. Path: " +
-          dbPath);
-    }
-    cacheDB = flusher.getCacheDB(dbPath.toString());
-    this.containerList = containerPipelines.toArray(new
-        Pipeline[containerPipelines.size()]);
-    this.volumeSize = volumeSize;
-
-    blockWriter = new AsyncBlockWriter(conf, this);
-    blockReader = new SyncBlockReader(conf, this);
-    if (this.traceEnabled) {
-      getTracer().info("Task=StartingCache");
-    }
-  }
-
-  private void setClientManager(XceiverClientManager manager) {
-    this.clientManager = manager;
-  }
-
-  private void setCblockTargetMetrics(CBlockTargetMetrics targetMetrics) {
-    this.cblockTargetMetrics = targetMetrics;
-  }
-
-  /**
-   * Returns new builder class that builds a CBlockLocalCache.
-   *
-   * @return Builder
-   */
-  public static Builder newBuilder() {
-    return new Builder();
-  }
-
-  public void processDirtyMessage(String fileName) {
-    flusher.processDirtyBlocks(dbPath.toString(), fileName);
-  }
-
-  /**
-   * Get usable disk space.
-   *
-   * @param dbPathString - Path to db
-   * @return long bytes remaining.
-   */
-  private static long getRemainingDiskSpace(String dbPathString) {
-    try {
-      URI fileUri = new URI("file:///");
-      Path dbPath = Paths.get(fileUri).resolve(dbPathString);
-      FileStore disk = Files.getFileStore(dbPath);
-      return disk.getUsableSpace();
-    } catch (URISyntaxException | IOException ex) {
-      LOG.error("Unable to get free space on for path :" + dbPathString);
-    }
-    return 0L;
-  }
-
-  /**
-   * Returns the Max current CacheSize.
-   *
-   * @return - Cache Size
-   */
-  public long getCurrentCacheSize() {
-    return currentCacheSize;
-  }
-
-  /**
-   * Sets the Maximum Cache Size.
-   *
-   * @param currentCacheSize - Max current Cache Size.
-   */
-  public void setCurrentCacheSize(long currentCacheSize) {
-    this.currentCacheSize = currentCacheSize;
-  }
-
-  /**
-   * True if block tracing is enabled.
-   *
-   * @return - bool
-   */
-  public boolean isTraceEnabled() {
-    return traceEnabled;
-  }
-
-  /**
-   * Checks if Short Circuit I/O is enabled.
-   *
-   * @return - true if it is enabled.
-   */
-  public boolean isShortCircuitIOEnabled() {
-    return enableShortCircuitIO;
-  }
-
-  /**
-   * Returns the default block size of this device.
-   *
-   * @return - int
-   */
-  public int getBlockSize() {
-    return blockSize;
-  }
-
-  /**
-   * Gets the client manager.
-   *
-   * @return XceiverClientManager
-   */
-  public XceiverClientManager getClientManager() {
-    return clientManager;
-  }
-
-  /**
-   * check if the key is cached, if yes, returned the cached object.
-   * otherwise, load from data source. Then put it into cache.
-   *
-   * @param blockID
-   * @return the block associated to the blockID
-   */
-  @Override
-  public LogicalBlock get(long blockID) throws IOException {
-    cblockTargetMetrics.incNumReadOps();
-    return blockReader.readBlock(blockID);
-  }
-
-  /**
-   * put the value of the key into cache and remote container.
-   *
-   * @param blockID - BlockID
-   * @param data - byte[]
-   */
-  @Override
-  public void put(long blockID, byte[] data) throws IOException {
-    cblockTargetMetrics.incNumWriteOps();
-    LogicalBlock block = new DiskBlock(blockID, data, false);
-    blockWriter.writeBlock(block);
-  }
-
-  @Override
-  public void flush() throws IOException {
-
-  }
-
-  @Override
-  public void start() throws IOException {
-    flusher.register(getDbPath().getPath(), containerList);
-    blockWriter.start();
-  }
-
-  @Override
-  public void stop() throws IOException {
-  }
-
-  @Override
-  public void close() throws IOException {
-    blockReader.shutdown();
-    blockWriter.shutdown();
-    this.flusher.releaseCacheDB(dbPath.toString());
-    if (this.traceEnabled) {
-      getTracer().info("Task=ShutdownCache");
-    }
-  }
-
-  /**
-   * Returns true if cache still has blocks pending to write.
-   *
-   * @return false if we have no pending blocks to write.
-   */
-  @Override
-  public boolean isDirtyCache() {
-    return false;
-  }
-
-  /**
-   * Returns the local cache DB.
-   *
-   * @return - DB
-   */
-  LevelDBStore getCacheDB() {
-    return this.cacheDB;
-  }
-
-  /**
-   * Returns the current userName.
-   *
-   * @return - UserName
-   */
-  String getUserName() {
-    return this.userName;
-  }
-
-  /**
-   * Returns the volume name.
-   *
-   * @return VolumeName.
-   */
-  String getVolumeName() {
-    return this.volumeName;
-  }
-
-  /**
-   * Returns the target metrics.
-   *
-   * @return CBlock Target Metrics.
-   */
-  CBlockTargetMetrics getTargetMetrics() {
-    return this.cblockTargetMetrics;
-  }
-
-  /**
-   * Returns the pipeline to use given a container.
-   *
-   * @param blockId - blockID
-   * @return - pipeline.
-   */
-  Pipeline getPipeline(long blockId) {
-    int containerIdx = (int) blockId % containerList.length;
-    long cBlockIndex =
-        Longs.fromByteArray(containerList[containerIdx].getData());
-    if (cBlockIndex > 0) {
-      // This catches the case when we get a wrong container in the ordering
-      // of the containers.
-      Preconditions.checkState(containerIdx % cBlockIndex == 0,
-          "The container ID computed should match with the container index " +
-              "returned from cBlock Server.");
-    }
-    return containerList[containerIdx];
-  }
-
-  String getTraceID(long blockID) {
-    return flusher.getTraceID(dbPath, blockID);
-  }
-
-  /**
-   * Returns tracer.
-   *
-   * @return - Logger
-   */
-  Logger getTracer() {
-    return TRACER;
-  }
-
-  /**
-   * Builder class for CBlocklocalCache.
-   */
-  public static class Builder {
-    private Configuration configuration;
-    private String userName;
-    private String volumeName;
-    private List<Pipeline> pipelines;
-    private XceiverClientManager clientManager;
-    private int blockSize;
-    private long volumeSize;
-    private ContainerCacheFlusher flusher;
-    private CBlockTargetMetrics metrics;
-
-    /**
-     * Ctor.
-     */
-    Builder() {
-    }
-
-    /**
-     * Computes a cache size based on the configuration and available disk
-     * space.
-     *
-     * @param configuration - Config
-     * @param volumeSize - Size of Volume
-     * @param blockSize - Size of the block
-     * @return - cache size in bytes.
-     */
-    private static long computeCacheSize(Configuration configuration,
-        long volumeSize, int blockSize) {
-      long cacheSize = 0;
-      String dbPath = configuration.get(DFS_CBLOCK_DISK_CACHE_PATH_KEY,
-          DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT);
-      if (StringUtils.isBlank(dbPath)) {
-        return cacheSize;
-      }
-      long spaceRemaining = getRemainingDiskSpace(dbPath);
-      double cacheRatio = 1.0;
-
-      if (spaceRemaining < volumeSize) {
-        cacheRatio = (double)spaceRemaining / volumeSize;
-      }
-
-      // if cache is going to be at least 10% of the volume size it is worth
-      // doing, otherwise skip creating the  cache.
-      if (cacheRatio >= 0.10) {
-        cacheSize = Double.doubleToLongBits(volumeSize * cacheRatio);
-      }
-      return cacheSize;
-    }
-
-    /**
-     * Sets the Config to be used by this cache.
-     *
-     * @param conf - Config
-     * @return Builder
-     */
-    public Builder setConfiguration(Configuration conf) {
-      this.configuration = conf;
-      return this;
-    }
-
-    /**
-     * Sets the user name who is the owner of this volume.
-     *
-     * @param user - name of the owner, please note this is not the current
-     * user name.
-     * @return - Builder
-     */
-    public Builder setUserName(String user) {
-      this.userName = user;
-      return this;
-    }
-
-    /**
-     * Sets the VolumeName.
-     *
-     * @param volume - Name of the volume
-     * @return Builder
-     */
-    public Builder setVolumeName(String volume) {
-      this.volumeName = volume;
-      return this;
-    }
-
-    /**
-     * Sets the Pipelines that form this volume.
-     *
-     * @param pipelineList - list of pipelines
-     * @return Builder
-     */
-    public Builder setPipelines(List<Pipeline> pipelineList) {
-      this.pipelines = pipelineList;
-      return this;
-    }
-
-    /**
-     * Sets the Client Manager that manages the communication with containers.
-     *
-     * @param xceiverClientManager - clientManager.
-     * @return - Builder
-     */
-    public Builder setClientManager(XceiverClientManager xceiverClientManager) {
-      this.clientManager = xceiverClientManager;
-      return this;
-    }
-
-    /**
-     * Sets the block size -- Typical sizes are 4KB, 8KB etc.
-     *
-     * @param size - BlockSize.
-     * @return - Builder
-     */
-    public Builder setBlockSize(int size) {
-      this.blockSize = size;
-      return this;
-    }
-
-    /**
-     * Sets the volumeSize.
-     *
-     * @param size - VolumeSize
-     * @return - Builder
-     */
-    public Builder setVolumeSize(long size) {
-      this.volumeSize = size;
-      return this;
-    }
-
-    /**
-     * Set flusher.
-     * @param containerCacheFlusher - cache Flusher
-     * @return Builder.
-     */
-    public Builder setFlusher(ContainerCacheFlusher containerCacheFlusher) {
-      this.flusher = containerCacheFlusher;
-      return this;
-    }
-
-    /**
-     * Sets the cblock Metrics.
-     *
-     * @param targetMetrics - CBlock Target Metrics
-     * @return - Builder
-     */
-    public Builder setCBlockTargetMetrics(CBlockTargetMetrics targetMetrics) {
-      this.metrics = targetMetrics;
-      return this;
-    }
-
-    /**
-     * Constructs a CBlockLocalCache.
-     *
-     * @return the CBlockLocalCache with the preset properties.
-     * @throws IOException
-     */
-    public CBlockLocalCache build() throws IOException {
-      Preconditions.checkNotNull(this.configuration, "A valid configuration " +
-          "is needed");
-      Preconditions.checkState(StringUtils.isNotBlank(userName), "A valid " +
-          "username is needed");
-      Preconditions.checkState(StringUtils.isNotBlank(volumeName), " A valid" +
-          " volume name is needed");
-      Preconditions.checkNotNull(this.pipelines, "Pipelines cannot be null");
-      Preconditions.checkState(this.pipelines.size() > 0, "At least one " +
-          "pipeline location is needed for a volume");
-
-      for (int x = 0; x < pipelines.size(); x++) {
-        Preconditions.checkNotNull(pipelines.get(x).getData(), "cBlock " +
-            "relies on private data on the pipeline, null data found.");
-      }
-
-      Preconditions.checkNotNull(clientManager, "Client Manager cannot be " +
-          "null");
-      Preconditions.checkState(blockSize > 0, " Block size has to be a " +
-          "number greater than 0");
-
-      Preconditions.checkState(volumeSize > 0, "Volume Size cannot be less " +
-          "than 1");
-      Preconditions.checkNotNull(this.flusher, "Flusher cannot be null.");
-
-      CBlockLocalCache cache = new CBlockLocalCache(this.configuration,
-          this.volumeName, this.userName, this.pipelines, blockSize,
-          volumeSize, flusher);
-      cache.setCblockTargetMetrics(this.metrics);
-      cache.setClientManager(this.clientManager);
-
-      // TODO : Support user configurable maximum size.
-      long cacheSize = computeCacheSize(this.configuration, this.volumeSize,
-          this.blockSize);
-      cache.setCurrentCacheSize(cacheSize);
-      return cache;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java
deleted file mode 100644
index 26c174f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
-
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import java.nio.ByteBuffer;
-
-/**
- * Impl class for LogicalBlock.
- */
-public class DiskBlock implements LogicalBlock {
-  private ByteBuffer data;
-  private long blockID;
-  private boolean persisted;
-
-  /**
-   * Constructs a DiskBlock Class from the following params.
-   * @param blockID - 64-bit block ID
-   * @param data - Byte Array
-   * @param persisted - Flag which tells us if this is persisted to remote
-   */
-  public DiskBlock(long blockID, byte[] data, boolean persisted) {
-    if (data !=null) {
-      this.data = ByteBuffer.wrap(data);
-    }
-    this.blockID = blockID;
-    this.persisted = persisted;
-  }
-
-  @Override
-  public ByteBuffer getData() {
-    return data;
-  }
-
-  /**
-   * Frees the byte buffer since we don't need it any more.
-   */
-  @Override
-  public void clearData() {
-    data.clear();
-  }
-
-  @Override
-  public long getBlockID() {
-    return blockID;
-  }
-
-  @Override
-  public boolean isPersisted() {
-    return persisted;
-  }
-
-  /**
-   * Sets the value of persisted.
-   * @param value - True if this has been persisted to container, false
-   * otherwise.
-   */
-  public void setPersisted(boolean value) {
-    persisted = value;
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java
deleted file mode 100644
index 533e919..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
-
-import com.google.common.primitives.Longs;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.concurrent.HadoopThreadPoolExecutor;
-import org.apache.hadoop.utils.LevelDBStore;
-import org.iq80.leveldb.DBException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Reads blocks from the container via the local cache.
- */
-public class SyncBlockReader {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(SyncBlockReader.class);
-
-  /**
-   * Update Queue - The reason why we have the queue is that we want to
-   * return the block as soon as we read it from the containers. This queue
-   * is work queue which will take the read block and update the cache.
-   * During testing we found levelDB is slow during writes, hence we wanted
-   * to return as block as soon as possible and update levelDB asynchronously.
-   */
-  private final static int QUEUE_SIZE = 1024;
-  /**
-   * Config.
-   */
-  private final Configuration conf;
-  /**
-   * The parent cache this reader is operating against.
-   */
-  private final CBlockLocalCache parentCache;
-  private final BlockingQueue<Runnable> updateQueue;
-
-  /**
-   * executor is used for running LevelDB updates. In future, we might do
-   * read-aheads and this pool is useful for that too. The reason why we
-   * don't share an executor for reads and writes is because the write task
-   * is couple of magnitude slower than read task. So we don't want the
-   * update DB to queue up behind the writes.
-   */
-  private final ThreadPoolExecutor executor;
-
-  /**
-   * Number of threads that pool starts with.
-   */
-  private final int corePoolSize = 1;
-  /**
-   * Maximum number of threads our pool will ever create.
-   */
-  private final int maxPoolSize = 10;
-  /**
-   * The idle time a thread hangs around waiting for work. if we don't find
-   * new work in 60 seconds the worker thread is killed.
-   */
-  private final long keepAlive = 60L;
-
-  /**
-   * Constructs a SyncBlock reader.
-   *
-   * @param conf - Configuration
-   * @param cache - Cache
-   */
-  public SyncBlockReader(Configuration conf, CBlockLocalCache cache) {
-    this.conf = conf;
-    this.parentCache = cache;
-    updateQueue = new ArrayBlockingQueue<>(QUEUE_SIZE, true);
-    ThreadFactory workerThreadFactory = new ThreadFactoryBuilder()
-        .setNameFormat("SyncBlockReader Thread #%d").setDaemon(true).build();
-    executor = new HadoopThreadPoolExecutor(
-        corePoolSize, maxPoolSize, keepAlive, TimeUnit.SECONDS,
-        updateQueue, workerThreadFactory,
-        new ThreadPoolExecutor.CallerRunsPolicy());
-  }
-
-  /**
-   * Returns the cache DB.
-   *
-   * @return LevelDB
-   */
-  private LevelDBStore getCacheDB() {
-    return parentCache.getCacheDB();
-  }
-
-  /**
-   * Returns data from the local cache if found, else reads from the remote
-   * container.
-   *
-   * @param blockID - blockID
-   * @return LogicalBlock
-   */
-  LogicalBlock readBlock(long blockID) throws IOException {
-    XceiverClientSpi client = null;
-    byte[] data = getblockFromDB(blockID);
-    if (data != null) {
-      parentCache.getTargetMetrics().incNumReadCacheHits();
-      return new DiskBlock(blockID, data, false);
-    }
-
-    parentCache.getTargetMetrics().incNumReadCacheMiss();
-    try {
-      client = parentCache.getClientManager()
-          .acquireClient(parentCache.getPipeline(blockID));
-      LogicalBlock block = getBlockFromContainer(blockID, client);
-      return block;
-    } catch (Exception ex) {
-      parentCache.getTargetMetrics().incNumFailedReadBlocks();
-      LOG.error("read failed for BlockId: {}", blockID, ex);
-      throw ex;
-    } finally {
-      if (client != null) {
-        parentCache.getClientManager().releaseClient(client);
-      }
-    }
-  }
-
-  /**
-   * Gets data from the DB if it exists.
-   *
-   * @param blockID - block id
-   * @return data
-   */
-  private byte[] getblockFromDB(long blockID) {
-    try {
-      if(parentCache.isShortCircuitIOEnabled()) {
-        long startTime = Time.monotonicNow();
-        byte[] data = getCacheDB().get(Longs.toByteArray(blockID));
-        long endTime = Time.monotonicNow();
-
-        if (parentCache.isTraceEnabled()) {
-          parentCache.getTracer().info(
-              "Task=ReadTaskDBRead,BlockID={},SHA={},Time={}",
-              blockID, (data != null && data.length > 0)
-                  ? DigestUtils.sha256Hex(data) : null,
-              endTime - startTime);
-        }
-        parentCache.getTargetMetrics().updateDBReadLatency(
-            endTime - startTime);
-        return data;
-      }
-
-
-    } catch (DBException dbe) {
-      LOG.error("Error while reading from cacheDB.", dbe);
-      throw dbe;
-    }
-    return null;
-  }
-
-
-  /**
-   * Returns a block from a Remote Container. if the key is not found on a
-   * remote container we just return a block initialzied with zeros.
-   *
-   * @param blockID - blockID
-   * @param client - client
-   * @return LogicalBlock
-   * @throws IOException
-   */
-  private LogicalBlock getBlockFromContainer(long blockID,
-      XceiverClientSpi client) throws IOException {
-    String containerName = parentCache.getPipeline(blockID).getContainerName();
-    try {
-      long startTime = Time.monotonicNow();
-      ContainerProtos.GetSmallFileResponseProto response =
-          ContainerProtocolCalls.readSmallFile(client, containerName,
-              Long.toString(blockID), parentCache.getTraceID(blockID));
-      long endTime = Time.monotonicNow();
-      if (parentCache.isTraceEnabled()) {
-        parentCache.getTracer().info(
-            "Task=ReadTaskContainerRead,BlockID={},SHA={},Time={}",
-            blockID, response.getData().getData().toByteArray().length > 0 ?
-                DigestUtils.sha256Hex(response.getData()
-                    .getData().toByteArray()) : null, endTime - startTime);
-      }
-
-      parentCache.getTargetMetrics().updateContainerReadLatency(
-          endTime - startTime);
-      DiskBlock block = new DiskBlock(blockID,
-          response.getData().getData().toByteArray(), false);
-
-      if(parentCache.isShortCircuitIOEnabled()) {
-        queueUpdateTask(block);
-      }
-
-      return block;
-    } catch (IOException ex) {
-      if (ex instanceof StorageContainerException) {
-        parentCache.getTargetMetrics().incNumReadLostBlocks();
-        StorageContainerException sce = (StorageContainerException) ex;
-        if (sce.getResult() == ContainerProtos.Result.NO_SUCH_KEY ||
-            sce.getResult() == ContainerProtos.Result.IO_EXCEPTION) {
-          return new DiskBlock(blockID, new byte[parentCache.getBlockSize()],
-              false);
-        }
-      }
-      throw ex;
-    }
-  }
-
-  /**
-   * Updates the cache with the block that we just read.
-   *
-   * @param block
-   */
-  private void queueUpdateTask(final DiskBlock block) {
-    Runnable updateTask = () -> {
-      if(block.getData().array().length > 0) {
-        getCacheDB().put(Longs.toByteArray(block.getBlockID()),
-            block.getData().array());
-        block.setPersisted(true);
-      } else {
-        LOG.error("Refusing to update the a null block in the local cache.");
-      }
-    };
-    if (this.executor.isShutdown() || this.executor.isTerminated()) {
-      LOG.error("Thread executor is not running.");
-    } else {
-      this.executor.submit(updateTask);
-    }
-  }
-
-  /**
-   * This is a read operation, we don't care if we updated the cache with the
-   * last block e read.
-   */
-  void shutdown() {
-    this.executor.shutdownNow();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java
deleted file mode 100644
index dfac110..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper.cache.impl;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java
deleted file mode 100644
index 47f76b8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper.cache;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[31/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
deleted file mode 100644
index 62fcd66..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
+++ /dev/null
@@ -1,1101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.impl;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
-import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.*;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMStorageReport;
-import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.utils.MetadataKeyFilters;
-import org.apache.hadoop.utils.MetadataStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.security.DigestInputStream;
-import java.security.DigestOutputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentNavigableMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.CONTAINER_EXISTS;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.CONTAINER_INTERNAL_ERROR;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.CONTAINER_NOT_FOUND;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.INVALID_CONFIG;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.IO_EXCEPTION;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.NO_SUCH_ALGORITHM;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.UNABLE_TO_READ_METADATA_DB;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.UNSUPPORTED_REQUEST;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-   .Result.ERROR_IN_COMPACT_DB;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.UNCLOSED_CONTAINER_IO;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_EXTENSION;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_META;
-
-/**
- * A Generic ContainerManagerImpl that will be called from Ozone
- * ContainerManagerImpl. This allows us to support delta changes to ozone
- * version without having to rewrite the containerManager.
- */
-public class ContainerManagerImpl implements ContainerManager {
-  static final Logger LOG =
-      LoggerFactory.getLogger(ContainerManagerImpl.class);
-
-  private final ConcurrentSkipListMap<String, ContainerStatus>
-      containerMap = new ConcurrentSkipListMap<>();
-
-  // Use a non-fair RW lock for better throughput, we may revisit this decision
-  // if this causes fairness issues.
-  private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
-  private ContainerLocationManager locationManager;
-  private ChunkManager chunkManager;
-  private KeyManager keyManager;
-  private Configuration conf;
-  private DatanodeID datanodeID;
-
-  private ContainerDeletionChoosingPolicy containerDeletionChooser;
-  private ContainerReportManager containerReportManager;
-
-  /**
-   * Init call that sets up a container Manager.
-   *
-   * @param config - Configuration.
-   * @param containerDirs - List of Metadata Container locations.
-   * @param datanode - Datanode ID.
-   * @throws IOException
-   */
-  @Override
-  public void init(
-      Configuration config, List<StorageLocation> containerDirs,
-      DatanodeID datanode) throws IOException {
-    Preconditions.checkNotNull(config, "Config must not be null");
-    Preconditions.checkNotNull(containerDirs, "Container directories cannot " +
-        "be null");
-    Preconditions.checkNotNull(datanode, "Datanode ID cannot " +
-        "be null");
-
-    Preconditions.checkState(containerDirs.size() > 0, "Number of container" +
-        " directories must be greater than zero.");
-
-    this.conf = config;
-    this.datanodeID = datanode;
-
-    readLock();
-    try {
-      containerDeletionChooser = ReflectionUtils.newInstance(conf.getClass(
-          ScmConfigKeys.OZONE_SCM_CONTAINER_DELETION_CHOOSING_POLICY,
-          TopNOrderedContainerDeletionChoosingPolicy.class,
-          ContainerDeletionChoosingPolicy.class), conf);
-
-      for (StorageLocation path : containerDirs) {
-        File directory = Paths.get(path.getNormalizedUri()).toFile();
-        if (!directory.exists() && !directory.mkdirs()) {
-          LOG.error("Container metadata directory doesn't exist "
-              + "and cannot be created. Path: {}", path.toString());
-          throw new StorageContainerException("Container metadata "
-              + "directory doesn't exist and cannot be created " + path
-              .toString(), INVALID_CONFIG);
-        }
-
-        // TODO: This will fail if any directory is invalid.
-        // We should fix this to handle invalid directories and continue.
-        // Leaving it this way to fail fast for time being.
-        if (!directory.isDirectory()) {
-          LOG.error("Invalid path to container metadata directory. path: {}",
-              path.toString());
-          throw new StorageContainerException("Invalid path to container " +
-              "metadata directory." + path, INVALID_CONFIG);
-        }
-        LOG.info("Loading containers under {}", path);
-        File[] files = directory.listFiles(new ContainerFilter());
-        if (files != null) {
-          for (File containerFile : files) {
-            LOG.debug("Loading container {}", containerFile);
-            String containerPath =
-                ContainerUtils.getContainerNameFromFile(containerFile);
-            Preconditions.checkNotNull(containerPath, "Container path cannot" +
-                " be null");
-            readContainerInfo(containerPath);
-          }
-        }
-      }
-
-      List<StorageLocation> dataDirs = new LinkedList<>();
-      for (String dir : config.getStrings(DFS_DATANODE_DATA_DIR_KEY)) {
-        StorageLocation location = StorageLocation.parse(dir);
-        dataDirs.add(location);
-      }
-      this.locationManager =
-          new ContainerLocationManagerImpl(containerDirs, dataDirs, config);
-
-      this.containerReportManager =
-          new ContainerReportManagerImpl(config);
-    } finally {
-      readUnlock();
-    }
-  }
-
-  /**
-   * Reads the Container Info from a file and verifies that checksum match. If
-   * the checksums match, then that file is added to containerMap.
-   *
-   * @param containerName - Name which points to the persisted container.
-   * @throws StorageContainerException
-   */
-  private void readContainerInfo(String containerName)
-      throws StorageContainerException {
-    Preconditions.checkState(containerName.length() > 0,
-        "Container name length cannot be zero.");
-    FileInputStream containerStream = null;
-    DigestInputStream dis = null;
-    FileInputStream metaStream = null;
-    Path cPath = Paths.get(containerName).getFileName();
-    String keyName = null;
-    if (cPath != null) {
-      keyName = cPath.toString();
-    }
-    Preconditions.checkNotNull(keyName,
-        "Container Name  to container key mapping is null");
-
-    try {
-      String containerFileName = containerName.concat(CONTAINER_EXTENSION);
-      String metaFileName = containerName.concat(CONTAINER_META);
-
-      containerStream = new FileInputStream(containerFileName);
-
-      metaStream = new FileInputStream(metaFileName);
-
-      MessageDigest sha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
-
-      dis = new DigestInputStream(containerStream, sha);
-
-      ContainerProtos.ContainerData containerDataProto =
-          ContainerProtos.ContainerData.parseDelimitedFrom(dis);
-      ContainerData containerData;
-      if (containerDataProto == null) {
-        // Sometimes container metadata might have been created but empty,
-        // when loading the info we get a null, this often means last time
-        // SCM was ending up at some middle phase causing that the metadata
-        // was not populated. Such containers are marked as inactive.
-        containerMap.put(keyName, new ContainerStatus(null));
-        return;
-      }
-      containerData = ContainerData.getFromProtBuf(containerDataProto, conf);
-      ContainerProtos.ContainerMeta meta =
-          ContainerProtos.ContainerMeta.parseDelimitedFrom(metaStream);
-      if (meta != null && !DigestUtils.sha256Hex(sha.digest())
-          .equals(meta.getHash())) {
-        // This means we were not able read data from the disk when booted the
-        // datanode. We are going to rely on SCM understanding that we don't
-        // have valid data for this container when we send container reports.
-        // Hopefully SCM will ask us to delete this container and rebuild it.
-        LOG.error("Invalid SHA found for container data. Name :{}"
-            + "cowardly refusing to read invalid data", containerName);
-        containerMap.put(keyName, new ContainerStatus(null));
-        return;
-      }
-
-      ContainerStatus containerStatus = new ContainerStatus(containerData);
-      // Initialize pending deletion blocks count in in-memory
-      // container status.
-      MetadataStore metadata = KeyUtils.getDB(containerData, conf);
-      List<Map.Entry<byte[], byte[]>> underDeletionBlocks = metadata
-          .getSequentialRangeKVs(null, Integer.MAX_VALUE,
-              MetadataKeyFilters.getDeletingKeyFilter());
-      containerStatus.incrPendingDeletionBlocks(underDeletionBlocks.size());
-
-      List<Map.Entry<byte[], byte[]>> liveKeys = metadata
-          .getRangeKVs(null, Integer.MAX_VALUE,
-              MetadataKeyFilters.getNormalKeyFilter());
-
-      // Get container bytesUsed upon loading container
-      // The in-memory state is updated upon key write or delete
-      // TODO: update containerDataProto and persist it into container MetaFile
-      long bytesUsed = 0;
-      bytesUsed = liveKeys.parallelStream().mapToLong(e-> {
-        KeyData keyData;
-        try {
-          keyData = KeyUtils.getKeyData(e.getValue());
-          return keyData.getSize();
-        } catch (IOException ex) {
-          return 0L;
-        }
-      }).sum();
-      containerStatus.setBytesUsed(bytesUsed);
-
-      containerMap.put(keyName, containerStatus);
-    } catch (IOException | NoSuchAlgorithmException ex) {
-      LOG.error("read failed for file: {} ex: {}", containerName,
-          ex.getMessage());
-
-      // TODO : Add this file to a recovery Queue.
-
-      // Remember that this container is busted and we cannot use it.
-      containerMap.put(keyName, new ContainerStatus(null));
-      throw new StorageContainerException("Unable to read container info",
-          UNABLE_TO_READ_METADATA_DB);
-    } finally {
-      IOUtils.closeStream(dis);
-      IOUtils.closeStream(containerStream);
-      IOUtils.closeStream(metaStream);
-    }
-  }
-
-  /**
-   * Creates a container with the given name.
-   *
-   * @param pipeline -- Nodes which make up this container.
-   * @param containerData - Container Name and metadata.
-   * @throws StorageContainerException - Exception
-   */
-  @Override
-  public void createContainer(Pipeline pipeline, ContainerData containerData)
-      throws StorageContainerException {
-    Preconditions.checkNotNull(containerData, "Container data cannot be null");
-    writeLock();
-    try {
-      if (containerMap.containsKey(containerData.getName())) {
-        LOG.debug("container already exists. {}", containerData.getName());
-        throw new StorageContainerException("container already exists.",
-            CONTAINER_EXISTS);
-      }
-
-      // This is by design. We first write and close the
-      // container Info and metadata to a directory.
-      // Then read back and put that info into the containerMap.
-      // This allows us to make sure that our write is consistent.
-
-      writeContainerInfo(containerData, false);
-      File cFile = new File(containerData.getContainerPath());
-      readContainerInfo(ContainerUtils.getContainerNameFromFile(cFile));
-    } catch (NoSuchAlgorithmException ex) {
-      LOG.error("Internal error: We seem to be running a JVM without a " +
-          "needed hash algorithm.");
-      throw new StorageContainerException("failed to create container",
-          NO_SUCH_ALGORITHM);
-    } finally {
-      writeUnlock();
-    }
-
-  }
-
-  /**
-   * Writes a container to a chosen location and updates the container Map.
-   *
-   * The file formats of ContainerData and Container Meta is the following.
-   *
-   * message ContainerData {
-   * required string name = 1;
-   * repeated KeyValue metadata = 2;
-   * optional string dbPath = 3;
-   * optional string containerPath = 4;
-   * optional int64 bytesUsed = 5;
-   * optional int64 size = 6;
-   * }
-   *
-   * message ContainerMeta {
-   * required string fileName = 1;
-   * required string hash = 2;
-   * }
-   *
-   * @param containerData - container Data
-   * @param overwrite - Whether we are overwriting.
-   * @throws StorageContainerException, NoSuchAlgorithmException
-   */
-  private void writeContainerInfo(ContainerData containerData,
-      boolean  overwrite)
-      throws StorageContainerException, NoSuchAlgorithmException {
-
-    Preconditions.checkNotNull(this.locationManager,
-        "Internal error: location manager cannot be null");
-
-    FileOutputStream containerStream = null;
-    DigestOutputStream dos = null;
-    FileOutputStream metaStream = null;
-
-    try {
-      Path metadataPath = null;
-      Path location = (!overwrite) ? locationManager.getContainerPath():
-          Paths.get(containerData.getContainerPath()).getParent();
-      if (location == null) {
-        throw new StorageContainerException(
-            "Failed to get container file path.",
-            CONTAINER_INTERNAL_ERROR);
-      }
-
-      File containerFile = ContainerUtils.getContainerFile(containerData,
-          location);
-      File metadataFile = ContainerUtils.getMetadataFile(containerData,
-          location);
-      String containerName = containerData.getContainerName();
-
-      if(!overwrite) {
-        ContainerUtils.verifyIsNewContainer(containerFile, metadataFile);
-        metadataPath = this.locationManager.getDataPath(containerName);
-        metadataPath = ContainerUtils.createMetadata(metadataPath,
-            containerName, conf);
-      }  else {
-        metadataPath = ContainerUtils.getMetadataDirectory(containerData);
-      }
-
-      containerStream = new FileOutputStream(containerFile);
-      metaStream = new FileOutputStream(metadataFile);
-      MessageDigest sha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
-
-      dos = new DigestOutputStream(containerStream, sha);
-      containerData.setDBPath(metadataPath.resolve(
-          ContainerUtils.getContainerDbFileName(containerName))
-          .toString());
-      containerData.setContainerPath(containerFile.toString());
-
-      ContainerProtos.ContainerData protoData = containerData
-          .getProtoBufMessage();
-      protoData.writeDelimitedTo(dos);
-
-      ContainerProtos.ContainerMeta protoMeta = ContainerProtos
-          .ContainerMeta.newBuilder()
-          .setFileName(containerFile.toString())
-          .setHash(DigestUtils.sha256Hex(sha.digest()))
-          .build();
-      protoMeta.writeDelimitedTo(metaStream);
-
-    } catch (IOException ex) {
-      // TODO : we need to clean up partially constructed files
-      // The proper way to do would be for a thread
-      // to read all these 3 artifacts and make sure they are
-      // sane. That info needs to come from the replication
-      // pipeline, and if not consistent delete these file.
-
-      // In case of ozone this is *not* a deal breaker since
-      // SCM is guaranteed to generate unique container names.
-      // The saving grace is that we check if we have residue files
-      // lying around when creating a new container. We need to queue
-      // this information to a cleaner thread.
-
-      LOG.error("Creation of container failed. Name: {}, we might need to " +
-              "cleanup partially created artifacts. ",
-          containerData.getContainerName(), ex);
-      throw new StorageContainerException("Container creation failed. ",
-          ex, CONTAINER_INTERNAL_ERROR);
-    } finally {
-      IOUtils.closeStream(dos);
-      IOUtils.closeStream(containerStream);
-      IOUtils.closeStream(metaStream);
-    }
-  }
-
-  /**
-   * Deletes an existing container.
-   *
-   * @param pipeline - nodes that make this container.
-   * @param containerName - name of the container.
-   * @param forceDelete - whether this container should be deleted forcibly.
-   * @throws StorageContainerException
-   */
-  @Override
-  public void deleteContainer(Pipeline pipeline, String containerName,
-      boolean forceDelete) throws StorageContainerException {
-    Preconditions.checkNotNull(containerName, "Container name cannot be null");
-    Preconditions.checkState(containerName.length() > 0,
-        "Container name length cannot be zero.");
-    writeLock();
-    try {
-      if (isOpen(pipeline.getContainerName())) {
-        throw new StorageContainerException(
-            "Deleting an open container is not allowed.",
-            UNCLOSED_CONTAINER_IO);
-      }
-
-      ContainerStatus status = containerMap.get(containerName);
-      if (status == null) {
-        LOG.debug("No such container. Name: {}", containerName);
-        throw new StorageContainerException("No such container. Name : " +
-            containerName, CONTAINER_NOT_FOUND);
-      }
-      if (status.getContainer() == null) {
-        LOG.debug("Invalid container data. Name: {}", containerName);
-        throw new StorageContainerException("Invalid container data. Name : " +
-            containerName, CONTAINER_NOT_FOUND);
-      }
-      ContainerUtils.removeContainer(status.getContainer(), conf, forceDelete);
-      containerMap.remove(containerName);
-    } catch (StorageContainerException e) {
-      throw e;
-    } catch (IOException e) {
-      // TODO : An I/O error during delete can leave partial artifacts on the
-      // disk. We will need the cleaner thread to cleanup this information.
-      LOG.error("Failed to cleanup container. Name: {}", containerName, e);
-      throw new StorageContainerException(containerName, e, IO_EXCEPTION);
-    } finally {
-      writeUnlock();
-    }
-  }
-
-  /**
-   * A simple interface for container Iterations.
-   * <p/>
-   * This call make no guarantees about consistency of the data between
-   * different list calls. It just returns the best known data at that point of
-   * time. It is possible that using this iteration you can miss certain
-   * container from the listing.
-   *
-   * @param prefix -  Return keys that match this prefix.
-   * @param count - how many to return
-   * @param prevKey - Previous Key Value or empty String.
-   * @param data - Actual containerData
-   * @throws StorageContainerException
-   */
-  @Override
-  public void listContainer(String prefix, long count, String prevKey,
-      List<ContainerData> data) throws StorageContainerException {
-    // TODO : Support list with Prefix and PrevKey
-    Preconditions.checkNotNull(data,
-        "Internal assertion: data cannot be null");
-    readLock();
-    try {
-      ConcurrentNavigableMap<String, ContainerStatus> map;
-      if (prevKey == null || prevKey.isEmpty()) {
-        map = containerMap.tailMap(containerMap.firstKey(), true);
-      } else {
-        map = containerMap.tailMap(prevKey, false);
-      }
-
-      int currentCount = 0;
-      for (ContainerStatus entry : map.values()) {
-        if (currentCount < count) {
-          data.add(entry.getContainer());
-          currentCount++;
-        } else {
-          return;
-        }
-      }
-    } finally {
-      readUnlock();
-    }
-  }
-
-  /**
-   * Get metadata about a specific container.
-   *
-   * @param containerName - Name of the container
-   * @return ContainerData - Container Data.
-   * @throws StorageContainerException
-   */
-  @Override
-  public ContainerData readContainer(String containerName) throws
-      StorageContainerException {
-    Preconditions.checkNotNull(containerName, "Container name cannot be null");
-    Preconditions.checkState(containerName.length() > 0,
-        "Container name length cannot be zero.");
-    if (!containerMap.containsKey(containerName)) {
-      throw new StorageContainerException("Unable to find the container. Name: "
-          + containerName, CONTAINER_NOT_FOUND);
-    }
-    ContainerData cData = containerMap.get(containerName).getContainer();
-    if (cData == null) {
-      throw new StorageContainerException("Invalid container data. Name: "
-          + containerName, CONTAINER_INTERNAL_ERROR);
-    }
-    return cData;
-  }
-
-  /**
-   * Closes a open container, if it is already closed or does not exist a
-   * StorageContainerException is thrown.
-   *
-   * @param containerName - Name of the container.
-   * @throws StorageContainerException
-   */
-  @Override
-  public void closeContainer(String containerName)
-      throws StorageContainerException, NoSuchAlgorithmException {
-    ContainerData containerData = readContainer(containerName);
-    containerData.closeContainer();
-    writeContainerInfo(containerData, true);
-    MetadataStore db = KeyUtils.getDB(containerData, conf);
-
-    // It is ok if this operation takes a bit of time.
-    // Close container is not expected to be instantaneous.
-    try {
-      db.compactDB();
-    } catch (IOException e) {
-      LOG.error("Error in DB compaction while closing container", e);
-      throw new StorageContainerException(e, ERROR_IN_COMPACT_DB);
-    }
-
-    // Active is different from closed. Closed means it is immutable, active
-    // false means we have some internal error that is happening to this
-    // container. This is a way to track damaged containers if we have an
-    // I/O failure, this allows us to take quick action in case of container
-    // issues.
-
-    ContainerStatus status = new ContainerStatus(containerData);
-    containerMap.put(containerName, status);
-  }
-
-  @Override
-  public void updateContainer(Pipeline pipeline, String containerName,
-      ContainerData data, boolean forceUpdate)
-      throws StorageContainerException {
-    Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
-    Preconditions.checkNotNull(containerName, "Container name cannot be null");
-    Preconditions.checkNotNull(data, "Container data cannot be null");
-    FileOutputStream containerStream = null;
-    DigestOutputStream dos = null;
-    MessageDigest sha = null;
-    File containerFileBK = null, containerFile = null;
-    boolean deleted = false;
-
-    if(!containerMap.containsKey(containerName)) {
-      throw new StorageContainerException("Container doesn't exist. Name :"
-          + containerName, CONTAINER_NOT_FOUND);
-    }
-
-    try {
-      sha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
-    } catch (NoSuchAlgorithmException e) {
-      throw new StorageContainerException("Unable to create Message Digest,"
-          + " usually this is a java configuration issue.",
-          NO_SUCH_ALGORITHM);
-    }
-
-    try {
-      Path location = locationManager.getContainerPath();
-      ContainerData orgData = containerMap.get(containerName).getContainer();
-      if (orgData == null) {
-        // updating a invalid container
-        throw new StorageContainerException("Update a container with invalid" +
-            "container meta data", CONTAINER_INTERNAL_ERROR);
-      }
-
-      if (!forceUpdate && !orgData.isOpen()) {
-        throw new StorageContainerException(
-            "Update a closed container is not allowed. Name: " + containerName,
-            UNSUPPORTED_REQUEST);
-      }
-
-      containerFile = ContainerUtils.getContainerFile(orgData, location);
-      // If forceUpdate is true, there is no need to check
-      // whether the container file exists.
-      if (!forceUpdate) {
-        if (!containerFile.exists() || !containerFile.canWrite()) {
-          throw new StorageContainerException(
-              "Container file not exists or corrupted. Name: " + containerName,
-              CONTAINER_INTERNAL_ERROR);
-        }
-
-        // Backup the container file
-        containerFileBK = File.createTempFile(
-            "tmp_" + System.currentTimeMillis() + "_",
-            containerFile.getName(), containerFile.getParentFile());
-        FileUtils.copyFile(containerFile, containerFileBK);
-
-        deleted = containerFile.delete();
-        containerStream = new FileOutputStream(containerFile);
-        dos = new DigestOutputStream(containerStream, sha);
-
-        ContainerProtos.ContainerData protoData = data.getProtoBufMessage();
-        protoData.writeDelimitedTo(dos);
-      }
-
-      // Update the in-memory map
-      ContainerStatus newStatus = new ContainerStatus(data);
-      containerMap.replace(containerName, newStatus);
-    } catch (IOException e) {
-      // Restore the container file from backup
-      if(containerFileBK != null && containerFileBK.exists() && deleted) {
-        if(containerFile.delete()
-            && containerFileBK.renameTo(containerFile)) {
-          throw new StorageContainerException("Container update failed,"
-              + " container data restored from the backup.",
-              CONTAINER_INTERNAL_ERROR);
-        } else {
-          throw new StorageContainerException(
-              "Failed to restore container data from the backup. Name: "
-                  + containerName, CONTAINER_INTERNAL_ERROR);
-        }
-      } else {
-        throw new StorageContainerException(
-            e.getMessage(), CONTAINER_INTERNAL_ERROR);
-      }
-    } finally {
-      if (containerFileBK != null && containerFileBK.exists()) {
-        if(!containerFileBK.delete()) {
-          LOG.warn("Unable to delete container file backup : {}.",
-              containerFileBK.getAbsolutePath());
-        }
-      }
-      IOUtils.closeStream(dos);
-      IOUtils.closeStream(containerStream);
-    }
-  }
-
-  @VisibleForTesting
-  protected File getContainerFile(ContainerData data) throws IOException {
-    return ContainerUtils.getContainerFile(data,
-        this.locationManager.getContainerPath());
-  }
-
-  /**
-   * Checks if a container exists.
-   *
-   * @param containerName - Name of the container.
-   * @return true if the container is open false otherwise.
-   * @throws StorageContainerException - Throws Exception if we are not able to
-   *                                   find the container.
-   */
-  @Override
-  public boolean isOpen(String containerName) throws StorageContainerException {
-    final ContainerStatus status = containerMap.get(containerName);
-    if (status == null) {
-      throw new StorageContainerException(
-          "Container status not found: " + containerName, CONTAINER_NOT_FOUND);
-    }
-    final ContainerData cData = status.getContainer();
-    if (cData == null) {
-      throw new StorageContainerException(
-          "Container not found: " + containerName, CONTAINER_NOT_FOUND);
-    }
-    return cData.isOpen();
-  }
-
-  /**
-   * Supports clean shutdown of container.
-   *
-   * @throws IOException
-   */
-  @Override
-  public void shutdown() throws IOException {
-    Preconditions.checkState(this.hasWriteLock(),
-        "Assumption that we are holding the lock violated.");
-    this.containerMap.clear();
-    this.locationManager.shutdown();
-  }
-
-
-  @VisibleForTesting
-  public ConcurrentSkipListMap<String, ContainerStatus> getContainerMap() {
-    return containerMap;
-  }
-
-  /**
-   * Acquire read lock.
-   */
-  @Override
-  public void readLock() {
-    this.lock.readLock().lock();
-
-  }
-
-    @Override
-  public void readLockInterruptibly() throws InterruptedException {
-    this.lock.readLock().lockInterruptibly();
-  }
-
-  /**
-   * Release read lock.
-   */
-  @Override
-  public void readUnlock() {
-    this.lock.readLock().unlock();
-  }
-
-  /**
-   * Check if the current thread holds read lock.
-   */
-  @Override
-  public boolean hasReadLock() {
-    return this.lock.readLock().tryLock();
-  }
-
-  /**
-   * Acquire write lock.
-   */
-  @Override
-  public void writeLock() {
-    this.lock.writeLock().lock();
-  }
-
-  /**
-   * Acquire write lock, unless interrupted while waiting.
-   */
-  @Override
-  public void writeLockInterruptibly() throws InterruptedException {
-    this.lock.writeLock().lockInterruptibly();
-
-  }
-
-  /**
-   * Release write lock.
-   */
-  @Override
-  public void writeUnlock() {
-    this.lock.writeLock().unlock();
-
-  }
-
-  /**
-   * Check if the current thread holds write lock.
-   */
-  @Override
-  public boolean hasWriteLock() {
-    return this.lock.writeLock().isHeldByCurrentThread();
-  }
-
-  public ChunkManager getChunkManager() {
-    return this.chunkManager;
-  }
-
-  /**
-   * Sets the chunk Manager.
-   *
-   * @param chunkManager - Chunk Manager
-   */
-  public void setChunkManager(ChunkManager chunkManager) {
-    this.chunkManager = chunkManager;
-  }
-
-  /**
-   * Gets the Key Manager.
-   *
-   * @return KeyManager.
-   */
-  @Override
-  public KeyManager getKeyManager() {
-    return this.keyManager;
-  }
-
-  /**
-   * Get the node report.
-   * @return node report.
-   */
-  @Override
-  public SCMNodeReport getNodeReport() throws IOException {
-    StorageLocationReport[] reports = locationManager.getLocationReport();
-    SCMNodeReport.Builder nrb = SCMNodeReport.newBuilder();
-    for (int i = 0; i < reports.length; i++) {
-      SCMStorageReport.Builder srb = SCMStorageReport.newBuilder();
-      nrb.addStorageReport(i, srb.setStorageUuid(reports[i].getId())
-          .setCapacity(reports[i].getCapacity())
-          .setScmUsed(reports[i].getScmUsed())
-          .setRemaining(reports[i].getRemaining())
-          .build());
-    }
-    return nrb.build();
-  }
-
-
-  /**
-   * Gets container reports.
-   *
-   * @return List of all closed containers.
-   * @throws IOException
-   */
-  @Override
-  public List<ContainerData> getContainerReports() throws IOException {
-    LOG.debug("Starting container report iteration.");
-    // No need for locking since containerMap is a ConcurrentSkipListMap
-    // And we can never get the exact state since close might happen
-    // after we iterate a point.
-    return containerMap.entrySet().stream()
-        .filter(containerStatus ->
-            !containerStatus.getValue().getContainer().isOpen())
-        .map(containerStatus -> containerStatus.getValue().getContainer())
-        .collect(Collectors.toList());
-  }
-
-  /**
-   * Get container report.
-   *
-   * @return The container report.
-   * @throws IOException
-   */
-  @Override
-  public ContainerReportsRequestProto getContainerReport() throws IOException {
-    LOG.debug("Starting container report iteration.");
-    // No need for locking since containerMap is a ConcurrentSkipListMap
-    // And we can never get the exact state since close might happen
-    // after we iterate a point.
-    List<ContainerStatus> containers = containerMap.values().stream()
-        .collect(Collectors.toList());
-
-    ContainerReportsRequestProto.Builder crBuilder =
-        ContainerReportsRequestProto.newBuilder();
-
-    // TODO: support delta based container report
-    crBuilder.setDatanodeID(datanodeID.getProtoBufMessage())
-        .setType(ContainerReportsRequestProto.reportType.fullReport);
-
-    for (ContainerStatus container: containers) {
-      StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder =
-          StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder();
-      ciBuilder.setContainerName(container.getContainer().getContainerName())
-          .setSize(container.getContainer().getMaxSize())
-          .setUsed(container.getContainer().getBytesUsed())
-          .setKeyCount(container.getContainer().getKeyCount())
-          .setReadCount(container.getReadCount())
-          .setWriteCount(container.getWriteCount())
-          .setReadBytes(container.getReadBytes())
-          .setWriteBytes(container.getWriteBytes())
-          .setContainerID(container.getContainer().getContainerID());
-
-      if (container.getContainer().getHash() != null) {
-        ciBuilder.setFinalhash(container.getContainer().getHash());
-      }
-      crBuilder.addReports(ciBuilder.build());
-    }
-
-    return crBuilder.build();
-  }
-
-  /**
-   * Sets the Key Manager.
-   *
-   * @param keyManager - Key Manager.
-   */
-  @Override
-  public void setKeyManager(KeyManager keyManager) {
-    this.keyManager = keyManager;
-  }
-
-  /**
-   * Filter out only container files from the container metadata dir.
-   */
-  private static class ContainerFilter implements FilenameFilter {
-    /**
-     * Tests if a specified file should be included in a file list.
-     *
-     * @param dir the directory in which the file was found.
-     * @param name the name of the file.
-     * @return <code>true</code> if and only if the name should be included in
-     * the file list; <code>false</code> otherwise.
-     */
-    @Override
-    public boolean accept(File dir, String name) {
-      return name.endsWith(CONTAINER_EXTENSION);
-    }
-  }
-
-  @Override
-  public List<ContainerData> chooseContainerForBlockDeletion(
-      int count) throws StorageContainerException {
-    readLock();
-    try {
-      return containerDeletionChooser.chooseContainerForBlockDeletion(
-          count, containerMap);
-    } finally {
-      readUnlock();
-    }
-  }
-
-  @VisibleForTesting
-  public ContainerDeletionChoosingPolicy getContainerDeletionChooser() {
-    return containerDeletionChooser;
-  }
-
-  @Override
-  public void incrPendingDeletionBlocks(int numBlocks, String containerId) {
-    writeLock();
-    try {
-      ContainerStatus status = containerMap.get(containerId);
-      status.incrPendingDeletionBlocks(numBlocks);
-    } finally {
-      writeUnlock();
-    }
-  }
-
-  @Override
-  public void decrPendingDeletionBlocks(int numBlocks, String containerId) {
-    writeLock();
-    try {
-      ContainerStatus status = containerMap.get(containerId);
-      status.decrPendingDeletionBlocks(numBlocks);
-    } finally {
-      writeUnlock();
-    }
-  }
-
-  /**
-   * Increase the read count of the container.
-   *
-   * @param containerName - Name of the container.
-   */
-  @Override
-  public void incrReadCount(String containerName) {
-    ContainerStatus status = containerMap.get(containerName);
-    status.incrReadCount();
-  }
-
-  public long getReadCount(String containerName) {
-    ContainerStatus status = containerMap.get(containerName);
-    return status.getReadCount();
-  }
-
-  /**
-   * Increse the read counter for bytes read from the container.
-   *
-   * @param containerName - Name of the container.
-   * @param readBytes     - bytes read from the container.
-   */
-  @Override
-  public void incrReadBytes(String containerName, long readBytes) {
-    ContainerStatus status = containerMap.get(containerName);
-    status.incrReadBytes(readBytes);
-  }
-
-  public long getReadBytes(String containerName) {
-    readLock();
-    try {
-      ContainerStatus status = containerMap.get(containerName);
-      return status.getReadBytes();
-    } finally {
-      readUnlock();
-    }
-  }
-
-  /**
-   * Increase the write count of the container.
-   *
-   * @param containerName - Name of the container.
-   */
-  @Override
-  public void incrWriteCount(String containerName) {
-    ContainerStatus status = containerMap.get(containerName);
-    status.incrWriteCount();
-  }
-
-  public long getWriteCount(String containerName) {
-    ContainerStatus status = containerMap.get(containerName);
-    return status.getWriteCount();
-  }
-
-  /**
-   * Increse the write counter for bytes write into the container.
-   *
-   * @param containerName - Name of the container.
-   * @param writeBytes    - bytes write into the container.
-   */
-  @Override
-  public void incrWriteBytes(String containerName, long writeBytes) {
-    ContainerStatus status = containerMap.get(containerName);
-    status.incrWriteBytes(writeBytes);
-  }
-
-  public long getWriteBytes(String containerName) {
-    ContainerStatus status = containerMap.get(containerName);
-    return status.getWriteBytes();
-  }
-
-  /**
-   * Increase the bytes used by the container.
-   *
-   * @param containerName - Name of the container.
-   * @param used          - additional bytes used by the container.
-   * @return the current bytes used.
-   */
-  @Override
-  public long incrBytesUsed(String containerName, long used) {
-    ContainerStatus status = containerMap.get(containerName);
-    return status.incrBytesUsed(used);
-  }
-
-  /**
-   * Decrease the bytes used by the container.
-   *
-   * @param containerName - Name of the container.
-   * @param used          - additional bytes reclaimed by the container.
-   * @return the current bytes used.
-   */
-  @Override
-  public long decrBytesUsed(String containerName, long used) {
-    ContainerStatus status = containerMap.get(containerName);
-    return status.decrBytesUsed(used);
-  }
-
-  public long getBytesUsed(String containerName) {
-    ContainerStatus status = containerMap.get(containerName);
-    return status.getBytesUsed();
-  }
-
-  /**
-   * Get the number of keys in the container.
-   *
-   * @param containerName - Name of the container.
-   * @return the current key count.
-   */
-  @Override
-  public long getNumKeys(String containerName) {
-    ContainerStatus status = containerMap.get(containerName);
-    return status.getNumKeys();  }
-
-  /**
-   * Get the container report state to send via HB to SCM.
-   *
-   * @return container report state.
-   */
-  @Override
-  public ReportState getContainerReportState() {
-    return containerReportManager.getContainerReportState();
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerReportManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerReportManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerReportManagerImpl.java
deleted file mode 100644
index 4669737..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerReportManagerImpl.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerReportManager;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.util.Time;
-
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Class wraps the container report operations on datanode.
- * // TODO: support incremental/delta container report
- */
-public class ContainerReportManagerImpl implements ContainerReportManager {
-  // Last non-empty container report time
-  private long lastContainerReportTime;
-  private final long containerReportInterval;
-  private final long heartbeatInterval;
-  private AtomicLong reportCount;
-  private static final ReportState NO_CONTAINER_REPORTSTATE =
-      ReportState.newBuilder()
-          .setState(ReportState.states.noContainerReports)
-          .setCount(0).build();
-
-  public ContainerReportManagerImpl(Configuration config) {
-    this.lastContainerReportTime = -1;
-    this.reportCount = new AtomicLong(0L);
-    this.containerReportInterval = config.getTimeDuration(
-        OzoneConfigKeys.OZONE_CONTAINER_REPORT_INTERVAL,
-        OzoneConfigKeys.OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT,
-        TimeUnit.MILLISECONDS);
-    this.heartbeatInterval = OzoneClientUtils.getScmHeartbeatInterval(config);
-  }
-
-  public ReportState getContainerReportState() {
-    if (lastContainerReportTime < 0) {
-      return getFullContainerReportState();
-    } else {
-      // Add a random delay (0~30s) on top of the container report
-      // interval (60s) so tha the SCM is overwhelmed by the container reports
-      // sent in sync.
-      if (Time.monotonicNow() - lastContainerReportTime >
-          (containerReportInterval + getRandomReportDelay())) {
-        return getFullContainerReportState();
-      } else {
-        return getNoContainerReportState();
-      }
-    }
-  }
-
-  private ReportState getFullContainerReportState() {
-    ReportState.Builder rsBuilder = ReportState.newBuilder();
-    rsBuilder.setState(ReportState.states.completeContinerReport);
-    rsBuilder.setCount(reportCount.incrementAndGet());
-    this.lastContainerReportTime = Time.monotonicNow();
-    return rsBuilder.build();
-  }
-
-  private ReportState getNoContainerReportState() {
-    return NO_CONTAINER_REPORTSTATE;
-  }
-
-  private long getRandomReportDelay() {
-    return RandomUtils.nextLong(0, heartbeatInterval);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStatus.java
deleted file mode 100644
index 5577323..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStatus.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * This class represents the state of a container. if the
- * container reading encountered an error when we boot up we will post that
- * info to a recovery queue and keep the info in the containerMap.
- * <p/>
- * if and when the issue is fixed, the expectation is that this entry will be
- * deleted by the recovery thread from the containerMap and will insert entry
- * instead of modifying this class.
- */
-public class ContainerStatus {
-  private final ContainerData containerData;
-
-  /**
-   * Number of pending deletion blocks in container.
-   */
-  private int numPendingDeletionBlocks;
-
-  private AtomicLong readBytes;
-
-  private AtomicLong writeBytes;
-
-  private AtomicLong readCount;
-
-  private AtomicLong writeCount;
-
-  /**
-   * Creates a Container Status class.
-   *
-   * @param containerData - ContainerData.
-   */
-  ContainerStatus(ContainerData containerData) {
-    this.numPendingDeletionBlocks = 0;
-    this.containerData = containerData;
-    this.readCount = new AtomicLong(0L);
-    this.readBytes =  new AtomicLong(0L);
-    this.writeCount =  new AtomicLong(0L);
-    this.writeBytes =  new AtomicLong(0L);
-  }
-
-  /**
-   * Returns container if it is active. It is not active if we have had an
-   * error and we are waiting for the background threads to fix the issue.
-   *
-   * @return ContainerData.
-   */
-  public ContainerData getContainer() {
-    return containerData;
-  }
-
-  /**
-   * Increase the count of pending deletion blocks.
-   *
-   * @param numBlocks increment number
-   */
-  public void incrPendingDeletionBlocks(int numBlocks) {
-    this.numPendingDeletionBlocks += numBlocks;
-  }
-
-  /**
-   * Decrease the count of pending deletion blocks.
-   *
-   * @param numBlocks decrement number
-   */
-  public void decrPendingDeletionBlocks(int numBlocks) {
-    this.numPendingDeletionBlocks -= numBlocks;
-  }
-
-  /**
-   * Get the number of pending deletion blocks.
-   */
-  public int getNumPendingDeletionBlocks() {
-    return this.numPendingDeletionBlocks;
-  }
-
-  /**
-   * Get the number of bytes read from the container.
-   * @return the number of bytes read from the container.
-   */
-  public long getReadBytes() {
-    return readBytes.get();
-  }
-
-  /**
-   * Increase the number of bytes read from the container.
-   * @param bytes number of bytes read.
-   */
-  public void incrReadBytes(long bytes) {
-    this.readBytes.addAndGet(bytes);
-  }
-
-  /**
-   * Get the number of times the container is read.
-   * @return the number of times the container is read.
-   */
-  public long getReadCount() {
-    return readCount.get();
-  }
-
-  /**
-   * Increase the number of container read count by 1.
-   */
-  public void incrReadCount() {
-    this.readCount.incrementAndGet();
-  }
-
-  /**
-   * Get the number of bytes write into the container.
-   * @return the number of bytes write into the container.
-   */
-  public long getWriteBytes() {
-    return writeBytes.get();
-  }
-
-  /**
-   * Increase the number of bytes write into the container.
-   * @param bytes the number of bytes write into the container.
-   */
-  public void incrWriteBytes(long bytes) {
-    this.writeBytes.addAndGet(bytes);
-  }
-
-  /**
-   * Get the number of writes into the container.
-   * @return the number of writes into the container.
-   */
-  public long getWriteCount() {
-    return writeCount.get();
-  }
-
-  /**
-   * Increase the number of writes into the container by 1.
-   */
-  public void incrWriteCount() {
-    this.writeCount.incrementAndGet();
-  }
-
-  /**
-   * Get the number of bytes used by the container.
-   * @return the number of bytes used by the container.
-   */
-  public long getBytesUsed() {
-    return containerData.getBytesUsed();
-  }
-
-  /**
-   * Increase the number of bytes used by the container.
-   * @param used number of bytes used by the container.
-   * @return the current number of bytes used by the container afert increase.
-   */
-  public long incrBytesUsed(long used) {
-    return containerData.addBytesUsed(used);
-  }
-
-  /**
-   * Set the number of bytes used by the container.
-   * @param used the number of bytes used by the container.
-   */
-  public void setBytesUsed(long used) {
-    containerData.setBytesUsed(used);
-  }
-
-  /**
-   * Decrease the number of bytes used by the container.
-   * @param reclaimed the number of bytes reclaimed from the container.
-   * @return the current number of bytes used by the container after decrease.
-   */
-  public long decrBytesUsed(long reclaimed) {
-    return this.containerData.addBytesUsed(-1L * reclaimed);
-  }
-
-  /**
-   * Get the maximum container size.
-   * @return the maximum container size.
-   */
-  public long getMaxSize() {
-    return containerData.getMaxSize();
-  }
-
-  /**
-   * Set the maximum container size.
-   * @param size the maximum container size.
-   */
-  public void setMaxSize(long size) {
-    this.containerData.setMaxSize(size);
-  }
-
-  /**
-   * Get the number of keys in the container.
-   * @return the number of keys in the container.
-   */
-  public long getNumKeys() {
-    return containerData.getKeyCount();
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java
deleted file mode 100644
index 07a3a53..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.impl;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CachingGetSpaceUsed;
-import org.apache.hadoop.fs.DF;
-import org.apache.hadoop.fs.GetSpaceUsed;
-import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.util.ShutdownHookManager;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.FileNotFoundException;
-import java.io.OutputStreamWriter;
-import java.io.FileOutputStream;
-import java.net.URI;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Paths;
-import java.util.Scanner;
-
-import static org.apache.hadoop.util.RunJar.SHUTDOWN_HOOK_PRIORITY;
-
-/**
- * Class that wraps the space usage of the Datanode Container Storage Location
- * by SCM containers.
- */
-public class ContainerStorageLocation {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ContainerStorageLocation.class);
-
-  private static final String DU_CACHE_FILE = "scmUsed";
-  private volatile boolean scmUsedSaved = false;
-
-  private final StorageLocation dataLocation;
-  private final String storageUuId;
-  private final DF usage;
-  private final GetSpaceUsed scmUsage;
-  private final File scmUsedFile;
-
-  public ContainerStorageLocation(StorageLocation dataLoc, Configuration conf)
-      throws IOException {
-    this.dataLocation = dataLoc;
-    this.storageUuId = DatanodeStorage.generateUuid();
-    File dataDir = Paths.get(dataLoc.getNormalizedUri()).resolve(
-        OzoneConsts.CONTAINER_PREFIX).toFile();
-    // Initialize container data root if it does not exist as required by DF/DU
-    if (!dataDir.exists()) {
-      if (!dataDir.mkdirs()) {
-        LOG.error("Unable to create the container storage location at : {}",
-            dataDir);
-        throw new IllegalArgumentException("Unable to create the container" +
-            " storage location at : " + dataDir);
-      }
-    }
-    scmUsedFile = new File(dataDir, DU_CACHE_FILE);
-    // get overall disk usage
-    this.usage = new DF(dataDir, conf);
-    // get SCM specific usage
-    this.scmUsage = new CachingGetSpaceUsed.Builder().setPath(dataDir)
-        .setConf(conf)
-        .setInitialUsed(loadScmUsed())
-        .build();
-
-    // Ensure scm usage is saved during shutdown.
-    ShutdownHookManager.get().addShutdownHook(
-        new Runnable() {
-          @Override
-          public void run() {
-            if (!scmUsedSaved) {
-              saveScmUsed();
-            }
-          }
-        }, SHUTDOWN_HOOK_PRIORITY);
-  }
-
-  public URI getNormalizedUri() {
-    return dataLocation.getNormalizedUri();
-  }
-
-  public String getStorageUuId() {
-    return storageUuId;
-  }
-  public long getCapacity() {
-    long capacity = usage.getCapacity();
-    return (capacity > 0) ? capacity : 0;
-  }
-
-  public long getAvailable() throws IOException {
-    long remaining = getCapacity() - getScmUsed();
-    long available = usage.getAvailable();
-    if (remaining > available) {
-      remaining = available;
-    }
-    return (remaining > 0) ? remaining : 0;
-  }
-
-  public long getScmUsed() throws IOException{
-    return scmUsage.getUsed();
-  }
-
-  public void shutdown() {
-    saveScmUsed();
-    scmUsedSaved = true;
-
-    if (scmUsage instanceof CachingGetSpaceUsed) {
-      IOUtils.cleanupWithLogger(null, ((CachingGetSpaceUsed) scmUsage));
-    }
-  }
-
-  /**
-   * Read in the cached DU value and return it if it is less than 600 seconds
-   * old (DU update interval). Slight imprecision of scmUsed is not critical
-   * and skipping DU can significantly shorten the startup time.
-   * If the cached value is not available or too old, -1 is returned.
-   */
-  long loadScmUsed() {
-    long cachedScmUsed;
-    long mtime;
-    Scanner sc;
-
-    try {
-      sc = new Scanner(scmUsedFile, "UTF-8");
-    } catch (FileNotFoundException fnfe) {
-      return -1;
-    }
-
-    try {
-      // Get the recorded scmUsed from the file.
-      if (sc.hasNextLong()) {
-        cachedScmUsed = sc.nextLong();
-      } else {
-        return -1;
-      }
-      // Get the recorded mtime from the file.
-      if (sc.hasNextLong()) {
-        mtime = sc.nextLong();
-      } else {
-        return -1;
-      }
-
-      // Return the cached value if mtime is okay.
-      if (mtime > 0 && (Time.now() - mtime < 600000L)) {
-        LOG.info("Cached ScmUsed found for {} : {} ", dataLocation,
-            cachedScmUsed);
-        return cachedScmUsed;
-      }
-      return -1;
-    } finally {
-      sc.close();
-    }
-  }
-
-  /**
-   * Write the current scmUsed to the cache file.
-   */
-  void saveScmUsed() {
-    if (scmUsedFile.exists() && !scmUsedFile.delete()) {
-      LOG.warn("Failed to delete old scmUsed file in {}.", dataLocation);
-    }
-    OutputStreamWriter out = null;
-    try {
-      long used = getScmUsed();
-      if (used > 0) {
-        out = new OutputStreamWriter(new FileOutputStream(scmUsedFile),
-            StandardCharsets.UTF_8);
-        // mtime is written last, so that truncated writes won't be valid.
-        out.write(Long.toString(used) + " " + Long.toString(Time.now()));
-        out.flush();
-        out.close();
-        out = null;
-      }
-    } catch (IOException ioe) {
-      // If write failed, the volume might be bad. Since the cache file is
-      // not critical, log the error and continue.
-      LOG.warn("Failed to write scmUsed to " + scmUsedFile, ioe);
-    } finally {
-      IOUtils.cleanupWithLogger(null, out);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/Dispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/Dispatcher.java
deleted file mode 100644
index 40bc7ba..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/Dispatcher.java
+++ /dev/null
@@ -1,708 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.impl;
-
-import com.google.common.base.Preconditions;
-import com.google.protobuf.ByteString;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .ContainerCommandRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .ContainerCommandResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Type;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkUtils;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.helpers.FileUtils;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
-import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-import java.util.LinkedList;
-import java.util.List;
-
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Result.CLOSED_CONTAINER_IO;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Result.GET_SMALL_FILE_ERROR;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Result.NO_SUCH_ALGORITHM;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Result.PUT_SMALL_FILE_ERROR;
-
-/**
- * Ozone Container dispatcher takes a call from the netty server and routes it
- * to the right handler function.
- */
-public class Dispatcher implements ContainerDispatcher {
-  static final Logger LOG = LoggerFactory.getLogger(Dispatcher.class);
-
-  private final ContainerManager containerManager;
-  private ContainerMetrics metrics;
-  private Configuration conf;
-
-  /**
-   * Constructs an OzoneContainer that receives calls from
-   * XceiverServerHandler.
-   *
-   * @param containerManager - A class that manages containers.
-   */
-  public Dispatcher(ContainerManager containerManager, Configuration config) {
-    Preconditions.checkNotNull(containerManager);
-    this.containerManager = containerManager;
-    this.metrics = null;
-    this.conf = config;
-  }
-
-  @Override
-  public void init() {
-    this.metrics = ContainerMetrics.create(conf);
-  }
-
-  @Override
-  public void shutdown() {
-  }
-
-  @Override
-  public ContainerCommandResponseProto dispatch(
-      ContainerCommandRequestProto msg) {
-    LOG.trace("Command {}, trace ID: {} ", msg.getCmdType().toString(),
-        msg.getTraceID());
-    long startNanos = System.nanoTime();
-    ContainerCommandResponseProto resp = null;
-    try {
-      Preconditions.checkNotNull(msg);
-      Type cmdType = msg.getCmdType();
-      metrics.incContainerOpcMetrics(cmdType);
-      if ((cmdType == Type.CreateContainer) ||
-          (cmdType == Type.DeleteContainer) ||
-          (cmdType == Type.ReadContainer) ||
-          (cmdType == Type.ListContainer) ||
-          (cmdType == Type.UpdateContainer) ||
-          (cmdType == Type.CloseContainer)) {
-        resp = containerProcessHandler(msg);
-      }
-
-      if ((cmdType == Type.PutKey) ||
-          (cmdType == Type.GetKey) ||
-          (cmdType == Type.DeleteKey) ||
-          (cmdType == Type.ListKey)) {
-        resp = keyProcessHandler(msg);
-      }
-
-      if ((cmdType == Type.WriteChunk) ||
-          (cmdType == Type.ReadChunk) ||
-          (cmdType == Type.DeleteChunk)) {
-        resp = chunkProcessHandler(msg);
-      }
-
-      if ((cmdType == Type.PutSmallFile) ||
-          (cmdType == Type.GetSmallFile)) {
-        resp = smallFileHandler(msg);
-      }
-
-      if (resp != null) {
-        metrics.incContainerOpsLatencies(cmdType,
-            System.nanoTime() - startNanos);
-        return resp;
-      }
-
-      return ContainerUtils.unsupportedRequest(msg);
-    } catch (StorageContainerException e) {
-      // This useful since the trace ID will allow us to correlate failures.
-      return ContainerUtils.logAndReturnError(LOG, e, msg);
-    } catch (IllegalStateException | NullPointerException e) {
-      return ContainerUtils.logAndReturnError(LOG, e, msg);
-    }
-  }
-
-  public ContainerMetrics getContainerMetrics() {
-    return metrics;
-  }
-
-  /**
-   * Handles the all Container related functionality.
-   *
-   * @param msg - command
-   * @return - response
-   * @throws StorageContainerException
-   */
-  private ContainerCommandResponseProto containerProcessHandler(
-      ContainerCommandRequestProto msg) throws StorageContainerException {
-    try {
-
-      switch (msg.getCmdType()) {
-      case CreateContainer:
-        return handleCreateContainer(msg);
-
-      case DeleteContainer:
-        return handleDeleteContainer(msg);
-
-      case ListContainer:
-        // TODO : Support List Container.
-        return ContainerUtils.unsupportedRequest(msg);
-
-      case UpdateContainer:
-        return handleUpdateContainer(msg);
-
-      case ReadContainer:
-        return handleReadContainer(msg);
-
-      case CloseContainer:
-        return handleCloseContainer(msg);
-
-      default:
-        return ContainerUtils.unsupportedRequest(msg);
-      }
-    } catch (StorageContainerException e) {
-      return ContainerUtils.logAndReturnError(LOG, e, msg);
-    } catch (IOException ex) {
-      LOG.warn("Container operation failed. " +
-              "Container: {} Operation: {}  trace ID: {} Error: {}",
-          msg.getCreateContainer().getContainerData().getName(),
-          msg.getCmdType().name(),
-          msg.getTraceID(),
-          ex.toString(), ex);
-
-      // TODO : Replace with finer error codes.
-      return ContainerUtils.getContainerResponse(msg,
-          ContainerProtos.Result.CONTAINER_INTERNAL_ERROR,
-          ex.toString()).build();
-    }
-  }
-
-  /**
-   * Handles the all key related functionality.
-   *
-   * @param msg - command
-   * @return - response
-   * @throws StorageContainerException
-   */
-  private ContainerCommandResponseProto keyProcessHandler(
-      ContainerCommandRequestProto msg) throws StorageContainerException {
-    try {
-      switch (msg.getCmdType()) {
-      case PutKey:
-        return handlePutKey(msg);
-
-      case GetKey:
-        return handleGetKey(msg);
-
-      case DeleteKey:
-        return handleDeleteKey(msg);
-
-      case ListKey:
-        return ContainerUtils.unsupportedRequest(msg);
-
-      default:
-        return ContainerUtils.unsupportedRequest(msg);
-
-      }
-    } catch (StorageContainerException e) {
-      return ContainerUtils.logAndReturnError(LOG, e, msg);
-    } catch (IOException ex) {
-      LOG.warn("Container operation failed. " +
-              "Container: {} Operation: {}  trace ID: {} Error: {}",
-          msg.getCreateContainer().getContainerData().getName(),
-          msg.getCmdType().name(),
-          msg.getTraceID(),
-          ex.toString(), ex);
-
-      // TODO : Replace with finer error codes.
-      return ContainerUtils.getContainerResponse(msg,
-          ContainerProtos.Result.CONTAINER_INTERNAL_ERROR,
-          ex.toString()).build();
-    }
-  }
-
-  /**
-   * Handles the all chunk related functionality.
-   *
-   * @param msg - command
-   * @return - response
-   * @throws StorageContainerException
-   */
-  private ContainerCommandResponseProto chunkProcessHandler(
-      ContainerCommandRequestProto msg) throws StorageContainerException {
-    try {
-      switch (msg.getCmdType()) {
-      case WriteChunk:
-        return handleWriteChunk(msg);
-
-      case ReadChunk:
-        return handleReadChunk(msg);
-
-      case DeleteChunk:
-        return handleDeleteChunk(msg);
-
-      case ListChunk:
-        return ContainerUtils.unsupportedRequest(msg);
-
-      default:
-        return ContainerUtils.unsupportedRequest(msg);
-      }
-    } catch (StorageContainerException e) {
-      return ContainerUtils.logAndReturnError(LOG, e, msg);
-    } catch (IOException ex) {
-      LOG.warn("Container operation failed. " +
-              "Container: {} Operation: {}  trace ID: {} Error: {}",
-          msg.getCreateContainer().getContainerData().getName(),
-          msg.getCmdType().name(),
-          msg.getTraceID(),
-          ex.toString(), ex);
-
-      // TODO : Replace with finer error codes.
-      return ContainerUtils.getContainerResponse(msg,
-          ContainerProtos.Result.CONTAINER_INTERNAL_ERROR,
-          ex.toString()).build();
-    }
-  }
-
-  /**
-   * Dispatch calls to small file hanlder.
-   * @param msg - request
-   * @return response
-   * @throws StorageContainerException
-   */
-  private ContainerCommandResponseProto smallFileHandler(
-      ContainerCommandRequestProto msg) throws StorageContainerException {
-    switch (msg.getCmdType()) {
-    case PutSmallFile:
-      return handlePutSmallFile(msg);
-    case GetSmallFile:
-      return handleGetSmallFile(msg);
-    default:
-      return ContainerUtils.unsupportedRequest(msg);
-    }
-  }
-
-  /**
-   * Update an existing container with the new container data.
-   *
-   * @param msg Request
-   * @return ContainerCommandResponseProto
-   * @throws IOException
-   */
-  private ContainerCommandResponseProto handleUpdateContainer(
-      ContainerCommandRequestProto msg)
-      throws IOException {
-    if (!msg.hasUpdateContainer()) {
-      LOG.debug("Malformed read container request. trace ID: {}",
-          msg.getTraceID());
-      return ContainerUtils.malformedRequest(msg);
-    }
-
-    Pipeline pipeline = Pipeline.getFromProtoBuf(
-        msg.getUpdateContainer().getPipeline());
-    String containerName = msg.getUpdateContainer()
-        .getContainerData().getName();
-
-    ContainerData data = ContainerData.getFromProtBuf(
-        msg.getUpdateContainer().getContainerData(), conf);
-    boolean forceUpdate = msg.getUpdateContainer().getForceUpdate();
-    this.containerManager.updateContainer(
-        pipeline, containerName, data, forceUpdate);
-    return ContainerUtils.getContainerResponse(msg);
-  }
-
-  /**
-   * Calls into container logic and returns appropriate response.
-   *
-   * @param msg - Request
-   * @return ContainerCommandResponseProto
-   * @throws IOException
-   */
-  private ContainerCommandResponseProto handleReadContainer(
-      ContainerCommandRequestProto msg)
-      throws IOException {
-
-    if (!msg.hasReadContainer()) {
-      LOG.debug("Malformed read container request. trace ID: {}",
-          msg.getTraceID());
-      return ContainerUtils.malformedRequest(msg);
-    }
-
-    String name = msg.getReadContainer().getName();
-    ContainerData container = this.containerManager.readContainer(name);
-    return ContainerUtils.getReadContainerResponse(msg, container);
-  }
-
-  /**
-   * Calls into container logic and returns appropriate response.
-   *
-   * @param msg - Request
-   * @return Response.
-   * @throws IOException
-   */
-  private ContainerCommandResponseProto handleDeleteContainer(
-      ContainerCommandRequestProto msg) throws IOException {
-
-    if (!msg.hasDeleteContainer()) {
-      LOG.debug("Malformed delete container request. trace ID: {}",
-          msg.getTraceID());
-      return ContainerUtils.malformedRequest(msg);
-    }
-
-    Pipeline pipeline = Pipeline.getFromProtoBuf(
-        msg.getDeleteContainer().getPipeline());
-    Preconditions.checkNotNull(pipeline);
-    String name = msg.getDeleteContainer().getName();
-    boolean forceDelete = msg.getDeleteContainer().getForceDelete();
-    this.containerManager.deleteContainer(pipeline, name, forceDelete);
-    return ContainerUtils.getContainerResponse(msg);
-  }
-
-  /**
-   * Calls into container logic and returns appropriate response.
-   *
-   * @param msg - Request
-   * @return Response.
-   * @throws IOException
-   */
-  private ContainerCommandResponseProto handleCreateContainer(
-      ContainerCommandRequestProto msg) throws IOException {
-    if (!msg.hasCreateContainer()) {
-      LOG.debug("Malformed create container request. trace ID: {}",
-          msg.getTraceID());
-      return ContainerUtils.malformedRequest(msg);
-    }
-    ContainerData cData = ContainerData.getFromProtBuf(
-        msg.getCreateContainer().getContainerData(), conf);
-    Preconditions.checkNotNull(cData, "Container data is null");
-
-    Pipeline pipeline = Pipeline.getFromProtoBuf(
-        msg.getCreateContainer().getPipeline());
-    Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
-
-    this.containerManager.createContainer(pipeline, cData);
-    return ContainerUtils.getContainerResponse(msg);
-  }
-
-  /**
-   * closes an open container.
-   *
-   * @param msg -
-   * @return
-   * @throws IOException
-   */
-  private ContainerCommandResponseProto handleCloseContainer(
-      ContainerCommandRequestProto msg) throws IOException {
-    try {
-      if (!msg.hasCloseContainer()) {
-        LOG.debug("Malformed close Container request. trace ID: {}",
-            msg.getTraceID());
-        return ContainerUtils.malformedRequest(msg);
-      }
-      Pipeline pipeline = Pipeline.getFromProtoBuf(msg.getCloseContainer()
-          .getPipeline());
-      Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
-      if (!this.containerManager.isOpen(pipeline.getContainerName())) {
-        throw new StorageContainerException("Attempting to close a closed " +
-            "container.", CLOSED_CONTAINER_IO);
-      }
-      this.containerManager.closeContainer(pipeline.getContainerName());
-      return ContainerUtils.getContainerResponse(msg);
-    } catch (NoSuchAlgorithmException e) {
-      throw new StorageContainerException("No such Algorithm", e,
-          NO_SUCH_ALGORITHM);
-    }
-  }
-
-  /**
-   * Calls into chunk manager to write a chunk.
-   *
-   * @param msg - Request.
-   * @return Response.
-   * @throws IOException
-   */
-  private ContainerCommandResponseProto handleWriteChunk(
-      ContainerCommandRequestProto msg) throws IOException {
-    if (!msg.hasWriteChunk()) {
-      LOG.debug("Malformed write chunk request. trace ID: {}",
-          msg.getTraceID());
-      return ContainerUtils.malformedRequest(msg);
-    }
-    String keyName = msg.getWriteChunk().getKeyName();
-    Pipeline pipeline = Pipeline.getFromProtoBuf(
-        msg.getWriteChunk().getPipeline());
-    Preconditions.checkNotNull(pipeline);
-    if (!this.containerManager.isOpen(pipeline.getContainerName())) {
-      throw new StorageContainerException("Write to closed container.",
-          CLOSED_CONTAINER_IO);
-    }
-
-    ChunkInfo chunkInfo = ChunkInfo.getFromProtoBuf(msg.getWriteChunk()
-        .getChunkData());
-    Preconditions.checkNotNull(chunkInfo);
-    byte[] data = null;
-    if (msg.getWriteChunk().getStage() == ContainerProtos.Stage.WRITE_DATA
-        || msg.getWriteChunk().getStage() == ContainerProtos.Stage.COMBINED) {
-       data = msg.getWriteChunk().getData().toByteArray();
-      metrics.incContainerBytesStats(Type.WriteChunk, data.length);
-
-    }
-    this.containerManager.getChunkManager()
-        .writeChunk(pipeline, keyName, chunkInfo,
-            data, msg.getWriteChunk().getStage());
-
-    return ChunkUtils.getChunkResponse(msg);
-  }
-
-  /**
-   * Calls into chunk manager to read a chunk.
-   *
-   * @param msg - Request.
-   * @return - Response.
-   * @throws IOException
-   */
-  private ContainerCommandResponseProto handleReadChunk(
-      ContainerCommandRequestProto msg) throws IOException {
-    if (!msg.hasReadChunk()) {
-      LOG.debug("Malformed read chunk request. trace ID: {}",
-          msg.getTraceID());
-      return ContainerUtils.malformedRequest(msg);
-    }
-
-    String keyName = msg.getReadChunk().getKeyName();
-    Pipeline pipeline = Pipeline.getFromProtoBuf(
-        msg.getReadChunk().getPipeline());
-    Preconditions.checkNotNull(pipeline);
-
-    ChunkInfo chunkInfo = ChunkInfo.getFromProtoBuf(msg.getReadChunk()
-        .getChunkData());
-    Preconditions.checkNotNull(chunkInfo);
-    byte[] data = this.containerManager.getChunkManager().readChunk(pipeline,
-        keyName, chunkInfo);
-    metrics.incContainerBytesStats(Type.ReadChunk, data.length);
-    return ChunkUtils.getReadChunkResponse(msg, data, chunkInfo);
-  }
-
-  /**
-   * Calls into chunk manager to write a chunk.
-   *
-   * @param msg - Request.
-   * @return Response.
-   * @throws IOException
-   */
-  private ContainerCommandResponseProto handleDeleteChunk(
-      ContainerCommandRequestProto msg) throws IOException {
-    if (!msg.hasDeleteChunk()) {
-      LOG.debug("Malformed delete chunk request. trace ID: {}",
-          msg.getTraceID());
-      return ContainerUtils.malformedRequest(msg);
-    }
-
-    String keyName = msg.getDeleteChunk().getKeyName();
-    Pipeline pipeline = Pipeline.getFromProtoBuf(
-        msg.getDeleteChunk().getPipeline());
-    Preconditions.checkNotNull(pipeline);
-    if (!this.containerManager.isOpen(pipeline.getContainerName())) {
-      throw new StorageContainerException("Write to closed container.",
-          CLOSED_CONTAINER_IO);
-    }
-    ChunkInfo chunkInfo = ChunkInfo.getFromProtoBuf(msg.getDeleteChunk()
-        .getChunkData());
-    Preconditions.checkNotNull(chunkInfo);
-
-    this.containerManager.getChunkManager().deleteChunk(pipeline, keyName,
-        chunkInfo);
-    return ChunkUtils.getChunkResponse(msg);
-  }
-
-  /**
-   * Put Key handler.
-   *
-   * @param msg - Request.
-   * @return - Response.
-   * @throws IOException
-   */
-  private ContainerCommandResponseProto handlePutKey(
-      ContainerCommandRequestProto msg) throws IOException {
-    if (!msg.hasPutKey()) {
-      LOG.debug("Malformed put key request. trace ID: {}",
-          msg.getTraceID());
-      return ContainerUtils.malformedRequest(msg);
-    }
-    Pipeline pipeline = Pipeline.getFromProtoBuf(msg.getPutKey().getPipeline());
-    Preconditions.checkNotNull(pipeline);
-    if (!this.containerManager.isOpen(pipeline.getContainerName())) {
-      throw new StorageContainerException("Write to closed container.",
-          CLOSED_CONTAINER_IO);
-    }
-    KeyData keyData = KeyData.getFromProtoBuf(msg.getPutKey().getKeyData());
-    Preconditions.checkNotNull(keyData);
-    this.containerManager.getKeyManager().putKey(pipeline, keyData);
-    long numBytes = keyData.getProtoBufMessage().toByteArray().length;
-    metrics.incContainerBytesStats(Type.PutKey, numBytes);
-    return KeyUtils.getKeyResponse(msg);
-  }
-
-  /**
-   * Handle Get Key.
-   *
-   * @param msg - Request.
-   * @return - Response.
-   * @throws IOException
-   */
-  private ContainerCommandResponseProto handleGetKey(
-      ContainerCommandRequestProto msg) throws IOException {
-    if (!msg.hasGetKey()) {
-      LOG.debug("Malformed get key request. trace ID: {}",
-          msg.getTraceID());
-      return ContainerUtils.malformedRequest(msg);
-    }
-    KeyData keyData = KeyData.getFromProtoBuf(msg.getGetKey().getKeyData());
-    Preconditions.checkNotNull(keyData);
-    KeyData responseData =
-        this.containerManager.getKeyManager().getKey(keyData);
-    long numBytes = responseData.getProtoBufMessage().toByteArray().length;
-    metrics.incContainerBytesStats(Type.GetKey, numBytes);
-    return KeyUtils.getKeyDataResponse(msg, responseData);
-  }
-
-  /**
-   * Handle Delete Key.
-   *
-   * @param msg - Request.
-   * @return - Response.
-   * @throws IOException
-   */
-  private ContainerCommandResponseProto handleDeleteKey(
-      ContainerCommandRequestProto msg) throws IOException {
-    if (!msg.hasDeleteKey()) {
-      LOG.debug("Malformed delete key request. trace ID: {}",
-          msg.getTraceID());
-      return ContainerUtils.malformedRequest(msg);
-    }
-    Pipeline pipeline =
-        Pipeline.getFromProtoBuf(msg.getDeleteKey().getPipeline());
-    Preconditions.checkNotNull(pipeline);
-    if (!this.containerManager.isOpen(pipeline.getContainerName())) {
-      throw new StorageContainerException("Write to closed container.",
-          CLOSED_CONTAINER_IO);
-    }
-    String keyName = msg.getDeleteKey().getName();
-    Preconditions.checkNotNull(keyName);
-    Preconditions.checkState(!keyName.isEmpty());
-    this.containerManager.getKeyManager().deleteKey(pipeline, keyName);
-    return KeyUtils.getKeyResponse(msg);
-  }
-
-  /**
-   * Handles writing a chunk and associated key using single RPC.
-   *
-   * @param msg - Message.
-   * @return ContainerCommandResponseProto
-   * @throws StorageContainerException
-   */
-  private ContainerCommandResponseProto handlePutSmallFile(
-      ContainerCommandRequestProto msg) throws StorageContainerException {
-
-    if (!msg.hasPutSmallFile()) {
-      LOG.debug("Malformed put small file request. trace ID: {}",
-          msg.getTraceID());
-      return ContainerUtils.malformedRequest(msg);
-    }
-    try {
-
-      Pipeline pipeline =
-          Pipeline.getFromProtoBuf(msg.getPutSmallFile()
-              .getKey().getPipeline());
-
-      Preconditions.checkNotNull(pipeline);
-      if (!this.containerManager.isOpen(pipeline.getContainerName())) {
-        throw new StorageContainerException("Write to closed container.",
-            CLOSED_CONTAINER_IO);
-      }
-      KeyData keyData = KeyData.getFromProtoBuf(msg.getPutSmallFile().getKey()
-          .getKeyData());
-      ChunkInfo chunkInfo = ChunkInfo.getFromProtoBuf(msg.getPutSmallFile()
-          .getChunkInfo());
-      byte[] data = msg.getPutSmallFile().getData().toByteArray();
-
-      metrics.incContainerBytesStats(Type.PutSmallFile, data.length);
-      this.containerManager.getChunkManager().writeChunk(pipeline, keyData
-          .getKeyName(), chunkInfo, data, ContainerProtos.Stage.COMBINED);
-      List<ContainerProtos.ChunkInfo> chunks = new LinkedList<>();
-      chunks.add(chunkInfo.getProtoBufMessage());
-      keyData.setChunks(chunks);
-      this.containerManager.getKeyManager().putKey(pipeline, keyData);
-      return FileUtils.getPutFileResponse(msg);
-    } catch (StorageContainerException e) {
-      return ContainerUtils.logAndReturnError(LOG, e, msg);
-    } catch (IOException e) {
-      throw new StorageContainerException("Put Small File Failed.", e,
-          PUT_SMALL_FILE_ERROR);
-    }
-  }
-
-  /**
-   * Handles getting a data stream using a key. This helps in reducing the RPC
-   * overhead for small files.
-   *
-   * @param msg - ContainerCommandRequestProto
-   * @return ContainerCommandResponseProto
-   * @throws StorageContainerException
-   */
-  private ContainerCommandResponseProto handleGetSmallFile(
-      ContainerCommandRequestProto msg) throws StorageContainerException {
-    ByteString dataBuf = ByteString.EMPTY;
-    if (!msg.hasGetSmallFile()) {
-      LOG.debug("Malformed get small file request. trace ID: {}",
-          msg.getTraceID());
-      return ContainerUtils.malformedRequest(msg);
-    }
-    try {
-      Pipeline pipeline =
-          Pipeline.getFromProtoBuf(msg.getGetSmallFile()
-              .getKey().getPipeline());
-
-      long bytes = 0;
-      Preconditions.checkNotNull(pipeline);
-      KeyData keyData = KeyData.getFromProtoBuf(msg.getGetSmallFile()
-          .getKey().getKeyData());
-      KeyData data = this.containerManager.getKeyManager().getKey(keyData);
-      ContainerProtos.ChunkInfo c = null;
-      for (ContainerProtos.ChunkInfo chunk : data.getChunks()) {
-        bytes += chunk.getSerializedSize();
-        ByteString current =
-            ByteString.copyFrom(this.containerManager.getChunkManager()
-                .readChunk(
-                    pipeline, keyData.getKeyName(), ChunkInfo.getFromProtoBuf(
-                        chunk)));
-        dataBuf = dataBuf.concat(current);
-        c = chunk;
-      }
-      metrics.incContainerBytesStats(Type.GetSmallFile, bytes);
-      return FileUtils.getGetSmallFileResponse(msg, dataBuf.toByteArray(),
-          ChunkInfo.getFromProtoBuf(c));
-    } catch (StorageContainerException e) {
-      return ContainerUtils.logAndReturnError(LOG, e, msg);
-    } catch (IOException e) {
-      throw new StorageContainerException("Get Small File Failed", e,
-          GET_SMALL_FILE_ERROR);
-    }
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[25/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java
deleted file mode 100644
index 09d9f32..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-/*
- This package contains the keyspace manager classes.
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/Lease.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/Lease.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/Lease.java
deleted file mode 100644
index dfa9315..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/Lease.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.lease;
-
-import org.apache.hadoop.util.Time;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.Callable;
-
-/**
- * This class represents the lease created on a resource. Callback can be
- * registered on the lease which will be executed in case of timeout.
- *
- * @param <T> Resource type for which the lease can be associated
- */
-public class Lease<T> {
-
-  /**
-   * The resource for which this lease is created.
-   */
-  private final T resource;
-
-  private final long creationTime;
-
-  /**
-   * Lease lifetime in milliseconds.
-   */
-  private volatile long leaseTimeout;
-
-  private boolean expired;
-
-  /**
-   * Functions to be called in case of timeout.
-   */
-  private List<Callable<Void>> callbacks;
-
-
-  /**
-   * Creates a lease on the specified resource with given timeout.
-   *
-   * @param resource
-   *        Resource for which the lease has to be created
-   * @param timeout
-   *        Lease lifetime in milliseconds
-   */
-  public Lease(T resource, long timeout) {
-    this.resource = resource;
-    this.leaseTimeout = timeout;
-    this.callbacks = Collections.synchronizedList(new ArrayList<>());
-    this.creationTime = Time.monotonicNow();
-    this.expired = false;
-  }
-
-  /**
-   * Returns true if the lease has expired, else false.
-   *
-   * @return true if expired, else false
-   */
-  public boolean hasExpired() {
-    return expired;
-  }
-
-  /**
-   * Registers a callback which will be executed in case of timeout. Callbacks
-   * are executed in a separate Thread.
-   *
-   * @param callback
-   *        The Callable which has to be executed
-   * @throws LeaseExpiredException
-   *         If the lease has already timed out
-   */
-  public void registerCallBack(Callable<Void> callback)
-      throws LeaseExpiredException {
-    if(hasExpired()) {
-      throw new LeaseExpiredException("Resource: " + resource);
-    }
-    callbacks.add(callback);
-  }
-
-  /**
-   * Returns the time elapsed since the creation of lease.
-   *
-   * @return elapsed time in milliseconds
-   * @throws LeaseExpiredException
-   *         If the lease has already timed out
-   */
-  public long getElapsedTime() throws LeaseExpiredException {
-    if(hasExpired()) {
-      throw new LeaseExpiredException("Resource: " + resource);
-    }
-    return Time.monotonicNow() - creationTime;
-  }
-
-  /**
-   * Returns the time available before timeout.
-   *
-   * @return remaining time in milliseconds
-   * @throws LeaseExpiredException
-   *         If the lease has already timed out
-   */
-  public long getRemainingTime() throws LeaseExpiredException {
-    if(hasExpired()) {
-      throw new LeaseExpiredException("Resource: " + resource);
-    }
-    return leaseTimeout - getElapsedTime();
-  }
-
-  /**
-   * Returns total lease lifetime.
-   *
-   * @return total lifetime of lease in milliseconds
-   * @throws LeaseExpiredException
-   *         If the lease has already timed out
-   */
-  public long getLeaseLifeTime() throws LeaseExpiredException {
-    if(hasExpired()) {
-      throw new LeaseExpiredException("Resource: " + resource);
-    }
-    return leaseTimeout;
-  }
-
-  /**
-   * Renews the lease timeout period.
-   *
-   * @param timeout
-   *        Time to be added to the lease in milliseconds
-   * @throws LeaseExpiredException
-   *         If the lease has already timed out
-   */
-  public void renew(long timeout) throws LeaseExpiredException {
-    if(hasExpired()) {
-      throw new LeaseExpiredException("Resource: " + resource);
-    }
-    leaseTimeout += timeout;
-  }
-
-  @Override
-  public int hashCode() {
-    return resource.hashCode();
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if(obj instanceof Lease) {
-      return resource.equals(((Lease) obj).resource);
-    }
-    return false;
-  }
-
-  @Override
-  public String toString() {
-    return "Lease<" + resource.toString() + ">";
-  }
-
-  /**
-   * Returns the callbacks to be executed for the lease in case of timeout.
-   *
-   * @return callbacks to be executed
-   */
-  List<Callable<Void>> getCallbacks() {
-    return callbacks;
-  }
-
-  /**
-   * Expires/Invalidates the lease.
-   */
-  void invalidate() {
-    callbacks = null;
-    expired = true;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseAlreadyExistException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseAlreadyExistException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseAlreadyExistException.java
deleted file mode 100644
index a39ea22..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseAlreadyExistException.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.lease;
-
-/**
- * This exception represents that there is already a lease acquired on the
- * same resource.
- */
-public class LeaseAlreadyExistException  extends LeaseException {
-
-  /**
-   * Constructs an {@code LeaseAlreadyExistException} with {@code null}
-   * as its error detail message.
-   */
-  public LeaseAlreadyExistException() {
-    super();
-  }
-
-  /**
-   * Constructs an {@code LeaseAlreadyExistException} with the specified
-   * detail message.
-   *
-   * @param message
-   *        The detail message (which is saved for later retrieval
-   *        by the {@link #getMessage()} method)
-   */
-  public LeaseAlreadyExistException(String message) {
-    super(message);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseCallbackExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseCallbackExecutor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseCallbackExecutor.java
deleted file mode 100644
index 1b7391b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseCallbackExecutor.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.lease;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.concurrent.Callable;
-
-/**
- * This class is responsible for executing the callbacks of a lease in case of
- * timeout.
- */
-public class LeaseCallbackExecutor<T> implements Runnable {
-
-  private static final Logger LOG = LoggerFactory.getLogger(Lease.class);
-
-  private final T resource;
-  private final List<Callable<Void>> callbacks;
-
-  /**
-   * Constructs LeaseCallbackExecutor instance with list of callbacks.
-   *
-   * @param resource
-   *        The resource for which the callbacks are executed
-   * @param callbacks
-   *        Callbacks to be executed by this executor
-   */
-  public LeaseCallbackExecutor(T resource, List<Callable<Void>> callbacks) {
-    this.resource = resource;
-    this.callbacks = callbacks;
-  }
-
-  @Override
-  public void run() {
-    if(LOG.isDebugEnabled()) {
-      LOG.debug("Executing callbacks for lease on {}", resource);
-    }
-    for(Callable<Void> callback : callbacks) {
-      try {
-        callback.call();
-      } catch (Exception e) {
-        LOG.warn("Exception while executing callback for lease on {}",
-            resource, e);
-      }
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseException.java
deleted file mode 100644
index 418f412..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseException.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.lease;
-
-/**
- * This exception represents all lease related exceptions.
- */
-public class LeaseException extends Exception {
-
-  /**
-   * Constructs an {@code LeaseException} with {@code null}
-   * as its error detail message.
-   */
-  public LeaseException() {
-    super();
-  }
-
-  /**
-   * Constructs an {@code LeaseException} with the specified
-   * detail message.
-   *
-   * @param message
-   *        The detail message (which is saved for later retrieval
-   *        by the {@link #getMessage()} method)
-   */
-  public LeaseException(String message) {
-    super(message);
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseExpiredException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseExpiredException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseExpiredException.java
deleted file mode 100644
index 440a023..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseExpiredException.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.lease;
-
-/**
- * This exception represents that the lease that is being accessed has expired.
- */
-public class LeaseExpiredException extends LeaseException {
-
-  /**
-   * Constructs an {@code LeaseExpiredException} with {@code null}
-   * as its error detail message.
-   */
-  public LeaseExpiredException() {
-    super();
-  }
-
-  /**
-   * Constructs an {@code LeaseExpiredException} with the specified
-   * detail message.
-   *
-   * @param message
-   *        The detail message (which is saved for later retrieval
-   *        by the {@link #getMessage()} method)
-   */
-  public LeaseExpiredException(String message) {
-    super(message);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseManager.java
deleted file mode 100644
index b8390dd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseManager.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.lease;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-/**
- * LeaseManager is someone who can provide you leases based on your
- * requirement. If you want to return the lease back before it expires,
- * you can give it back to Lease Manager. He is the one responsible for
- * the lifecycle of leases. The resource for which lease is created
- * should have proper {@code equals} method implementation, resource
- * equality is checked while the lease is created.
- *
- * @param <T> Type of leases that this lease manager can create
- */
-public class LeaseManager<T> {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(LeaseManager.class);
-
-  private final long defaultTimeout;
-  private Map<T, Lease<T>> activeLeases;
-  private LeaseMonitor leaseMonitor;
-  private Thread leaseMonitorThread;
-  private boolean isRunning;
-
-  /**
-   * Creates an instance of lease manager.
-   *
-   * @param defaultTimeout
-   *        Default timeout in milliseconds to be used for lease creation.
-   */
-  public LeaseManager(long defaultTimeout) {
-    this.defaultTimeout = defaultTimeout;
-  }
-
-  /**
-   * Starts the lease manager service.
-   */
-  public void start() {
-    LOG.debug("Starting LeaseManager service");
-    activeLeases = new ConcurrentHashMap<>();
-    leaseMonitor = new LeaseMonitor();
-    leaseMonitorThread = new Thread(leaseMonitor);
-    leaseMonitorThread.setName("LeaseManager#LeaseMonitor");
-    leaseMonitorThread.setDaemon(true);
-    leaseMonitorThread.setUncaughtExceptionHandler((thread, throwable) -> {
-      // Let us just restart this thread after logging an error.
-      // if this thread is not running we cannot handle Lease expiry.
-      LOG.error("LeaseMonitor thread encountered an error. Thread: {}",
-          thread.toString(), throwable);
-      leaseMonitorThread.start();
-    });
-    LOG.debug("Starting LeaseManager#LeaseMonitor Thread");
-    leaseMonitorThread.start();
-    isRunning = true;
-  }
-
-  /**
-   * Returns a lease for the specified resource with default timeout.
-   *
-   * @param resource
-   *        Resource for which lease has to be created
-   * @throws LeaseAlreadyExistException
-   *         If there is already a lease on the resource
-   */
-  public synchronized Lease<T> acquire(T resource)
-      throws LeaseAlreadyExistException {
-    return acquire(resource, defaultTimeout);
-  }
-
-  /**
-   * Returns a lease for the specified resource with the timeout provided.
-   *
-   * @param resource
-   *        Resource for which lease has to be created
-   * @param timeout
-   *        The timeout in milliseconds which has to be set on the lease
-   * @throws LeaseAlreadyExistException
-   *         If there is already a lease on the resource
-   */
-  public synchronized Lease<T> acquire(T resource, long timeout)
-      throws LeaseAlreadyExistException {
-    checkStatus();
-    if(LOG.isDebugEnabled()) {
-      LOG.debug("Acquiring lease on {} for {} milliseconds", resource, timeout);
-    }
-    if(activeLeases.containsKey(resource)) {
-      throw new LeaseAlreadyExistException("Resource: " + resource);
-    }
-    Lease<T> lease = new Lease<>(resource, timeout);
-    activeLeases.put(resource, lease);
-    leaseMonitorThread.interrupt();
-    return lease;
-  }
-
-  /**
-   * Returns a lease associated with the specified resource.
-   *
-   * @param resource
-   *        Resource for which the lease has to be returned
-   * @throws LeaseNotFoundException
-   *         If there is no active lease on the resource
-   */
-  public Lease<T> get(T resource) throws LeaseNotFoundException {
-    checkStatus();
-    Lease<T> lease = activeLeases.get(resource);
-    if(lease != null) {
-      return lease;
-    }
-    throw new LeaseNotFoundException("Resource: " + resource);
-  }
-
-  /**
-   * Releases the lease associated with the specified resource.
-   *
-   * @param resource
-   *        The for which the lease has to be released
-   * @throws LeaseNotFoundException
-   *         If there is no active lease on the resource
-   */
-  public synchronized void release(T resource)
-      throws LeaseNotFoundException {
-    checkStatus();
-    if(LOG.isDebugEnabled()) {
-      LOG.debug("Releasing lease on {}", resource);
-    }
-    Lease<T> lease = activeLeases.remove(resource);
-    if(lease == null) {
-      throw new LeaseNotFoundException("Resource: " + resource);
-    }
-    lease.invalidate();
-  }
-
-  /**
-   * Shuts down the LeaseManager and releases the resources. All the active
-   * {@link Lease} will be released (callbacks on leases will not be
-   * executed).
-   */
-  public void shutdown() {
-    checkStatus();
-    LOG.debug("Shutting down LeaseManager service");
-    leaseMonitor.disable();
-    leaseMonitorThread.interrupt();
-    for(T resource : activeLeases.keySet()) {
-      try {
-        release(resource);
-      }  catch(LeaseNotFoundException ex) {
-        //Ignore the exception, someone might have released the lease
-      }
-    }
-    isRunning = false;
-  }
-
-  /**
-   * Throws {@link LeaseManagerNotRunningException} if the service is not
-   * running.
-   */
-  private void checkStatus() {
-    if(!isRunning) {
-      throw new LeaseManagerNotRunningException("LeaseManager not running.");
-    }
-  }
-
-  /**
-   * Monitors the leases and expires them based on the timeout, also
-   * responsible for executing the callbacks of expired leases.
-   */
-  private final class LeaseMonitor implements Runnable {
-
-    private boolean monitor = true;
-    private ExecutorService executorService;
-
-    private LeaseMonitor() {
-      this.monitor = true;
-      this.executorService = Executors.newCachedThreadPool();
-    }
-
-    @Override
-    public void run() {
-      while(monitor) {
-        LOG.debug("LeaseMonitor: checking for lease expiry");
-        long sleepTime = Long.MAX_VALUE;
-
-        for (T resource : activeLeases.keySet()) {
-          try {
-            Lease<T> lease = get(resource);
-            long remainingTime = lease.getRemainingTime();
-            if (remainingTime <= 0) {
-              //Lease has timed out
-              List<Callable<Void>> leaseCallbacks = lease.getCallbacks();
-              release(resource);
-              executorService.execute(
-                  new LeaseCallbackExecutor(resource, leaseCallbacks));
-            } else {
-              sleepTime = remainingTime > sleepTime ?
-                  sleepTime : remainingTime;
-            }
-          } catch (LeaseNotFoundException | LeaseExpiredException ex) {
-            //Ignore the exception, someone might have released the lease
-          }
-        }
-
-        try {
-          if(!Thread.interrupted()) {
-            Thread.sleep(sleepTime);
-          }
-        } catch (InterruptedException ignored) {
-          // This means a new lease is added to activeLeases.
-        }
-      }
-    }
-
-    /**
-     * Disables lease monitor, next interrupt call on the thread
-     * will stop lease monitor.
-     */
-    public void disable() {
-      monitor = false;
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseManagerNotRunningException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseManagerNotRunningException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseManagerNotRunningException.java
deleted file mode 100644
index ced31de..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseManagerNotRunningException.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.lease;
-
-/**
- * This exception represents that there LeaseManager service is not running.
- */
-public class LeaseManagerNotRunningException  extends RuntimeException {
-
-  /**
-   * Constructs an {@code LeaseManagerNotRunningException} with {@code null}
-   * as its error detail message.
-   */
-  public LeaseManagerNotRunningException() {
-    super();
-  }
-
-  /**
-   * Constructs an {@code LeaseManagerNotRunningException} with the specified
-   * detail message.
-   *
-   * @param message
-   *        The detail message (which is saved for later retrieval
-   *        by the {@link #getMessage()} method)
-   */
-  public LeaseManagerNotRunningException(String message) {
-    super(message);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseNotFoundException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseNotFoundException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseNotFoundException.java
deleted file mode 100644
index c292d33..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseNotFoundException.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.lease;
-
-/**
- * This exception represents that the lease that is being accessed does not
- * exist.
- */
-public class LeaseNotFoundException extends LeaseException {
-
-  /**
-   * Constructs an {@code LeaseNotFoundException} with {@code null}
-   * as its error detail message.
-   */
-  public LeaseNotFoundException() {
-    super();
-  }
-
-  /**
-   * Constructs an {@code LeaseNotFoundException} with the specified
-   * detail message.
-   *
-   * @param message
-   *        The detail message (which is saved for later retrieval
-   *        by the {@link #getMessage()} method)
-   */
-  public LeaseNotFoundException(String message) {
-    super(message);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/package-info.java
deleted file mode 100644
index 48ee2e1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/package-info.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-/**
- * A generic lease management API which can be used if a service
- * needs any kind of lease management.
- */
-
-package org.apache.hadoop.ozone.lease;
-/*
- This package contains lease management related classes.
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/package-info.java
deleted file mode 100644
index db399db..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/package-info.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone;
-
-/**
- This package contains class that support ozone implementation on the datanode
- side.
-
- Main parts of ozone on datanode are:
-
- 1. REST Interface - This code lives under the web directory and listens to the
- WebHDFS port.
-
- 2. Datanode container classes: This support persistence of ozone objects on
- datanode. These classes live under container directory.
-
- 3. Client and Shell: We also support a ozone REST client lib, they are under
- web/client and web/ozShell.
-
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
deleted file mode 100644
index ddcc261..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.protocol;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMRegisteredCmdResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKResponseProto;
-import java.io.IOException;
-
-/**
- * The protocol spoken between datanodes and SCM. For specifics please the
- * Protoc file that defines this protocol.
- */
-@InterfaceAudience.Private
-public interface StorageContainerDatanodeProtocol {
-  /**
-   * Returns SCM version.
-   * @return Version info.
-   */
-  SCMVersionResponseProto getVersion(SCMVersionRequestProto versionRequest)
-      throws IOException;
-
-  /**
-   * Used by data node to send a Heartbeat.
-   * @param datanodeID - Datanode ID.
-   * @param nodeReport - node report state
-   * @param reportState - container report state.
-   * @return - SCMHeartbeatResponseProto
-   * @throws IOException
-   */
-  SCMHeartbeatResponseProto sendHeartbeat(DatanodeID datanodeID,
-      SCMNodeReport nodeReport, ReportState reportState) throws IOException;
-
-  /**
-   * Register Datanode.
-   * @param datanodeID - DatanodID.
-   * @param scmAddresses - List of SCMs this datanode is configured to
-   *                     communicate.
-   * @return SCM Command.
-   */
-  SCMRegisteredCmdResponseProto register(DatanodeID datanodeID,
-      String[] scmAddresses) throws IOException;
-
-  /**
-   * Send a container report.
-   * @param reports -- Container report.
-   * @return container reports response.
-   * @throws IOException
-   */
-  ContainerReportsResponseProto sendContainerReport(
-      ContainerReportsRequestProto reports) throws IOException;
-
-  /**
-   * Used by datanode to send block deletion ACK to SCM.
-   * @param request block deletion transactions.
-   * @return block deletion transaction response.
-   * @throws IOException
-   */
-  ContainerBlocksDeletionACKResponseProto sendContainerBlocksDeletionACK(
-      ContainerBlocksDeletionACKProto request) throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
deleted file mode 100644
index f4a4f32..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocol;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-
-import java.util.List;
-
-/**
- * The protocol spoken between datanodes and SCM.
- *
- * Please note that the full protocol spoken between a datanode and SCM is
- * separated into 2 interfaces. One interface that deals with node state and
- * another interface that deals with containers.
- *
- * This interface has functions that deals with the state of datanode.
- */
-@InterfaceAudience.Private
-public interface StorageContainerNodeProtocol {
-  /**
-   * Gets the version info from SCM.
-   * @param versionRequest - version Request.
-   * @return - returns SCM version info and other required information needed
-   * by datanode.
-   */
-  VersionResponse getVersion(SCMVersionRequestProto versionRequest);
-
-  /**
-   * Register the node if the node finds that it is not registered with any SCM.
-   * @param datanodeID - Send datanodeID with Node info, but datanode UUID is
-   *                   empty. Server returns a datanodeID for the given node.
-   * @return  SCMHeartbeatResponseProto
-   */
-  SCMCommand register(DatanodeID datanodeID);
-
-  /**
-   * Send heartbeat to indicate the datanode is alive and doing well.
-   * @param datanodeID - Datanode ID.
-   * @param nodeReport - node report.
-   * @param reportState - container report.
-   * @return SCMheartbeat response list
-   */
-  List<SCMCommand> sendHeartbeat(DatanodeID datanodeID,
-      SCMNodeReport nodeReport, ReportState reportState);
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java
deleted file mode 100644
index 8d8f4f1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java
+++ /dev/null
@@ -1,150 +0,0 @@
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.protocol;
-
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
-
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-/**
- * Version response class.
- */
-public class VersionResponse {
-  private final int version;
-  private final Map<String, String> values;
-
-  /**
-   * Creates a version response class.
-   * @param version
-   * @param values
-   */
-  public VersionResponse(int version, Map<String, String> values) {
-    this.version = version;
-    this.values = values;
-  }
-
-  /**
-   * Creates a version Response class.
-   * @param version
-   */
-  public VersionResponse(int version) {
-    this.version = version;
-    this.values = new HashMap<>();
-  }
-
-  /**
-   * Returns a new Builder.
-   * @return - Builder.
-   */
-  public static Builder newBuilder() {
-    return new Builder();
-  }
-
-  /**
-   * Returns this class from protobuf message.
-   * @param response - SCMVersionResponseProto
-   * @return VersionResponse
-   */
-  public static VersionResponse getFromProtobuf(SCMVersionResponseProto
-                                                    response) {
-    return new VersionResponse(response.getSoftwareVersion(),
-        response.getKeysList().stream()
-            .collect(Collectors.toMap(KeyValue::getKey,
-                KeyValue::getValue)));
-  }
-
-  /**
-   * Adds a value to version Response.
-   * @param key - String
-   * @param value - String
-   */
-  public void put(String key, String value) {
-    if (this.values.containsKey(key)) {
-      throw new IllegalArgumentException("Duplicate key in version response");
-    }
-    values.put(key, value);
-  }
-
-  /**
-   * Return a protobuf message.
-   * @return SCMVersionResponseProto.
-   */
-  public SCMVersionResponseProto getProtobufMessage() {
-
-    List<KeyValue> list = new LinkedList<>();
-    for (Map.Entry<String, String> entry : values.entrySet()) {
-      list.add(KeyValue.newBuilder().setKey(entry.getKey()).
-          setValue(entry.getValue()).build());
-    }
-    return
-        SCMVersionResponseProto.newBuilder()
-            .setSoftwareVersion(this.version)
-            .addAllKeys(list).build();
-  }
-
-  /**
-   * Builder class.
-   */
-  public static class Builder {
-    private int version;
-    private Map<String, String> values;
-
-    Builder() {
-      values = new HashMap<>();
-    }
-
-    /**
-     * Sets the version.
-     * @param ver - version
-     * @return Builder
-     */
-    public Builder setVersion(int ver) {
-      this.version = ver;
-      return this;
-    }
-
-    /**
-     * Adds a value to version Response.
-     * @param key - String
-     * @param value - String
-     */
-    public Builder addValue(String key, String value) {
-      if (this.values.containsKey(key)) {
-        throw new IllegalArgumentException("Duplicate key in version response");
-      }
-      values.put(key, value);
-      return this;
-    }
-
-    /**
-     * Builds the version response.
-     * @return VersionResponse.
-     */
-    public VersionResponse build() {
-      return new VersionResponse(this.version, this.values);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
deleted file mode 100644
index 79aa132..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocol.commands;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMCloseContainerCmdResponseProto;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.Type;
-import static org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.Type.closeContainerCommand;
-
-/**
- * Asks datanode to close a container.
- */
-public class CloseContainerCommand
-    extends SCMCommand<SCMCloseContainerCmdResponseProto> {
-
-  private String containerName;
-
-  public CloseContainerCommand(String containerName) {
-    this.containerName = containerName;
-  }
-
-  /**
-   * Returns the type of this command.
-   *
-   * @return Type
-   */
-  @Override
-  public Type getType() {
-    return closeContainerCommand;
-  }
-
-  /**
-   * Gets the protobuf message of this object.
-   *
-   * @return A protobuf message.
-   */
-  @Override
-  public byte[] getProtoBufMessage() {
-    return getProto().toByteArray();
-  }
-
-  public SCMCloseContainerCmdResponseProto getProto() {
-    return SCMCloseContainerCmdResponseProto.newBuilder()
-        .setContainerName(containerName).build();
-  }
-
-  public static CloseContainerCommand getFromProtobuf(
-      SCMCloseContainerCmdResponseProto closeContainerProto) {
-    Preconditions.checkNotNull(closeContainerProto);
-    return new CloseContainerCommand(closeContainerProto.getContainerName());
-
-  }
-
-  public String getContainerName() {
-    return containerName;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteBlocksCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteBlocksCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteBlocksCommand.java
deleted file mode 100644
index 8e3463d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteBlocksCommand.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocol.commands;
-
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMDeleteBlocksCmdResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.Type;
-
-import java.util.List;
-
-/**
- * A SCM command asks a datanode to delete a number of blocks.
- */
-public class DeleteBlocksCommand extends
-    SCMCommand<SCMDeleteBlocksCmdResponseProto> {
-
-  private List<DeletedBlocksTransaction> blocksTobeDeleted;
-
-
-  public DeleteBlocksCommand(List<DeletedBlocksTransaction> blocks) {
-    this.blocksTobeDeleted = blocks;
-  }
-
-  public List<DeletedBlocksTransaction> blocksTobeDeleted() {
-    return this.blocksTobeDeleted;
-  }
-
-  @Override
-  public Type getType() {
-    return Type.deleteBlocksCommand;
-  }
-
-  @Override
-  public byte[] getProtoBufMessage() {
-    return getProto().toByteArray();
-  }
-
-  public static DeleteBlocksCommand getFromProtobuf(
-      SCMDeleteBlocksCmdResponseProto deleteBlocksProto) {
-    return new DeleteBlocksCommand(deleteBlocksProto
-        .getDeletedBlocksTransactionsList());
-  }
-
-  public SCMDeleteBlocksCmdResponseProto getProto() {
-    return SCMDeleteBlocksCmdResponseProto.newBuilder()
-        .addAllDeletedBlocksTransactions(blocksTobeDeleted).build();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/RegisteredCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/RegisteredCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/RegisteredCommand.java
deleted file mode 100644
index bf430ac..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/RegisteredCommand.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocol.commands;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMRegisteredCmdResponseProto;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMRegisteredCmdResponseProto
-    .ErrorCode;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.Type;
-
-/**
- * Response to Datanode Register call.
- */
-public class RegisteredCommand extends
-    SCMCommand<SCMRegisteredCmdResponseProto> {
-  private String datanodeUUID;
-  private String clusterID;
-  private ErrorCode error;
-
-  public RegisteredCommand(final ErrorCode error, final String datanodeUUID,
-                           final String clusterID) {
-    this.datanodeUUID = datanodeUUID;
-    this.clusterID = clusterID;
-    this.error = error;
-  }
-
-  /**
-   * Returns a new builder.
-   *
-   * @return - Builder
-   */
-  public static Builder newBuilder() {
-    return new Builder();
-  }
-
-  /**
-   * Returns the type of this command.
-   *
-   * @return Type
-   */
-  @Override
-  public Type getType() {
-    return Type.registeredCommand;
-  }
-
-  /**
-   * Returns datanode UUID.
-   *
-   * @return - Datanode ID.
-   */
-  public String getDatanodeUUID() {
-    return datanodeUUID;
-  }
-
-  /**
-   * Returns cluster ID.
-   *
-   * @return -- ClusterID
-   */
-  public String getClusterID() {
-    return clusterID;
-  }
-
-  /**
-   * Returns ErrorCode.
-   *
-   * @return - ErrorCode
-   */
-  public ErrorCode getError() {
-    return error;
-  }
-
-  /**
-   * Gets the protobuf message of this object.
-   *
-   * @return A protobuf message.
-   */
-  @Override
-  public byte[] getProtoBufMessage() {
-    return SCMRegisteredCmdResponseProto.newBuilder()
-        .setClusterID(this.clusterID)
-        .setDatanodeUUID(this.datanodeUUID)
-        .setErrorCode(this.error)
-        .build().toByteArray();
-  }
-
-  /**
-   * A builder class to verify all values are sane.
-   */
-  public static class Builder {
-    private String datanodeUUID;
-    private String clusterID;
-    private ErrorCode error;
-
-    /**
-     * sets UUID.
-     *
-     * @param dnUUID - datanode UUID
-     * @return Builder
-     */
-    public Builder setDatanodeUUID(String dnUUID) {
-      this.datanodeUUID = dnUUID;
-      return this;
-    }
-
-    /**
-     * Create this object from a Protobuf message.
-     *
-     * @param response - RegisteredCmdResponseProto
-     * @return RegisteredCommand
-     */
-    public  RegisteredCommand getFromProtobuf(SCMRegisteredCmdResponseProto
-                                                        response) {
-      Preconditions.checkNotNull(response);
-      return new RegisteredCommand(response.getErrorCode(),
-          response.hasDatanodeUUID() ? response.getDatanodeUUID(): "",
-          response.hasClusterID() ? response.getClusterID(): "");
-    }
-
-    /**
-     * Sets cluster ID.
-     *
-     * @param cluster - clusterID
-     * @return Builder
-     */
-    public Builder setClusterID(String cluster) {
-      this.clusterID = cluster;
-      return this;
-    }
-
-    /**
-     * Sets Error code.
-     *
-     * @param errorCode - error code
-     * @return Builder
-     */
-    public Builder setErrorCode(ErrorCode errorCode) {
-      this.error = errorCode;
-      return this;
-    }
-
-    /**
-     * Build the command object.
-     *
-     * @return RegisteredCommand
-     */
-    public RegisteredCommand build() {
-      if ((this.error == ErrorCode.success) &&
-          (this.datanodeUUID == null || this.datanodeUUID.isEmpty()) ||
-          (this.clusterID == null || this.clusterID.isEmpty())) {
-        throw new IllegalArgumentException("On success, RegisteredCommand " +
-            "needs datanodeUUID and ClusterID.");
-      }
-
-      return new
-          RegisteredCommand(this.error, this.datanodeUUID, this.clusterID);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReregisterCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReregisterCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReregisterCommand.java
deleted file mode 100644
index f87d35b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReregisterCommand.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocol.commands;
-
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.Type;
-
-import static org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMReregisterCmdResponseProto;
-import static org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.Type.reregisterCommand;
-
-/**
- * Informs a datanode to register itself with SCM again.
- */
-public class ReregisterCommand extends
-    SCMCommand<SCMReregisterCmdResponseProto>{
-
-  /**
-   * Returns the type of this command.
-   *
-   * @return Type
-   */
-  @Override
-  public Type getType() {
-    return reregisterCommand;
-  }
-
-  /**
-   * Gets the protobuf message of this object.
-   *
-   * @return A protobuf message.
-   */
-  @Override
-  public byte[] getProtoBufMessage() {
-    return getProto().toByteArray();
-  }
-
-  public SCMReregisterCmdResponseProto getProto() {
-    return SCMReregisterCmdResponseProto
-        .newBuilder()
-        .build();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/SCMCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/SCMCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/SCMCommand.java
deleted file mode 100644
index fe9b12d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/SCMCommand.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocol.commands;
-
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.Type;
-import com.google.protobuf.GeneratedMessage;
-
-/**
- * A class that acts as the base class to convert between Java and SCM
- * commands in protobuf format.
- * @param <T>
- */
-public abstract class SCMCommand<T extends GeneratedMessage> {
-  /**
-   * Returns the type of this command.
-   * @return Type
-   */
-  public  abstract Type getType();
-
-  /**
-   * Gets the protobuf message of this object.
-   * @return A protobuf message.
-   */
-  public abstract byte[] getProtoBufMessage();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/SendContainerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/SendContainerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/SendContainerCommand.java
deleted file mode 100644
index 6a9fc44..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/SendContainerCommand.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.protocol.commands;
-
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SendContainerReportProto;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.Type;
-
-/**
- * Allows a Datanode to send in the container report.
- */
-public class SendContainerCommand extends SCMCommand<SendContainerReportProto> {
-  /**
-   * Returns a NullCommand class from NullCommandResponse Proto.
-   * @param unused  - unused
-   * @return NullCommand
-   */
-  public static SendContainerCommand getFromProtobuf(
-      final SendContainerReportProto unused) {
-    return new SendContainerCommand();
-  }
-
-  /**
-   * returns a new builder.
-   * @return Builder
-   */
-  public static SendContainerCommand.Builder newBuilder() {
-    return new SendContainerCommand.Builder();
-  }
-
-  /**
-   * Returns the type of this command.
-   *
-   * @return Type
-   */
-  @Override
-  public Type getType() {
-    return Type.sendContainerReport;
-  }
-
-  /**
-   * Gets the protobuf message of this object.
-   *
-   * @return A protobuf message.
-   */
-  @Override
-  public byte[] getProtoBufMessage() {
-    return SendContainerReportProto.newBuilder().build().toByteArray();
-  }
-
-  /**
-   * A Builder class this is the standard pattern we are using for all commands.
-   */
-  public static class Builder {
-    /**
-     * Return a null command.
-     * @return - NullCommand.
-     */
-    public SendContainerCommand build() {
-      return new SendContainerCommand();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/package-info.java
deleted file mode 100644
index 7083c1b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocol.commands;
-/**
- Set of classes that help in protoc conversions.
- **/

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/package-info.java
deleted file mode 100644
index 8bc29aa..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.protocol;
-
-/**
- * This package contains classes for Ozone protocol definitions.
- */


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[14/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java
deleted file mode 100644
index 1fe9a18..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java
+++ /dev/null
@@ -1,1138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.web.localstorage;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.io.LengthInputStream;
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.KeyArgs;
-import org.apache.hadoop.ozone.web.handlers.ListArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-import org.apache.hadoop.ozone.web.response.ListBuckets;
-import org.apache.hadoop.ozone.web.response.ListKeys;
-import org.apache.hadoop.ozone.web.response.ListVolumes;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-import org.apache.hadoop.ozone.web.response.VolumeOwner;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.charset.Charset;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Locale;
-import java.util.TimeZone;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-/**
- * A stand alone Ozone implementation that allows us to run Ozone tests in local
- * mode. This acts as the ozone backend when using MiniDFSCluster for testing.
- */
-public final class OzoneMetadataManager {
-
-  /*
-    OzoneMetadataManager manages volume/bucket/object metadata and
-    data.
-
-    Metadata is maintained in 2 level DB files, UserDB and MetadataDB.
-
-    UserDB contains a Name and a List. For example volumes owned by the user
-    bilbo, would be maintained in UserDB as {bilbo}->{shire, rings}
-
-    This list part of mapping is context sensitive.  That is, if you use {user
-    name} as the key, the list you get is a list of volumes. if you use
-    {user/volume} as the key the list you get is list of buckets. if you use
-    {user/volume/bucket} as key the list you get is the list of objects.
-
-    All keys in the UserDB starts with the UserName.
-
-    We also need to maintain a flat namespace for volumes. This is
-    maintained by the MetadataDB. MetadataDB contains the name of an
-    object(volume, bucket or key) and its associated metadata.
-    The keys in the Metadata DB are {volume}, {volume/bucket} or
-    {volume/bucket/key}. User name is absent, so we have a common root name
-    space for the volume.
-
-    The value of part of metadataDB points to corresponding *Info structures.
-    {volume] -> volumeInfo
-    {volume/bucket} -> bucketInfo
-    {volume/bucket/key} -> keyInfo
-
-
-    Here are various work flows :
-
-    CreateVolume -> Check if Volume exists in metadataDB, if not update UserDB
-    with a list of volumes and update metadataDB with VolumeInfo.
-
-    DeleteVolume -> Check the Volume, and check the VolumeInfo->bucketCount.
-    if bucketCount == 0, delete volume from userDB->{List of volumes} and
-    metadataDB.
-
-    Very similar work flows exist for CreateBucket and DeleteBucket.
-
-      // Please note : These database operations are *not* transactional,
-      // which means that failure can lead to inconsistencies.
-      // Only way to recover is to reset to a clean state, or
-      // use rm -rf /tmp/ozone :)
-
-    We have very simple locking policy. We have a ReaderWriter lock that is
-    taken for each action, this lock is aptly named "lock".
-
-    All actions *must* be performed with a lock held, either a read
-    lock or a write lock. Violation of these locking policies can be harmful.
-
-
-      // // IMPORTANT :
-      // //  This is a simulation layer, this is NOT how the real
-      // //  OZONE functions. This is written to so that we can write
-      // //  stand-alone tests for the protocol and client code.
-
-*/
-  static final Logger LOG = LoggerFactory.getLogger(OzoneMetadataManager.class);
-  private static final String USER_DB = "/user.db";
-  private static final String META_DB = "/metadata.db";
-  private static OzoneMetadataManager bm = null;
-  private MetadataStore userDB;
-  private MetadataStore metadataDB;
-  private ReadWriteLock lock;
-  private Charset encoding = Charset.forName("UTF-8");
-  private String storageRoot;
-  private static final String OBJECT_DIR = "/_objects/";
-
-  // This table keeps a pointer to objects whose operations
-  // are in progress but not yet committed to persistent store
-  private ConcurrentHashMap<OutputStream, String> inProgressObjects;
-
-  /**
-   * Constructs OzoneMetadataManager.
-   */
-  private OzoneMetadataManager(Configuration conf) throws IOException {
-
-    lock = new ReentrantReadWriteLock();
-    storageRoot =
-        conf.getTrimmed(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
-            OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
-
-    File file = new File(storageRoot + OBJECT_DIR);
-
-    if (!file.exists() && !file.mkdirs()) {
-      LOG.error("Creation of Ozone root failed. " + file.toString());
-      throw new IOException("Creation of Ozone root failed.");
-    }
-
-    try {
-      userDB = MetadataStoreBuilder.newBuilder()
-          .setDbFile(new File(storageRoot + USER_DB))
-          .setCreateIfMissing(true)
-          .build();
-      metadataDB = MetadataStoreBuilder.newBuilder()
-          .setDbFile(new File(storageRoot + META_DB))
-          .setCreateIfMissing(true)
-          .build();
-      inProgressObjects = new ConcurrentHashMap<>();
-    } catch (IOException ex) {
-      LOG.error("Cannot open db :" + ex.getMessage());
-      throw ex;
-    }
-  }
-
-  /**
-   * Gets Ozone Manager.
-   *
-   * @return OzoneMetadataManager
-   */
-  public static synchronized OzoneMetadataManager
-      getOzoneMetadataManager(Configuration conf) throws IOException {
-    if (bm == null) {
-      bm = new OzoneMetadataManager(conf);
-    }
-    return bm;
-  }
-
-  /**
-   * Creates a volume.
-   *
-   * @param args - VolumeArgs
-   * @throws OzoneException
-   */
-  public void createVolume(VolumeArgs args) throws OzoneException {
-    lock.writeLock().lock();
-    try {
-      SimpleDateFormat format =
-          new SimpleDateFormat(OzoneConsts.OZONE_DATE_FORMAT, Locale.US);
-      format.setTimeZone(TimeZone.getTimeZone(OzoneConsts.OZONE_TIME_ZONE));
-
-      byte[] volumeName =
-          metadataDB.get(args.getVolumeName().getBytes(encoding));
-
-      if (volumeName != null) {
-        LOG.debug("Volume {} already exists.", volumeName);
-        throw ErrorTable.newError(ErrorTable.VOLUME_ALREADY_EXISTS, args);
-      }
-
-      VolumeInfo newVInfo = new VolumeInfo(args.getVolumeName(), format
-          .format(new Date(System.currentTimeMillis())), args.getAdminName());
-
-      newVInfo.setQuota(args.getQuota());
-      VolumeOwner owner = new VolumeOwner(args.getUserName());
-      newVInfo.setOwner(owner);
-
-      ListVolumes volumeList;
-      byte[] userVolumes = userDB.get(args.getUserName().getBytes(encoding));
-      if (userVolumes == null) {
-        volumeList = new ListVolumes();
-      } else {
-        volumeList = ListVolumes.parse(new String(userVolumes, encoding));
-      }
-
-      volumeList.addVolume(newVInfo);
-      volumeList.sort();
-
-      // Please note : These database operations are *not* transactional,
-      // which means that failure can lead to inconsistencies.
-      // Only way to recover is to reset to a clean state, or
-      // use rm -rf /tmp/ozone :)
-
-
-      userDB.put(args.getUserName().getBytes(encoding),
-          volumeList.toDBString().getBytes(encoding));
-
-      metadataDB.put(args.getVolumeName().getBytes(encoding),
-          newVInfo.toDBString().getBytes(encoding));
-
-    } catch (IOException ex) {
-      throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
-    } finally {
-      lock.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Updates the Volume properties like Owner Name and Quota.
-   *
-   * @param args     - Volume Args
-   * @param property - Flag which tells us what property to upgrade
-   * @throws OzoneException
-   */
-  public void setVolumeProperty(VolumeArgs args, VolumeProperty property)
-      throws OzoneException {
-    lock.writeLock().lock();
-    try {
-      byte[] volumeInfo =
-          metadataDB.get(args.getVolumeName().getBytes(encoding));
-      if (volumeInfo == null) {
-        throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, args);
-      }
-      VolumeInfo info = VolumeInfo.parse(new String(volumeInfo, encoding));
-
-      byte[] userBytes = userDB.get(args.getResourceName().getBytes(encoding));
-      ListVolumes volumeList;
-      if (userBytes == null) {
-        volumeList = new ListVolumes();
-      } else {
-        volumeList = ListVolumes.parse(new String(userBytes, encoding));
-      }
-
-      switch (property) {
-      case OWNER:
-        // needs new owner, we delete the volume object from the
-        // old user's volume list
-        removeOldOwner(info);
-        VolumeOwner owner = new VolumeOwner(args.getUserName());
-        // set the new owner
-        info.setOwner(owner);
-        break;
-      case QUOTA:
-        // if this is quota update we just remove the old object from the
-        // current users list and update the same object later.
-        volumeList.getVolumes().remove(info);
-        info.setQuota(args.getQuota());
-        break;
-      default:
-        OzoneException ozEx =
-            ErrorTable.newError(ErrorTable.BAD_PROPERTY, args);
-        ozEx.setMessage("Volume property is not recognized");
-        throw ozEx;
-      }
-
-      volumeList.addVolume(info);
-
-      metadataDB.put(args.getVolumeName().getBytes(encoding),
-          info.toDBString().getBytes(encoding));
-
-      // if this is an owner change this put will create a new owner or update
-      // the owner's volume list.
-      userDB.put(args.getResourceName().getBytes(encoding),
-          volumeList.toDBString().getBytes(encoding));
-
-    } catch (IOException ex) {
-      throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
-    } finally {
-      lock.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Removes the old owner from the volume.
-   *
-   * @param info - VolumeInfo
-   * @throws IOException
-   */
-  private void removeOldOwner(VolumeInfo info) throws IOException {
-    // We need to look the owner that we know is the current owner
-    byte[] volumeBytes =
-        userDB.get(info.getOwner().getName().getBytes(encoding));
-    ListVolumes volumeList =
-        ListVolumes.parse(new String(volumeBytes, encoding));
-    volumeList.getVolumes().remove(info);
-
-    // Write the new list info to the old user data
-    userDB.put(info.getOwner().getName().getBytes(encoding),
-        volumeList.toDBString().getBytes(encoding));
-  }
-
-  /**
-   * Checks if you are the owner of a specific volume.
-   *
-   * @param volume - Volume Name whose access permissions needs to be checked
-   * @param acl - requested acls which needs to be checked for access
-   * @return - True if you are the owner, false otherwise
-   * @throws OzoneException
-   */
-  public boolean checkVolumeAccess(String volume, OzoneAcl acl)
-      throws OzoneException {
-    lock.readLock().lock();
-    try {
-      byte[] volumeInfo =
-          metadataDB.get(volume.getBytes(encoding));
-      if (volumeInfo == null) {
-        throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, null);
-      }
-
-      VolumeInfo info = VolumeInfo.parse(new String(volumeInfo, encoding));
-      return info.getOwner().getName().equals(acl.getName());
-    } catch (IOException ex) {
-      throw ErrorTable.newError(ErrorTable.SERVER_ERROR, null, ex);
-    } finally {
-      lock.readLock().unlock();
-    }
-  }
-
-  /**
-   * getVolumeInfo returns the Volume Info of a specific volume.
-   *
-   * @param args - Volume args
-   * @return VolumeInfo
-   * @throws OzoneException
-   */
-  public VolumeInfo getVolumeInfo(VolumeArgs args) throws OzoneException {
-    lock.readLock().lock();
-    try {
-      byte[] volumeInfo =
-          metadataDB.get(args.getVolumeName().getBytes(encoding));
-      if (volumeInfo == null) {
-        throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, args);
-      }
-
-      return VolumeInfo.parse(new String(volumeInfo, encoding));
-    } catch (IOException ex) {
-      throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
-    } finally {
-      lock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Returns all the volumes owned by a specific user.
-   *
-   * @param args - User Args
-   * @return - ListVolumes
-   * @throws OzoneException
-   */
-  public ListVolumes listVolumes(ListArgs args) throws OzoneException {
-    lock.readLock().lock();
-    try {
-      if (args.isRootScan()) {
-        return listAllVolumes(args);
-      }
-
-      UserArgs uArgs = (UserArgs) args.getArgs();
-      byte[] volumeList = userDB.get(uArgs.getUserName().getBytes(encoding));
-      if (volumeList == null) {
-        throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, uArgs);
-      }
-
-      String prefix = args.getPrefix();
-      int maxCount = args.getMaxKeys();
-      String prevKey = args.getPrevKey();
-      if (prevKey != null) {
-        // Format is username/volumeName, in local mode we don't use the
-        // user name since we have a userName DB.
-        String[] volName = args.getPrevKey().split("/");
-        if (volName.length < 2) {
-          throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, uArgs);
-        }
-        prevKey = volName[1];
-      }
-      return getFilteredVolumes(volumeList, prefix, prevKey, maxCount);
-    } catch (IOException ex) {
-      throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args.getArgs(), ex);
-    } finally {
-      lock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Returns a List of Volumes that meet the prefix, prevkey and maxCount
-   * constraints.
-   *
-   * @param volumeList - Byte Array of Volume Info.
-   * @param prefix     - prefix string.
-   * @param prevKey    - PrevKey
-   * @param maxCount   - Maximum Count.
-   * @return ListVolumes.
-   * @throws IOException
-   */
-  private ListVolumes getFilteredVolumes(byte[] volumeList, String prefix,
-                                         String prevKey, int maxCount) throws
-      IOException {
-    ListVolumes volumes = ListVolumes.parse(new String(volumeList,
-        encoding));
-    int currentCount = 0;
-    ListIterator<VolumeInfo> iter = volumes.getVolumes().listIterator();
-    ListVolumes filteredVolumes = new ListVolumes();
-    while (currentCount < maxCount && iter.hasNext()) {
-      VolumeInfo vInfo = iter.next();
-      if (isMatchingPrefix(prefix, vInfo) && isAfterKey(prevKey, vInfo)) {
-        filteredVolumes.addVolume(vInfo);
-        currentCount++;
-      }
-    }
-    return filteredVolumes;
-  }
-
-  /**
-   * Returns all volumes in a cluster.
-   *
-   * @param args - ListArgs.
-   * @return ListVolumes.
-   * @throws OzoneException
-   */
-  public ListVolumes listAllVolumes(ListArgs args)
-      throws OzoneException, IOException {
-    String prefix = args.getPrefix();
-    final String prevKey;
-    int maxCount = args.getMaxKeys();
-    String userName = null;
-
-    if (args.getPrevKey() != null) {
-      // Format is username/volumeName
-      String[] volName = args.getPrevKey().split("/");
-      if (volName.length < 2) {
-        throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, args.getArgs());
-      }
-
-      byte[] userNameBytes = userDB.get(volName[0].getBytes(encoding));
-      userName = new String(userNameBytes, encoding);
-      prevKey = volName[1];
-    } else {
-      userName = new String(userDB.peekAround(0, null).getKey(), encoding);
-      prevKey = null;
-    }
-
-    if (userName == null || userName.isEmpty()) {
-      throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, args.getArgs());
-    }
-
-    ListVolumes returnSet = new ListVolumes();
-    // we need to iterate through users until we get maxcount volumes
-    // or no more volumes are left.
-    userDB.iterate(null, (key, value) -> {
-      int currentSize = returnSet.getVolumes().size();
-      if (currentSize < maxCount) {
-        String name = new String(key, encoding);
-        byte[] volumeList = userDB.get(name.getBytes(encoding));
-        if (volumeList == null) {
-          throw new IOException(
-              ErrorTable.newError(ErrorTable.USER_NOT_FOUND, args.getArgs()));
-        }
-        returnSet.getVolumes().addAll(
-            getFilteredVolumes(volumeList, prefix, prevKey,
-                maxCount - currentSize).getVolumes());
-        return true;
-      } else {
-        return false;
-      }
-    });
-
-    return returnSet;
-  }
-
-  /**
-   * Checks if a name starts with a matching prefix.
-   *
-   * @param prefix - prefix string.
-   * @param vInfo  - volume info.
-   * @return true or false.
-   */
-  private boolean isMatchingPrefix(String prefix, VolumeInfo vInfo) {
-    if (prefix == null || prefix.isEmpty()) {
-      return true;
-    }
-    return vInfo.getVolumeName().startsWith(prefix);
-  }
-
-  /**
-   * Checks if the key is after the prevKey.
-   *
-   * @param prevKey - String prevKey.
-   * @param vInfo   - volume Info.
-   * @return - true or false.
-   */
-  private boolean isAfterKey(String prevKey, VolumeInfo vInfo) {
-    if (prevKey == null || prevKey.isEmpty()) {
-      return true;
-    }
-    return prevKey.compareTo(vInfo.getVolumeName()) < 0;
-  }
-
-  /**
-   * Deletes a volume if it exists and is empty.
-   *
-   * @param args - volume args
-   * @throws OzoneException
-   */
-  public void deleteVolume(VolumeArgs args) throws OzoneException {
-    lock.writeLock().lock();
-    try {
-      byte[] volumeName =
-          metadataDB.get(args.getVolumeName().getBytes(encoding));
-      if (volumeName == null) {
-        throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, args);
-      }
-
-      VolumeInfo vInfo = VolumeInfo.parse(new String(volumeName, encoding));
-
-      // Only remove volumes if they are empty.
-      if (vInfo.getBucketCount() > 0) {
-        throw ErrorTable.newError(ErrorTable.VOLUME_NOT_EMPTY, args);
-      }
-
-      ListVolumes volumeList;
-      String user = vInfo.getOwner().getName();
-      byte[] userVolumes = userDB.get(user.getBytes(encoding));
-      if (userVolumes == null) {
-        throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, args);
-      }
-
-      volumeList = ListVolumes.parse(new String(userVolumes, encoding));
-      volumeList.getVolumes().remove(vInfo);
-
-      metadataDB.delete(args.getVolumeName().getBytes(encoding));
-      userDB.put(user.getBytes(encoding),
-          volumeList.toDBString().getBytes(encoding));
-    } catch (IOException ex) {
-      throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
-    } finally {
-      lock.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Create a bucket if it does not exist.
-   *
-   * @param args - BucketArgs
-   * @throws OzoneException
-   */
-  public void createBucket(BucketArgs args) throws OzoneException {
-    lock.writeLock().lock();
-    try {
-      // check if volume exists, buckets cannot be created without volumes
-      byte[] volumeName = metadataDB.get(args.getVolumeName()
-          .getBytes(encoding));
-      if (volumeName == null) {
-        throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, args);
-      }
-
-      // A resource name is volume/bucket -- That is the key in metadata table
-      byte[] bucketName = metadataDB.get(args.getResourceName()
-          .getBytes(encoding));
-      if (bucketName != null) {
-        throw ErrorTable.newError(ErrorTable.BUCKET_ALREADY_EXISTS, args);
-      }
-
-      BucketInfo bucketInfo =
-          new BucketInfo(args.getVolumeName(), args.getBucketName());
-
-      if (args.getRemoveAcls() != null) {
-        OzoneException ex = ErrorTable.newError(ErrorTable.MALFORMED_ACL, args);
-        ex.setMessage("Remove ACLs specified in bucket create. Please remove "
-            + "them and retry.");
-        throw ex;
-      }
-
-      VolumeInfo volInfo = VolumeInfo.parse(new String(volumeName, encoding));
-      volInfo.setBucketCount(volInfo.getBucketCount() + 1);
-
-      bucketInfo.setAcls(args.getAddAcls());
-      bucketInfo.setStorageType(args.getStorageType());
-      bucketInfo.setVersioning(args.getVersioning());
-      ListBuckets bucketList;
-
-      // get bucket list from user/volume -> bucketList
-      byte[] volumeBuckets = userDB.get(args.getParentName()
-          .getBytes(encoding));
-      if (volumeBuckets == null) {
-        bucketList = new ListBuckets();
-      } else {
-        bucketList = ListBuckets.parse(new String(volumeBuckets, encoding));
-      }
-
-      bucketList.addBucket(bucketInfo);
-      bucketList.sort();
-
-      // Update Volume->bucketCount
-      userDB.put(args.getVolumeName().getBytes(encoding),
-          volInfo.toDBString().getBytes(encoding));
-
-      // Now update the userDB with user/volume -> bucketList
-      userDB.put(args.getParentName().getBytes(encoding),
-          bucketList.toDBString().getBytes(encoding));
-
-      // Update userDB with volume/bucket -> empty key list
-      userDB.put(args.getResourceName().getBytes(encoding),
-          new ListKeys().toDBString().getBytes(encoding));
-
-      // and update the metadataDB with volume/bucket->BucketInfo
-      metadataDB.put(args.getResourceName().getBytes(encoding),
-          bucketInfo.toDBString().getBytes(encoding));
-
-    } catch (IOException ex) {
-      throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
-    } finally {
-      lock.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Updates the Bucket properties like ACls and Storagetype.
-   *
-   * @param args     - Bucket Args
-   * @param property - Flag which tells us what property to upgrade
-   * @throws OzoneException
-   */
-  public void setBucketProperty(BucketArgs args, BucketProperty property)
-      throws OzoneException {
-
-    lock.writeLock().lock();
-    try {
-      // volume/bucket-> bucketInfo
-      byte[] bucketInfo = metadataDB.get(args.getResourceName().
-          getBytes(encoding));
-      if (bucketInfo == null) {
-        throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args);
-      }
-
-      BucketInfo info = BucketInfo.parse(new String(bucketInfo, encoding));
-      byte[] volumeBuckets = userDB.get(args.getParentName()
-          .getBytes(encoding));
-      ListBuckets bucketList = ListBuckets.parse(new String(volumeBuckets,
-          encoding));
-      bucketList.getBuckets().remove(info);
-
-      switch (property) {
-      case ACLS:
-        processRemoveAcls(args, info);
-        processAddAcls(args, info);
-        break;
-      case STORAGETYPE:
-        info.setStorageType(args.getStorageType());
-        break;
-      case VERSIONING:
-        info.setVersioning(args.getVersioning());
-        break;
-      default:
-        OzoneException ozEx =
-            ErrorTable.newError(ErrorTable.BAD_PROPERTY, args);
-        ozEx.setMessage("Bucket property is not recognized.");
-        throw ozEx;
-      }
-
-      bucketList.addBucket(info);
-      metadataDB.put(args.getResourceName().getBytes(encoding),
-          info.toDBString().getBytes(encoding));
-
-      userDB.put(args.getParentName().getBytes(encoding),
-          bucketList.toDBString().getBytes(encoding));
-    } catch (IOException ex) {
-      throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
-    } finally {
-      lock.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Process Remove Acls and remove them from the bucket.
-   *
-   * @param args - BucketArgs
-   * @param info - BucketInfo
-   */
-  private void processRemoveAcls(BucketArgs args, BucketInfo info) {
-    List<OzoneAcl> removeAcls = args.getRemoveAcls();
-    if ((removeAcls == null) || (info.getAcls() == null)) {
-      return;
-    }
-    for (OzoneAcl racl : args.getRemoveAcls()) {
-      ListIterator<OzoneAcl> aclIter = info.getAcls().listIterator();
-      while (aclIter.hasNext()) {
-        if (racl.equals(aclIter.next())) {
-          aclIter.remove();
-          break;
-        }
-      }
-    }
-  }
-
-  /**
-   * Process Add Acls and Add them to the bucket.
-   *
-   * @param args - BucketArgs
-   * @param info - BucketInfo
-   */
-  private void processAddAcls(BucketArgs args, BucketInfo info) {
-    List<OzoneAcl> addAcls = args.getAddAcls();
-    if ((addAcls == null)) {
-      return;
-    }
-
-    if (info.getAcls() == null) {
-      info.setAcls(addAcls);
-      return;
-    }
-
-    for (OzoneAcl newacl : addAcls) {
-      ListIterator<OzoneAcl> aclIter = info.getAcls().listIterator();
-      while (aclIter.hasNext()) {
-        if (newacl.equals(aclIter.next())) {
-          continue;
-        }
-      }
-      info.getAcls().add(newacl);
-    }
-  }
-
-  /**
-   * Deletes a given bucket.
-   *
-   * @param args - BucketArgs
-   * @throws OzoneException
-   */
-  public void deleteBucket(BucketArgs args) throws OzoneException {
-    lock.writeLock().lock();
-    try {
-      byte[] bucketInfo = metadataDB.get(args.getResourceName()
-          .getBytes(encoding));
-      if (bucketInfo == null) {
-        throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args);
-      }
-
-      BucketInfo bInfo = BucketInfo.parse(new String(bucketInfo, encoding));
-
-      // Only remove buckets if they are empty.
-      if (bInfo.getKeyCount() > 0) {
-        throw ErrorTable.newError(ErrorTable.BUCKET_NOT_EMPTY, args);
-      }
-
-      byte[] bucketBytes = userDB.get(args.getParentName().getBytes(encoding));
-      if (bucketBytes == null) {
-        throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args);
-      }
-
-      ListBuckets bucketList =
-          ListBuckets.parse(new String(bucketBytes, encoding));
-      bucketList.getBuckets().remove(bInfo);
-
-      metadataDB.delete(args.getResourceName().getBytes(encoding));
-      userDB.put(args.getParentName().getBytes(encoding),
-          bucketList.toDBString().getBytes(encoding));
-    } catch (IOException ex) {
-      throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
-    } finally {
-      lock.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Returns the Bucket info for a given bucket.
-   *
-   * @param args - Bucket Args
-   * @return BucketInfo   -  Bucket Information
-   * @throws OzoneException
-   */
-  public BucketInfo getBucketInfo(BucketArgs args) throws OzoneException {
-    lock.readLock().lock();
-    try {
-      byte[] bucketBytes = metadataDB.get(args.getResourceName()
-          .getBytes(encoding));
-      if (bucketBytes == null) {
-        throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args);
-      }
-
-      return BucketInfo.parse(new String(bucketBytes, encoding));
-    } catch (IOException ex) {
-      throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
-    } finally {
-      lock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Returns a list of buckets for a given volume.
-   *
-   * @param args - volume args
-   * @return List of buckets
-   * @throws OzoneException
-   */
-  public ListBuckets listBuckets(ListArgs args) throws OzoneException {
-    lock.readLock().lock();
-    try {
-      Preconditions.checkState(args.getArgs() instanceof VolumeArgs);
-      VolumeArgs vArgs = (VolumeArgs) args.getArgs();
-      String userVolKey = vArgs.getUserName() + "/" + vArgs.getVolumeName();
-
-      // TODO : Query using Prefix and PrevKey
-      byte[] bucketBytes = userDB.get(userVolKey.getBytes(encoding));
-      if (bucketBytes == null) {
-        throw ErrorTable.newError(ErrorTable.INVALID_VOLUME_NAME,
-            args.getArgs());
-      }
-      return ListBuckets.parse(new String(bucketBytes, encoding));
-    } catch (IOException ex) {
-      throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args.getArgs(), ex);
-    } finally {
-      lock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Creates a key and returns a stream to which this key can be written to.
-   *
-   * @param args KeyArgs
-   * @return - A stream into which key can be written to.
-   * @throws OzoneException
-   */
-  public OutputStream createKey(KeyArgs args) throws OzoneException {
-    lock.writeLock().lock();
-    try {
-      String fileNameHash = DigestUtils.sha256Hex(args.getResourceName());
-
-      // Please don't try trillion objects unless the physical file system
-      // is capable of doing that in a single directory.
-
-      String fullPath = storageRoot + OBJECT_DIR + fileNameHash;
-      File f = new File(fullPath);
-
-      // In real ozone it would not be this way, a file will be overwritten
-      // only if the upload is successful.
-      if (f.exists()) {
-        LOG.debug("we are overwriting a file. This is by design.");
-        if (!f.delete()) {
-          LOG.error("Unable to delete the file: {}", fullPath);
-          throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args);
-        }
-      }
-
-      // f.createNewFile();
-      FileOutputStream fsStream = new FileOutputStream(f);
-      inProgressObjects.put(fsStream, fullPath);
-
-      return fsStream;
-    } catch (IOException e) {
-      throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, e);
-    } finally {
-      lock.writeLock().unlock();
-    }
-  }
-
-  /**
-   * commit keys moves an In progress object into the metadata store so that key
-   * is visible in the metadata operations from that point onwards.
-   *
-   * @param args Object args
-   * @throws OzoneException
-   */
-  public void commitKey(KeyArgs args, OutputStream stream)
-      throws OzoneException {
-    SimpleDateFormat format =
-        new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
-    lock.writeLock().lock();
-
-    try {
-      byte[] bucketInfo = metadataDB.get(args.getParentName()
-          .getBytes(encoding));
-      if (bucketInfo == null) {
-        throw ErrorTable.newError(ErrorTable.INVALID_RESOURCE_NAME, args);
-      }
-      BucketInfo bInfo = BucketInfo.parse(new String(bucketInfo, encoding));
-      bInfo.setKeyCount(bInfo.getKeyCount() + 1);
-
-      String fileNameHash = inProgressObjects.get(stream);
-      inProgressObjects.remove(stream);
-      if (fileNameHash == null) {
-        throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args);
-      }
-
-      ListKeys keyList;
-      byte[] bucketListBytes = userDB.get(args.getParentName()
-          .getBytes(encoding));
-      keyList = ListKeys.parse(new String(bucketListBytes, encoding));
-      KeyInfo keyInfo;
-
-      byte[] objectBytes = metadataDB.get(args.getResourceName()
-          .getBytes(encoding));
-
-      if (objectBytes != null) {
-        // we are overwriting an existing object.
-        // TODO : Emit info for Accounting
-        keyInfo = KeyInfo.parse(new String(objectBytes, encoding));
-        keyList.getKeyList().remove(keyInfo);
-      } else {
-        keyInfo = new KeyInfo();
-      }
-
-      keyInfo.setCreatedOn(format.format(new Date(System.currentTimeMillis())));
-
-      // TODO : support version, we need to check if versioning
-      // is switched on the bucket and make appropriate calls.
-      keyInfo.setVersion(0);
-
-      keyInfo.setDataFileName(fileNameHash);
-      keyInfo.setKeyName(args.getKeyName());
-      keyInfo.setMd5hash(args.getHash());
-      keyInfo.setSize(args.getSize());
-
-      keyList.getKeyList().add(keyInfo);
-
-      // if the key exists, we overwrite happily :). since the
-      // earlier call - createObject -  has overwritten the data.
-
-      metadataDB.put(args.getResourceName().getBytes(encoding),
-          keyInfo.toDBString().getBytes(encoding));
-
-      metadataDB.put(args.getParentName().getBytes(encoding),
-          bInfo.toDBString().getBytes(encoding));
-
-      userDB.put(args.getParentName().getBytes(encoding),
-          keyList.toDBString().getBytes(encoding));
-
-    } catch (IOException e) {
-      throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, e);
-    } finally {
-      lock.writeLock().unlock();
-    }
-  }
-
-  /**
-   * deletes an key from a given bucket.
-   *
-   * @param args - ObjectArgs
-   * @throws OzoneException
-   */
-  public void deleteKey(KeyArgs args) throws OzoneException {
-    lock.writeLock().lock();
-    try {
-      byte[] bucketInfo = metadataDB.get(args.getParentName()
-          .getBytes(encoding));
-      if (bucketInfo == null) {
-        throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args);
-      }
-      BucketInfo bInfo = BucketInfo.parse(new String(bucketInfo, encoding));
-      bInfo.setKeyCount(bInfo.getKeyCount() - 1);
-
-
-      byte[] bucketListBytes = userDB.get(args.getParentName()
-          .getBytes(encoding));
-      if (bucketListBytes == null) {
-        throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args);
-      }
-      ListKeys keyList = ListKeys.parse(new String(bucketListBytes, encoding));
-
-
-      byte[] objectBytes = metadataDB.get(args.getResourceName()
-          .getBytes(encoding));
-      if (objectBytes == null) {
-        throw ErrorTable.newError(ErrorTable.INVALID_KEY, args);
-      }
-
-      KeyInfo oInfo = KeyInfo.parse(new String(objectBytes, encoding));
-      keyList.getKeyList().remove(oInfo);
-
-      String fileNameHash = DigestUtils.sha256Hex(args.getResourceName());
-
-      String fullPath = storageRoot + OBJECT_DIR + fileNameHash;
-      File f = new File(fullPath);
-
-      if (f.exists()) {
-        if (!f.delete()) {
-          throw ErrorTable.newError(ErrorTable.KEY_OPERATION_CONFLICT, args);
-        }
-      } else {
-        throw ErrorTable.newError(ErrorTable.INVALID_KEY, args);
-      }
-
-
-      metadataDB.delete(args.getResourceName().getBytes(encoding));
-      metadataDB.put(args.getParentName().getBytes(encoding),
-          bInfo.toDBString().getBytes(encoding));
-      userDB.put(args.getParentName().getBytes(encoding),
-          keyList.toDBString().getBytes(encoding));
-    } catch (IOException e) {
-      throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, e);
-    } finally {
-      lock.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Returns a Stream for the file.
-   *
-   * @param args - Object args
-   * @return Stream
-   * @throws IOException
-   * @throws OzoneException
-   */
-  public LengthInputStream newKeyReader(KeyArgs args)
-      throws IOException, OzoneException {
-    lock.readLock().lock();
-    try {
-      String fileNameHash = DigestUtils.sha256Hex(args.getResourceName());
-      String fullPath = storageRoot + OBJECT_DIR + fileNameHash;
-      File f = new File(fullPath);
-      if (!f.exists()) {
-        throw ErrorTable.newError(ErrorTable.INVALID_RESOURCE_NAME, args);
-      }
-      long size = f.length();
-
-      FileInputStream fileStream = new FileInputStream(f);
-      return new LengthInputStream(fileStream, size);
-    } finally {
-      lock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Returns keys in a bucket.
-   *
-   * @param args
-   * @return List of keys.
-   * @throws IOException
-   * @throws OzoneException
-   */
-  public ListKeys listKeys(ListArgs args) throws IOException, OzoneException {
-    lock.readLock().lock();
-    // TODO : Support Prefix and PrevKey lookup.
-    try {
-      Preconditions.checkState(args.getArgs() instanceof BucketArgs);
-      BucketArgs bArgs = (BucketArgs) args.getArgs();
-      byte[] bucketInfo = metadataDB.get(bArgs.getResourceName()
-          .getBytes(encoding));
-      if (bucketInfo == null) {
-        throw ErrorTable.newError(ErrorTable.INVALID_RESOURCE_NAME, bArgs);
-      }
-
-      byte[] bucketListBytes = userDB.get(bArgs.getResourceName()
-          .getBytes(encoding));
-      if (bucketListBytes == null) {
-        throw ErrorTable.newError(ErrorTable.INVALID_RESOURCE_NAME, bArgs);
-      }
-      return ListKeys.parse(new String(bucketListBytes, encoding));
-    } finally {
-      lock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Get the Key information for a given key.
-   *
-   * @param args - Key Args
-   * @return KeyInfo - Key Information
-   * @throws OzoneException
-   */
-  public KeyInfo getKeyInfo(KeyArgs args) throws IOException, OzoneException {
-    lock.readLock().lock();
-    try {
-      byte[] bucketInfo = metadataDB
-          .get(args.getParentName().getBytes(encoding));
-      if (bucketInfo == null) {
-        throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args);
-      }
-
-      byte[] bucketListBytes = userDB
-          .get(args.getParentName().getBytes(encoding));
-      if (bucketListBytes == null) {
-        throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args);
-      }
-
-      byte[] objectBytes = metadataDB
-          .get(args.getResourceName().getBytes(encoding));
-      if (objectBytes == null) {
-        throw ErrorTable.newError(ErrorTable.INVALID_KEY, args);
-      }
-
-      return KeyInfo.parse(new String(objectBytes, encoding));
-    } catch (IOException e) {
-      throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, e);
-    } finally {
-      lock.readLock().unlock();
-    }
-  }
-
-  /**
-   * This is used in updates to volume metadata.
-   */
-  public enum VolumeProperty {
-    OWNER, QUOTA
-  }
-
-  /**
-   * Bucket Properties.
-   */
-  public enum BucketProperty {
-    ACLS, STORAGETYPE, VERSIONING
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/package-info.java
deleted file mode 100644
index 6bf6643..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.localstorage;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/LengthInputStreamMessageBodyWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/LengthInputStreamMessageBodyWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/LengthInputStreamMessageBodyWriter.java
deleted file mode 100644
index 6db49d5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/LengthInputStreamMessageBodyWriter.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.messages;
-
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.client.io.LengthInputStream;
-
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.MultivaluedMap;
-import javax.ws.rs.ext.MessageBodyWriter;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.lang.annotation.Annotation;
-import java.lang.reflect.Type;
-
-/**
- * Writes outbound HTTP response object bytes.  The content length is determined
- * from the {@link LengthInputStream}.
- */
-public final class LengthInputStreamMessageBodyWriter
-    implements MessageBodyWriter<LengthInputStream> {
-  private static final int CHUNK_SIZE = 8192;
-
-  @Override
-  public long getSize(LengthInputStream lis, Class<?> type, Type genericType,
-                      Annotation[] annotations, MediaType mediaType) {
-    return lis.getLength();
-  }
-
-  @Override
-  public boolean isWriteable(Class<?> type, Type genericType,
-                             Annotation[] annotations, MediaType mediaType) {
-    return LengthInputStream.class.isAssignableFrom(type);
-  }
-
-  @Override
-  public void writeTo(LengthInputStream lis, Class<?> type, Type genericType,
-                      Annotation[] annotations, MediaType mediaType,
-                      MultivaluedMap<String, Object> httpHeaders,
-                      OutputStream out) throws IOException {
-    IOUtils.copyBytes(lis, out, CHUNK_SIZE);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/StringMessageBodyWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/StringMessageBodyWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/StringMessageBodyWriter.java
deleted file mode 100644
index ad637af..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/StringMessageBodyWriter.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.messages;
-
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.MultivaluedMap;
-import javax.ws.rs.ext.MessageBodyWriter;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.lang.annotation.Annotation;
-import java.lang.reflect.Type;
-
-/**
- * Writes outbound HTTP response strings.  We use this rather than the built-in
- * writer so that we can determine content length from the string length instead
- * of possibly falling back to a chunked response.
- */
-public final class StringMessageBodyWriter implements
-    MessageBodyWriter<String> {
-  private static final int CHUNK_SIZE = 8192;
-
-  @Override
-  public long getSize(String str, Class<?> type, Type genericType,
-                      Annotation[] annotations, MediaType mediaType) {
-    return str.length();
-  }
-
-  @Override
-  public boolean isWriteable(Class<?> type, Type genericType,
-                             Annotation[] annotations, MediaType mediaType) {
-    return String.class.isAssignableFrom(type);
-  }
-
-  @Override
-  public void writeTo(String str, Class<?> type, Type genericType,
-                      Annotation[] annotations, MediaType mediaType,
-                      MultivaluedMap<String, Object> httpHeaders,
-                      OutputStream out) throws IOException {
-    IOUtils.copyBytes(new ByteArrayInputStream(
-        str.getBytes(OzoneUtils.ENCODING)), out, CHUNK_SIZE);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/package-info.java
deleted file mode 100644
index 273b3f5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.messages;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/CloseableCleanupListener.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/CloseableCleanupListener.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/CloseableCleanupListener.java
deleted file mode 100644
index 3d9db20..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/CloseableCleanupListener.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.netty;
-
-import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelFutureListener;
-import org.apache.hadoop.io.IOUtils;
-
-import java.io.Closeable;
-
-/**
- * A {@link ChannelFutureListener} that closes {@link Closeable} resources.
- */
-final class CloseableCleanupListener implements ChannelFutureListener {
-
-  private final Closeable[] closeables;
-
-  /**
-   * Creates a new CloseableCleanupListener.
-   *
-   * @param closeables any number of closeable resources
-   */
-  CloseableCleanupListener(Closeable... closeables) {
-    this.closeables = closeables;
-  }
-
-  @Override
-  public void operationComplete(ChannelFuture future) {
-    IOUtils.cleanupWithLogger(null, closeables);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreChannelHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreChannelHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreChannelHandler.java
deleted file mode 100644
index 89c196c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreChannelHandler.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.netty;
-
-import io.netty.channel.ChannelFutureListener;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.SimpleChannelInboundHandler;
-import io.netty.handler.codec.http.DefaultFullHttpResponse;
-import io.netty.handler.codec.http.HttpResponse;
-import io.netty.handler.codec.http.HttpResponseStatus;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH;
-import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE;
-import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
-import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
-
-/**
- * Abstract base class for the multiple Netty channel handlers used in the
- * Object Store Netty channel pipeline.
- */
-abstract class ObjectStoreChannelHandler<T>
-    extends SimpleChannelInboundHandler<T> {
-
-  /** Log usable in all subclasses. */
-  protected static final Logger LOG =
-      LoggerFactory.getLogger(ObjectStoreChannelHandler.class);
-
-  /**
-   * Handles uncaught exceptions in the channel pipeline by sending an internal
-   * server error response if the channel is still active.
-   *
-   * @param ctx ChannelHandlerContext to receive response
-   * @param cause Throwable that was unhandled in the channel pipeline
-   */
-  @Override
-  public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
-    LOG.error("Unexpected exception in Netty pipeline.", cause);
-    if (ctx.channel().isActive()) {
-      sendErrorResponse(ctx, INTERNAL_SERVER_ERROR);
-    }
-  }
-
-  /**
-   * Sends an error response.  This method is used when an unexpected error is
-   * encountered within the channel pipeline, outside of the actual Object Store
-   * application.  It always closes the connection, because we can't in general
-   * know the state of the connection when these errors occur, so attempting to
-   * keep the connection alive could be unpredictable.
-   *
-   * @param ctx ChannelHandlerContext to receive response
-   * @param status HTTP response status
-   */
-  protected static void sendErrorResponse(ChannelHandlerContext ctx,
-      HttpResponseStatus status) {
-    HttpResponse nettyResp = new DefaultFullHttpResponse(HTTP_1_1, status);
-    nettyResp.headers().set(CONTENT_LENGTH, 0);
-    nettyResp.headers().set(CONNECTION, CLOSE);
-    ctx.writeAndFlush(nettyResp).addListener(ChannelFutureListener.CLOSE);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainer.java
deleted file mode 100644
index c7b516f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainer.java
+++ /dev/null
@@ -1,348 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.netty;
-
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH;
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
-import static io.netty.handler.codec.http.HttpHeaders.Names.TRANSFER_ENCODING;
-import static io.netty.handler.codec.http.HttpHeaders.Names.HOST;
-import static io.netty.handler.codec.http.HttpHeaders.Values.KEEP_ALIVE;
-import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE;
-import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CancellationException;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.TimeUnit;
-
-import com.sun.jersey.core.header.InBoundHeaders;
-import com.sun.jersey.spi.container.ContainerRequest;
-import com.sun.jersey.spi.container.ContainerResponse;
-import com.sun.jersey.spi.container.ContainerResponseWriter;
-import com.sun.jersey.spi.container.WebApplication;
-
-import io.netty.handler.codec.http.DefaultHttpResponse;
-//import io.netty.handler.codec.http.HttpUtil;
-import io.netty.handler.codec.http.HttpHeaders;
-import io.netty.handler.codec.http.HttpRequest;
-import io.netty.handler.codec.http.HttpResponse;
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.handlers.StorageHandlerBuilder;
-
-/**
- * This is a custom Jersey container that hosts the Object Store web
- * application. It supports dispatching an inbound Netty {@link HttpRequest}
- * to the Object Store Jersey application.  Request dispatching must run
- * asynchronously, because the Jersey application must consume the inbound
- * HTTP request from a  piped stream and produce the outbound HTTP response
- * for another piped stream.The Netty channel handlers consume the connected
- * ends of these piped streams. Request dispatching cannot run directly on
- * the Netty threads, or there would be a risk of deadlock (one thread
- * producing/consuming its end of the pipe  while no other thread is
- * producing/consuming the opposite end).
- */
-public final class ObjectStoreJerseyContainer {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ObjectStoreJerseyContainer.class);
-
-  private final WebApplication webapp;
-
-  private StorageHandler storageHandler;
-
-  /**
-   * Creates a new ObjectStoreJerseyContainer.
-   *
-   * @param webapp web application
-   */
-  public ObjectStoreJerseyContainer(WebApplication webapp) {
-    this.webapp = webapp;
-  }
-
-  /**
-   * Sets the {@link StorageHandler}. This must be called before dispatching any
-   * requests.
-   *
-   * @param newStorageHandler {@link StorageHandler} implementation
-   */
-  public void setStorageHandler(StorageHandler newStorageHandler) {
-    this.storageHandler = newStorageHandler;
-  }
-
-  /**
-   * Asynchronously executes an HTTP request.
-   *
-   * @param nettyReq HTTP request
-   * @param reqIn input stream for reading request body
-   * @param respOut output stream for writing response body
-   */
-  public Future<HttpResponse> dispatch(HttpRequest nettyReq, InputStream reqIn,
-                                       OutputStream respOut) {
-    // The request executes on a separate background thread.  As soon as enough
-    // processing has completed to bootstrap the outbound response, the thread
-    // counts down on a latch.  This latch also unblocks callers trying to get
-    // the asynchronous response out of the returned future.
-    final CountDownLatch latch = new CountDownLatch(1);
-    final RequestRunner runner = new RequestRunner(nettyReq, reqIn, respOut,
-        latch);
-    final Thread thread = new Thread(runner);
-    thread.setDaemon(true);
-    thread.start();
-    return new Future<HttpResponse>() {
-
-      private volatile boolean isCancelled = false;
-
-      @Override
-      public boolean cancel(boolean mayInterruptIfRunning) {
-        if (latch.getCount() == 0) {
-          return false;
-        }
-        if (!mayInterruptIfRunning) {
-          return false;
-        }
-        if (!thread.isAlive()) {
-          return false;
-        }
-        thread.interrupt();
-        try {
-          thread.join();
-        } catch (InterruptedException e) {
-          LOG.info("Interrupted while attempting to cancel dispatch thread.");
-          Thread.currentThread().interrupt();
-          return false;
-        }
-        isCancelled = true;
-        return true;
-      }
-
-      @Override
-      public HttpResponse get()
-          throws InterruptedException, ExecutionException {
-        checkCancelled();
-        latch.await();
-        return this.getOrThrow();
-      }
-
-      @Override
-      public HttpResponse get(long timeout, TimeUnit unit)
-          throws InterruptedException, ExecutionException, TimeoutException {
-        checkCancelled();
-        if (!latch.await(timeout, unit)) {
-          throw new TimeoutException(String.format(
-              "Timed out waiting for HttpResponse after %d %s.",
-              timeout, unit.toString().toLowerCase()));
-        }
-        return this.getOrThrow();
-      }
-
-      @Override
-      public boolean isCancelled() {
-        return isCancelled;
-      }
-
-      @Override
-      public boolean isDone() {
-        return !isCancelled && latch.getCount() == 0;
-      }
-
-      private void checkCancelled() {
-        if (isCancelled()) {
-          throw new CancellationException();
-        }
-      }
-
-      private HttpResponse getOrThrow() throws ExecutionException {
-        try {
-          return runner.getResponse();
-        } catch (Exception e) {
-          throw new ExecutionException(e);
-        }
-      }
-    };
-  }
-
-  /**
-   * Runs the actual handling of the HTTP request.
-   */
-  private final class RequestRunner implements Runnable,
-      ContainerResponseWriter {
-
-    private final CountDownLatch latch;
-    private final HttpRequest nettyReq;
-    private final InputStream reqIn;
-    private final OutputStream respOut;
-
-    private Exception exception;
-    private HttpResponse nettyResp;
-
-    /**
-     * Creates a new RequestRunner.
-     *
-     * @param nettyReq HTTP request
-     * @param reqIn input stream for reading request body
-     * @param respOut output stream for writing response body
-     * @param latch for coordinating asynchronous return of HTTP response
-     */
-    RequestRunner(HttpRequest nettyReq, InputStream reqIn,
-                         OutputStream respOut, CountDownLatch latch) {
-      this.latch = latch;
-      this.nettyReq = nettyReq;
-      this.reqIn = reqIn;
-      this.respOut = respOut;
-    }
-
-    @Override
-    public void run() {
-      LOG.trace("begin RequestRunner, nettyReq = {}", this.nettyReq);
-      StorageHandlerBuilder.setStorageHandler(
-          ObjectStoreJerseyContainer.this.storageHandler);
-      try {
-        ContainerRequest jerseyReq = nettyRequestToJerseyRequest(
-            ObjectStoreJerseyContainer.this.webapp, this.nettyReq, this.reqIn);
-        ObjectStoreJerseyContainer.this.webapp.handleRequest(jerseyReq, this);
-      } catch (Exception e) {
-        LOG.error("Error running Jersey Request Runner", e);
-        this.exception = e;
-        this.latch.countDown();
-      } finally {
-        IOUtils.cleanupWithLogger(null, this.reqIn, this.respOut);
-        StorageHandlerBuilder.removeStorageHandler();
-      }
-      LOG.trace("end RequestRunner, nettyReq = {}", this.nettyReq);
-    }
-
-    /**
-     * This is a callback triggered by Jersey as soon as dispatch has completed
-     * to the point of knowing what kind of response to return.  We save the
-     * response and trigger the latch to unblock callers waiting on the
-     * asynchronous return of the response.  Our response always sets a
-     * Content-Length header.  (We do not support Transfer-Encoding: chunked.)
-     * We also return the output stream for Jersey to use for writing the
-     * response body.
-     *
-     * @param contentLength length of response
-     * @param jerseyResp HTTP response returned by Jersey
-     * @return OutputStream for Jersey to use for writing the response body
-     */
-    @Override
-    public OutputStream writeStatusAndHeaders(long contentLength,
-                                              ContainerResponse jerseyResp) {
-      LOG.trace(
-          "begin writeStatusAndHeaders, contentLength = {}, jerseyResp = {}.",
-          contentLength, jerseyResp);
-      this.nettyResp = jerseyResponseToNettyResponse(jerseyResp);
-      this.nettyResp.headers().set(CONTENT_LENGTH, Math.max(0, contentLength));
-      this.nettyResp.headers().set(CONNECTION,
-          HttpHeaders.isKeepAlive(this.nettyReq) ? KEEP_ALIVE : CLOSE);
-      this.latch.countDown();
-      LOG.trace(
-          "end writeStatusAndHeaders, contentLength = {}, jerseyResp = {}.",
-          contentLength, jerseyResp);
-      return this.respOut;
-    }
-
-    /**
-     * This is a callback triggered by Jersey after it has completed writing the
-     * response body to the stream.  We must close the stream here to unblock
-     * the Netty thread consuming the last chunk of the response from the input
-     * end of the piped stream.
-     *
-     * @throws IOException if there is an I/O error
-     */
-    @Override
-    public void finish() throws IOException {
-      IOUtils.cleanupWithLogger(null, this.respOut);
-    }
-
-    /**
-     * Gets the HTTP response calculated by the Jersey application, or throws an
-     * exception if an error occurred during processing.  It only makes sense to
-     * call this method after waiting on the latch to trigger.
-     *
-     * @return HTTP response
-     * @throws Exception if there was an error executing the request
-     */
-    public HttpResponse getResponse() throws Exception {
-      if (this.exception != null) {
-        throw this.exception;
-      }
-      return this.nettyResp;
-    }
-  }
-
-  /**
-   * Converts a Jersey HTTP response object to a Netty HTTP response object.
-   *
-   * @param jerseyResp Jersey HTTP response
-   * @return Netty HTTP response
-   */
-  private static HttpResponse jerseyResponseToNettyResponse(
-      ContainerResponse jerseyResp) {
-    HttpResponse nettyResp = new DefaultHttpResponse(HTTP_1_1,
-        HttpResponseStatus.valueOf(jerseyResp.getStatus()));
-    for (Map.Entry<String, List<Object>> header :
-        jerseyResp.getHttpHeaders().entrySet()) {
-      if (!header.getKey().equalsIgnoreCase(CONTENT_LENGTH.toString()) &&
-          !header.getKey().equalsIgnoreCase(TRANSFER_ENCODING.toString())) {
-        nettyResp.headers().set(header.getKey(), header.getValue());
-      }
-    }
-    return nettyResp;
-  }
-
-  /**
-   * Converts a Netty HTTP request object to a Jersey HTTP request object.
-   *
-   * @param webapp web application
-   * @param nettyReq Netty HTTP request
-   * @param reqIn input stream for reading request body
-   * @return Jersey HTTP request
-   * @throws URISyntaxException if there is an error handling the request URI
-   */
-  private static ContainerRequest nettyRequestToJerseyRequest(
-      WebApplication webapp, HttpRequest nettyReq, InputStream reqIn)
-      throws URISyntaxException {
-    HttpHeaders nettyHeaders = nettyReq.headers();
-    InBoundHeaders jerseyHeaders = new InBoundHeaders();
-    for (String name : nettyHeaders.names()) {
-      jerseyHeaders.put(name, nettyHeaders.getAll(name));
-    }
-    String host = nettyHeaders.get(HOST);
-    String scheme = host.startsWith("https") ? "https://" : "http://";
-    String baseUri = scheme + host + "/";
-    String reqUri = scheme + host + nettyReq.getUri();
-    LOG.trace("baseUri = {}, reqUri = {}", baseUri, reqUri);
-    return new ContainerRequest(webapp, nettyReq.getMethod().name(),
-        new URI(baseUri), new URI(reqUri), jerseyHeaders, reqIn);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainerProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainerProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainerProvider.java
deleted file mode 100644
index e943969..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainerProvider.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.netty;
-
-import com.sun.jersey.api.container.ContainerException;
-import com.sun.jersey.api.core.ResourceConfig;
-import com.sun.jersey.spi.container.ContainerProvider;
-import com.sun.jersey.spi.container.WebApplication;
-
-/**
- * This is a Jersey {@link ContainerProvider} capable of boostrapping the
- * Object Store web application into a custom container.  It must be registered
- * using the Java service loader mechanism by listing it in
- * META-INF/services/com.sun.jersey.spi.container.ContainerProvider .
- */
-public final class ObjectStoreJerseyContainerProvider
-    implements ContainerProvider<ObjectStoreJerseyContainer> {
-
-  @Override
-  public ObjectStoreJerseyContainer createContainer(
-      Class<ObjectStoreJerseyContainer> type, ResourceConfig conf,
-      WebApplication webapp) throws ContainerException {
-    return new ObjectStoreJerseyContainer(webapp);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/RequestContentObjectStoreChannelHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/RequestContentObjectStoreChannelHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/RequestContentObjectStoreChannelHandler.java
deleted file mode 100644
index 0a2f22d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/RequestContentObjectStoreChannelHandler.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.netty;
-
-import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelFutureListener;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.http.HttpContent;
-import io.netty.handler.codec.http.HttpHeaders;
-import io.netty.handler.codec.http.HttpRequest;
-import io.netty.handler.codec.http.HttpResponse;
-import io.netty.handler.codec.http.LastHttpContent;
-import io.netty.handler.stream.ChunkedStream;
-import org.apache.hadoop.io.IOUtils;
-
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.concurrent.Future;
-
-/**
- * Object Store Netty channel pipeline handler that handles inbound
- * {@link HttpContent} fragments for the request body by sending the bytes into
- * the pipe so that the application dispatch thread can read it.
- * After receiving the {@link LastHttpContent}, this handler also flushes the
- * response.
- */
-public final class RequestContentObjectStoreChannelHandler
-    extends ObjectStoreChannelHandler<HttpContent> {
-
-  private final HttpRequest nettyReq;
-  private final Future<HttpResponse> nettyResp;
-  private final OutputStream reqOut;
-  private final InputStream respIn;
-  private ObjectStoreJerseyContainer jerseyContainer;
-
-  /**
-   * Creates a new RequestContentObjectStoreChannelHandler.
-   *
-   * @param nettyReq HTTP request
-   * @param nettyResp asynchronous HTTP response
-   * @param reqOut output stream for writing request body
-   * @param respIn input stream for reading response body
-   * @param jerseyContainer jerseyContainer to handle the request
-   */
-  public RequestContentObjectStoreChannelHandler(HttpRequest nettyReq,
-      Future<HttpResponse> nettyResp, OutputStream reqOut, InputStream respIn,
-      ObjectStoreJerseyContainer jerseyContainer) {
-    this.nettyReq = nettyReq;
-    this.nettyResp = nettyResp;
-    this.reqOut = reqOut;
-    this.respIn = respIn;
-    this.jerseyContainer = jerseyContainer;
-  }
-
-  @Override
-  public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
-    ctx.flush();
-  }
-
-  @Override
-  public void channelRead0(ChannelHandlerContext ctx, HttpContent content)
-      throws Exception {
-    LOG.trace(
-        "begin RequestContentObjectStoreChannelHandler channelRead0, " +
-        "ctx = {}, content = {}", ctx, content);
-    content.content().readBytes(this.reqOut, content.content().readableBytes());
-    if (content instanceof LastHttpContent) {
-      IOUtils.cleanupWithLogger(null, this.reqOut);
-      ctx.write(this.nettyResp.get());
-      ChannelFuture respFuture = ctx.writeAndFlush(new ChunkedStream(
-          this.respIn));
-      respFuture.addListener(new CloseableCleanupListener(this.respIn));
-      if (!HttpHeaders.isKeepAlive(this.nettyReq)) {
-        respFuture.addListener(ChannelFutureListener.CLOSE);
-      } else {
-        respFuture.addListener(new ChannelFutureListener() {
-          @Override
-          public void operationComplete(ChannelFuture future) throws Exception {
-            // Notify client this is the last content for current request.
-            ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
-            // Reset the pipeline handler for next request to reuses the
-            // same connection.
-            RequestDispatchObjectStoreChannelHandler h =
-                new RequestDispatchObjectStoreChannelHandler(jerseyContainer);
-            ctx.pipeline().replace(ctx.pipeline().last(),
-                RequestDispatchObjectStoreChannelHandler.class.getSimpleName(),
-                h);
-          }
-        });
-      }
-    }
-    LOG.trace(
-        "end RequestContentObjectStoreChannelHandler channelRead0, " +
-        "ctx = {}, content = {}", ctx, content);
-  }
-
-  @Override
-  public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
-    super.exceptionCaught(ctx, cause);
-    IOUtils.cleanupWithLogger(null, this.reqOut, this.respIn);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/RequestDispatchObjectStoreChannelHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/RequestDispatchObjectStoreChannelHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/RequestDispatchObjectStoreChannelHandler.java
deleted file mode 100644
index add827a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/RequestDispatchObjectStoreChannelHandler.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.netty;
-
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.http.DefaultFullHttpResponse;
-import io.netty.handler.codec.http.HttpHeaders;
-import io.netty.handler.codec.http.HttpRequest;
-import io.netty.handler.codec.http.HttpResponse;
-import org.apache.hadoop.io.IOUtils;
-
-import java.io.PipedInputStream;
-import java.io.PipedOutputStream;
-import java.util.concurrent.Future;
-
-import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST;
-import static io.netty.handler.codec.http.HttpResponseStatus.CONTINUE;
-import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
-
-/**
- * Object Store Netty channel pipeline handler that handles an inbound
- * {@link HttpRequest} by dispatching it to the Object Store Jersey container.
- * The handler establishes 2 sets of connected piped streams: one for inbound
- * request handling and another for outbound response handling.  The relevant
- * ends of these pipes are handed off to the Jersey application dispatch and the
- * next channel handler, which is responsible for streaming in the inbound
- * request body and flushing out the response body.
- */
-public final class RequestDispatchObjectStoreChannelHandler
-    extends ObjectStoreChannelHandler<HttpRequest> {
-
-  private final ObjectStoreJerseyContainer jerseyContainer;
-
-  private PipedInputStream reqIn;
-  private PipedOutputStream reqOut;
-  private PipedInputStream respIn;
-  private PipedOutputStream respOut;
-
-  /**
-   * Creates a new RequestDispatchObjectStoreChannelHandler.
-   *
-   * @param jerseyContainer Object Store application Jersey container for
-   * request dispatch
-   */
-  public RequestDispatchObjectStoreChannelHandler(
-      ObjectStoreJerseyContainer jerseyContainer) {
-    this.jerseyContainer = jerseyContainer;
-  }
-
-  @Override
-  public void channelRead0(ChannelHandlerContext ctx, HttpRequest nettyReq)
-      throws Exception {
-    LOG.trace("begin RequestDispatchObjectStoreChannelHandler channelRead0, " +
-        "ctx = {}, nettyReq = {}", ctx, nettyReq);
-    if (!nettyReq.getDecoderResult().isSuccess()) {
-      sendErrorResponse(ctx, BAD_REQUEST);
-      return;
-    }
-
-    this.reqIn = new PipedInputStream();
-    this.reqOut = new PipedOutputStream(reqIn);
-    this.respIn = new PipedInputStream();
-    this.respOut = new PipedOutputStream(respIn);
-
-    if (HttpHeaders.is100ContinueExpected(nettyReq)) {
-      LOG.trace("Sending continue response.");
-      ctx.writeAndFlush(new DefaultFullHttpResponse(HTTP_1_1, CONTINUE));
-    }
-
-    Future<HttpResponse> nettyResp = this.jerseyContainer.dispatch(nettyReq,
-        reqIn, respOut);
-
-    ctx.pipeline().replace(this,
-        RequestContentObjectStoreChannelHandler.class.getSimpleName(),
-        new RequestContentObjectStoreChannelHandler(nettyReq, nettyResp,
-            reqOut, respIn, jerseyContainer));
-
-    LOG.trace("end RequestDispatchObjectStoreChannelHandler channelRead0, " +
-        "ctx = {}, nettyReq = {}", ctx, nettyReq);
-  }
-
-  @Override
-  public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
-    super.exceptionCaught(ctx, cause);
-    IOUtils.cleanupWithLogger(null, this.reqIn, this.reqOut, this.respIn,
-        this.respOut);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/package-info.java
deleted file mode 100644
index f4aa675..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/package-info.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Netty-based HTTP server implementation for Ozone.
- */
-@InterfaceAudience.Private
-package org.apache.hadoop.ozone.web.netty;
-
-import org.apache.hadoop.classification.InterfaceAudience;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
deleted file mode 100644
index 015d46e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClient;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.http.client.utils.URIBuilder;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-/**
- * Common interface for command handling.
- */
-public abstract class Handler {
-  protected OzoneRestClient client;
-
-  /**
-   * Constructs a client object.
-   */
-  public Handler() {
-    client = new OzoneRestClient();
-  }
-
-  /**
-   * Executes the Client command.
-   *
-   * @param cmd - CommandLine
-   * @throws IOException
-   * @throws OzoneException
-   * @throws URISyntaxException
-   */
-  protected abstract void execute(CommandLine cmd)
-      throws IOException, OzoneException, URISyntaxException;
-
-  /**
-   * verifies user provided URI.
-   *
-   * @param uri - UriString
-   * @return URI
-   * @throws URISyntaxException
-   * @throws OzoneException
-   */
-  protected URI verifyURI(String uri) throws URISyntaxException,
-      OzoneException {
-    if ((uri == null) || uri.isEmpty()) {
-      throw new OzoneRestClientException(
-          "Ozone URI is needed to execute this command.");
-    }
-    URIBuilder ozoneURI = new URIBuilder(uri);
-
-    if (ozoneURI.getPort() == 0) {
-      ozoneURI.setPort(Shell.DEFAULT_OZONE_PORT);
-    }
-    return ozoneURI.build();
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[22/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DeletedBlockLogImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DeletedBlockLogImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DeletedBlockLogImpl.java
deleted file mode 100644
index c5fd749..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DeletedBlockLogImpl.java
+++ /dev/null
@@ -1,351 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.block;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
-import com.google.common.primitives.Longs;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.utils.BatchOperation;
-import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import static org.apache.hadoop.ozone.OzoneConsts.DELETED_BLOCK_DB;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_BLOCK_DELETION_MAX_RETRY;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_BLOCK_DELETION_MAX_RETRY_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB;
-
-/**
- * A implement class of {@link DeletedBlockLog}, and it uses
- * K/V db to maintain block deletion transactions between scm and datanode.
- * This is a very basic implementation, it simply scans the log and
- * memorize the position that scanned by last time, and uses this to
- * determine where the next scan starts. It has no notion about weight
- * of each transaction so as long as transaction is still valid, they get
- * equally same chance to be retrieved which only depends on the nature
- * order of the transaction ID.
- */
-public class DeletedBlockLogImpl implements DeletedBlockLog {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(DeletedBlockLogImpl.class);
-
-  private static final byte[] LATEST_TXID =
-      DFSUtil.string2Bytes("#LATEST_TXID#");
-
-  private final int maxRetry;
-  private final MetadataStore deletedStore;
-  private final Lock lock;
-  // The latest id of deleted blocks in the db.
-  private long lastTxID;
-  private long lastReadTxID;
-
-  public DeletedBlockLogImpl(Configuration conf) throws IOException {
-    maxRetry = conf.getInt(OZONE_SCM_BLOCK_DELETION_MAX_RETRY,
-        OZONE_SCM_BLOCK_DELETION_MAX_RETRY_DEFAULT);
-
-    File metaDir = OzoneUtils.getOzoneMetaDirPath(conf);
-    String scmMetaDataDir = metaDir.getPath();
-    File deletedLogDbPath = new File(scmMetaDataDir, DELETED_BLOCK_DB);
-    int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
-        OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
-    // Load store of all transactions.
-    deletedStore = MetadataStoreBuilder.newBuilder()
-        .setCreateIfMissing(true)
-        .setConf(conf)
-        .setDbFile(deletedLogDbPath)
-        .setCacheSize(cacheSize * OzoneConsts.MB)
-        .build();
-
-    this.lock = new ReentrantLock();
-    // start from the head of deleted store.
-    lastReadTxID = 0;
-    lastTxID = findLatestTxIDInStore();
-  }
-
-  @VisibleForTesting
-  MetadataStore getDeletedStore() {
-    return deletedStore;
-  }
-
-  /**
-   * There is no need to lock before reading because
-   * it's only used in construct method.
-   *
-   * @return latest txid.
-   * @throws IOException
-   */
-  private long findLatestTxIDInStore() throws IOException {
-    long txid = 0;
-    byte[] value = deletedStore.get(LATEST_TXID);
-    if (value != null) {
-      txid = Longs.fromByteArray(value);
-    }
-    return txid;
-  }
-
-  @Override
-  public List<DeletedBlocksTransaction> getTransactions(
-      int count) throws IOException {
-    List<DeletedBlocksTransaction> result = new ArrayList<>();
-    MetadataKeyFilter getNextTxID = (preKey, currentKey, nextKey)
-        -> Longs.fromByteArray(currentKey) > lastReadTxID;
-    MetadataKeyFilter avoidInvalidTxid = (preKey, currentKey, nextKey)
-        -> !Arrays.equals(LATEST_TXID, currentKey);
-    lock.lock();
-    try {
-      deletedStore.iterate(null, (key, value) -> {
-        if (getNextTxID.filterKey(null, key, null) &&
-            avoidInvalidTxid.filterKey(null, key, null)) {
-          DeletedBlocksTransaction block = DeletedBlocksTransaction
-              .parseFrom(value);
-          if (block.getCount() > -1 && block.getCount() <= maxRetry) {
-            result.add(block);
-          }
-        }
-        return result.size() < count;
-      });
-      // Scan the metadata from the beginning.
-      if (result.size() < count || result.size() < 1) {
-        lastReadTxID = 0;
-      } else {
-        lastReadTxID = result.get(result.size() - 1).getTxID();
-      }
-    } finally {
-      lock.unlock();
-    }
-    return result;
-  }
-
-  @Override
-  public List<DeletedBlocksTransaction> getFailedTransactions()
-      throws IOException {
-    lock.lock();
-    try {
-      final List<DeletedBlocksTransaction> failedTXs = Lists.newArrayList();
-      deletedStore.iterate(null, (key, value) -> {
-        if (!Arrays.equals(LATEST_TXID, key)) {
-          DeletedBlocksTransaction delTX =
-              DeletedBlocksTransaction.parseFrom(value);
-          if (delTX.getCount() == -1) {
-            failedTXs.add(delTX);
-          }
-        }
-        return true;
-      });
-      return failedTXs;
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * @param txIDs - transaction ID.
-   * @throws IOException
-   */
-  @Override
-  public void incrementCount(List<Long> txIDs) throws IOException {
-    BatchOperation batch = new BatchOperation();
-    lock.lock();
-    try {
-      for(Long txID : txIDs) {
-        try {
-          DeletedBlocksTransaction block = DeletedBlocksTransaction
-              .parseFrom(deletedStore.get(Longs.toByteArray(txID)));
-          DeletedBlocksTransaction.Builder builder = block.toBuilder();
-          int currentCount = block.getCount();
-          if (currentCount > -1) {
-            builder.setCount(++currentCount);
-          }
-          // if the retry time exceeds the maxRetry value
-          // then set the retry value to -1, stop retrying, admins can
-          // analyze those blocks and purge them manually by SCMCli.
-          if (currentCount > maxRetry) {
-            builder.setCount(-1);
-          }
-          deletedStore.put(Longs.toByteArray(txID),
-              builder.build().toByteArray());
-        } catch (IOException ex) {
-          LOG.warn("Cannot increase count for txID " + txID, ex);
-        }
-      }
-      deletedStore.writeBatch(batch);
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  private DeletedBlocksTransaction constructNewTransaction(long txID,
-      String containerName, List<String> blocks) {
-    return DeletedBlocksTransaction.newBuilder()
-        .setTxID(txID)
-        .setContainerName(containerName)
-        .addAllBlockID(blocks)
-        .setCount(0)
-        .build();
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * @param txIDs - transaction IDs.
-   * @throws IOException
-   */
-  @Override
-  public void commitTransactions(List<Long> txIDs) throws IOException {
-    lock.lock();
-    try {
-      for (Long txID : txIDs) {
-        try {
-          deletedStore.delete(Longs.toByteArray(txID));
-        } catch (IOException ex) {
-          LOG.warn("Cannot commit txID " + txID, ex);
-        }
-      }
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * @param containerName - container name.
-   * @param blocks - blocks that belong to the same container.
-   * @throws IOException
-   */
-  @Override
-  public void addTransaction(String containerName, List<String> blocks)
-      throws IOException {
-    BatchOperation batch = new BatchOperation();
-    lock.lock();
-    try {
-      DeletedBlocksTransaction tx = constructNewTransaction(lastTxID + 1,
-          containerName, blocks);
-      byte[] key = Longs.toByteArray(lastTxID + 1);
-
-      batch.put(key, tx.toByteArray());
-      batch.put(LATEST_TXID, Longs.toByteArray(lastTxID + 1));
-
-      deletedStore.writeBatch(batch);
-      lastTxID += 1;
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  @Override
-  public int getNumOfValidTransactions() throws IOException {
-    lock.lock();
-    try {
-      final AtomicInteger num = new AtomicInteger(0);
-      deletedStore.iterate(null, (key, value) -> {
-        // Exclude latest txid record
-        if (!Arrays.equals(LATEST_TXID, key)) {
-          DeletedBlocksTransaction delTX =
-              DeletedBlocksTransaction.parseFrom(value);
-          if (delTX.getCount() > -1) {
-            num.incrementAndGet();
-          }
-        }
-        return true;
-      });
-      return num.get();
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * @param containerBlocksMap a map of containerBlocks.
-   * @throws IOException
-   */
-  @Override
-  public void addTransactions(Map<String, List<String>> containerBlocksMap)
-      throws IOException {
-    BatchOperation batch = new BatchOperation();
-    lock.lock();
-    try {
-      long currentLatestID = lastTxID;
-      for (Map.Entry<String, List<String>> entry :
-          containerBlocksMap.entrySet()) {
-        currentLatestID += 1;
-        byte[] key = Longs.toByteArray(currentLatestID);
-        DeletedBlocksTransaction tx = constructNewTransaction(currentLatestID,
-            entry.getKey(), entry.getValue());
-        batch.put(key, tx.toByteArray());
-      }
-      lastTxID = currentLatestID;
-      batch.put(LATEST_TXID, Longs.toByteArray(lastTxID));
-      deletedStore.writeBatch(batch);
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  @Override
-  public void close() throws IOException {
-    if (deletedStore != null) {
-      deletedStore.close();
-    }
-  }
-
-  @Override
-  public void getTransactions(DatanodeDeletedBlockTransactions transactions)
-      throws IOException {
-    lock.lock();
-    try {
-      deletedStore.iterate(null, (key, value) -> {
-        if (!Arrays.equals(LATEST_TXID, key)) {
-          DeletedBlocksTransaction block = DeletedBlocksTransaction
-              .parseFrom(value);
-
-          if (block.getCount() > -1 && block.getCount() <= maxRetry) {
-            transactions.addTransaction(block);
-          }
-          return !transactions.isFull();
-        }
-        return true;
-      });
-    } finally {
-      lock.unlock();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/SCMBlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/SCMBlockDeletingService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/SCMBlockDeletingService.java
deleted file mode 100644
index eeef14b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/SCMBlockDeletingService.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.block;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-import org.apache.hadoop.ozone.scm.container.Mapping;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.BackgroundService;
-import org.apache.hadoop.utils.BackgroundTask;
-import org.apache.hadoop.utils.BackgroundTaskQueue;
-import org.apache.hadoop.utils.BackgroundTaskResult.EmptyTaskResult;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-/**
- * A background service running in SCM to delete blocks. This service scans
- * block deletion log in certain interval and caches block deletion commands
- * in {@link org.apache.hadoop.ozone.scm.node.CommandQueue}, asynchronously
- * SCM HB thread polls cached commands and sends them to datanode for physical
- * processing.
- */
-public class SCMBlockDeletingService extends BackgroundService {
-
-  static final Logger LOG =
-      LoggerFactory.getLogger(SCMBlockDeletingService.class);
-
-  // ThreadPoolSize=2, 1 for scheduler and the other for the scanner.
-  private final static int BLOCK_DELETING_SERVICE_CORE_POOL_SIZE = 2;
-  private final DeletedBlockLog deletedBlockLog;
-  private final Mapping mappingService;
-  private final NodeManager nodeManager;
-
-  // Block delete limit size is dynamically calculated based on container
-  // delete limit size (ozone.block.deleting.container.limit.per.interval)
-  // that configured for datanode. To ensure DN not wait for
-  // delete commands, we use this value multiply by a factor 2 as the final
-  // limit TX size for each node.
-  // Currently we implement a throttle algorithm that throttling delete blocks
-  // for each datanode. Each node is limited by the calculation size. Firstly
-  // current node info is fetched from nodemanager, then scan entire delLog
-  // from the beginning to end. If one node reaches maximum value, its records
-  // will be skipped. If not, keep scanning until it reaches maximum value.
-  // Once all node are full, the scan behavior will stop.
-  private int blockDeleteLimitSize;
-
-  public SCMBlockDeletingService(DeletedBlockLog deletedBlockLog,
-      Mapping mapper, NodeManager nodeManager,
-      long  interval, long serviceTimeout, Configuration conf) {
-    super("SCMBlockDeletingService", interval, TimeUnit.MILLISECONDS,
-        BLOCK_DELETING_SERVICE_CORE_POOL_SIZE, serviceTimeout);
-    this.deletedBlockLog = deletedBlockLog;
-    this.mappingService = mapper;
-    this.nodeManager = nodeManager;
-
-    int containerLimit = conf.getInt(
-        OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL,
-        OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT);
-    Preconditions.checkArgument(containerLimit > 0,
-        "Container limit size should be " + "positive.");
-    // Use container limit value multiply by a factor 2 to ensure DN
-    // not wait for orders.
-    this.blockDeleteLimitSize = containerLimit * 2;
-  }
-
-  @Override
-  public BackgroundTaskQueue getTasks() {
-    BackgroundTaskQueue queue = new BackgroundTaskQueue();
-    queue.add(new DeletedBlockTransactionScanner());
-    return queue;
-  }
-
-  private class DeletedBlockTransactionScanner
-      implements BackgroundTask<EmptyTaskResult> {
-
-    @Override
-    public int getPriority() {
-      return 1;
-    }
-
-    @Override
-    public EmptyTaskResult call() throws Exception {
-      int dnTxCount = 0;
-      long startTime = Time.monotonicNow();
-      // Scan SCM DB in HB interval and collect a throttled list of
-      // to delete blocks.
-      LOG.debug("Running DeletedBlockTransactionScanner");
-      DatanodeDeletedBlockTransactions transactions = null;
-      List<DatanodeID> datanodes = nodeManager.getNodes(NodeState.HEALTHY);
-      if (datanodes != null) {
-        transactions = new DatanodeDeletedBlockTransactions(mappingService,
-            blockDeleteLimitSize, datanodes.size());
-        try {
-          deletedBlockLog.getTransactions(transactions);
-        } catch (IOException e) {
-          // We may tolerant a number of failures for sometime
-          // but if it continues to fail, at some point we need to raise
-          // an exception and probably fail the SCM ? At present, it simply
-          // continues to retry the scanning.
-          LOG.error("Failed to get block deletion transactions from delTX log",
-              e);
-        }
-        LOG.debug("Scanned deleted blocks log and got {} delTX to process.",
-            transactions.getTXNum());
-      }
-
-      if (transactions != null && !transactions.isEmpty()) {
-        for (DatanodeID datanodeID : transactions.getDatanodes()) {
-          List<DeletedBlocksTransaction> dnTXs = transactions
-              .getDatanodeTransactions(datanodeID);
-          if (dnTXs != null && !dnTXs.isEmpty()) {
-            dnTxCount += dnTXs.size();
-            // TODO commandQueue needs a cap.
-            // We should stop caching new commands if num of un-processed
-            // command is bigger than a limit, e.g 50. In case datanode goes
-            // offline for sometime, the cached commands be flooded.
-            nodeManager.addDatanodeCommand(datanodeID,
-                new DeleteBlocksCommand(dnTXs));
-            LOG.debug(
-                "Added delete block command for datanode {} in the queue,"
-                    + " number of delete block transactions: {}, TxID list: {}",
-                datanodeID, dnTXs.size(), String.join(",",
-                    transactions.getTransactionIDList(datanodeID)));
-          }
-        }
-      }
-
-      if (dnTxCount > 0) {
-        LOG.info(
-            "Totally added {} delete blocks command for"
-                + " {} datanodes, task elapsed time: {}ms",
-            dnTxCount, transactions.getDatanodes().size(),
-            Time.monotonicNow() - startTime);
-      }
-
-      return EmptyTaskResult.newResult();
-    }
-  }
-
-  @VisibleForTesting
-  public void setBlockDeleteTXNum(int numTXs) {
-    blockDeleteLimitSize = numTXs;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/package-info.java
deleted file mode 100644
index 3c56627..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.block;
-/**
- * This package contains routines to manage the block location and
- * mapping inside SCM
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneBaseCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneBaseCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneBaseCLI.java
deleted file mode 100644
index 451b1b1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneBaseCLI.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.util.Tool;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-
-/**
- * This class is the base CLI for scm, ksm and scmadm.
- */
-public abstract class OzoneBaseCLI extends Configured implements Tool {
-
-  protected abstract int dispatch(CommandLine cmd, Options opts)
-      throws IOException, OzoneException, URISyntaxException;
-
-  protected abstract CommandLine parseArgs(String[] argv, Options opts)
-      throws ParseException;
-
-  protected abstract Options getOptions();
-
-  protected abstract void displayHelp();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneCommandHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneCommandHandler.java
deleted file mode 100644
index da12bd4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneCommandHandler.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.scm.client.ScmClient;
-
-import java.io.IOException;
-import java.io.PrintStream;
-
-/**
- * The abstract class of all SCM CLI commands.
- */
-public abstract class OzoneCommandHandler {
-
-  private ScmClient scmClient;
-  protected PrintStream out = System.out;
-  protected PrintStream err = System.err;
-
-  /**
-   * Constructs a handler object.
-   */
-  public OzoneCommandHandler(ScmClient scmClient) {
-    this.scmClient = scmClient;
-  }
-
-  protected ScmClient getScmClient() {
-    return scmClient;
-  }
-
-  /**
-   * Sets customized output stream to redirect the stdout to somewhere else.
-   * @param out
-   */
-  public void setOut(PrintStream out) {
-    this.out = out;
-  }
-
-  /**
-   * Sets customized error stream to redirect the stderr to somewhere else.
-   * @param err
-   */
-  public void setErr(PrintStream err) {
-    this.err = err;
-  }
-
-  public void logOut(String msg, String ... variable) {
-    this.out.println(String.format(msg, variable));
-  }
-
-  /**
-   * Executes the Client command.
-   *
-   * @param cmd - CommandLine.
-   * @throws IOException throws exception.
-   */
-  public abstract void execute(CommandLine cmd) throws IOException;
-
-  /**
-   * Display a help message describing the options the command takes.
-   * TODO : currently only prints to standard out, may want to change this.
-   */
-  public abstract void displayHelp();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/ResultCode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/ResultCode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/ResultCode.java
deleted file mode 100644
index 7b8704a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/ResultCode.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli;
-
-/**
- * The possible result code of SCM CLI.
- */
-public final class ResultCode {
-  public static final int SUCCESS = 1;
-
-  public static final int UNRECOGNIZED_CMD = 2;
-
-  public static final int EXECUTION_ERROR = 3;
-
-  private ResultCode() {}
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SCMCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SCMCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SCMCLI.java
deleted file mode 100644
index fb22d05..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SCMCLI.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli;
-
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.scm.cli.container.ContainerCommandHandler;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.client.ContainerOperationClient;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ToolRunner;
-
-import java.io.IOException;
-import java.io.PrintStream;
-import java.net.InetSocketAddress;
-import java.net.URISyntaxException;
-import java.util.Arrays;
-
-import static org.apache.hadoop.ozone.scm.cli.ResultCode.EXECUTION_ERROR;
-import static org.apache.hadoop.ozone.scm.cli.ResultCode.SUCCESS;
-import static org.apache.hadoop.ozone.scm.cli.ResultCode.UNRECOGNIZED_CMD;
-import static org.apache.hadoop.ozone.scm.cli.container.ContainerCommandHandler.CONTAINER_CMD;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB;
-
-/**
- * This class is the CLI of SCM.
- */
-public class SCMCLI extends OzoneBaseCLI {
-
-  public static final String HELP_OP = "help";
-  public static final int CMD_WIDTH = 80;
-
-  private final ScmClient scmClient;
-  private final PrintStream out;
-  private final PrintStream err;
-
-  private final Options options;
-
-  public SCMCLI(ScmClient scmClient) {
-    this(scmClient, System.out, System.err);
-  }
-
-  public SCMCLI(ScmClient scmClient, PrintStream out, PrintStream err) {
-    this.scmClient = scmClient;
-    this.out = out;
-    this.err = err;
-    this.options = getOptions();
-  }
-
-  /**
-   * Main for the scm shell Command handling.
-   *
-   * @param argv - System Args Strings[]
-   * @throws Exception
-   */
-  public static void main(String[] argv) throws Exception {
-    OzoneConfiguration conf = new OzoneConfiguration();
-    ScmClient scmClient = getScmClient(conf);
-    SCMCLI shell = new SCMCLI(scmClient);
-    conf.setQuietMode(false);
-    shell.setConf(conf);
-    int res = 0;
-    try {
-      res = ToolRunner.run(shell, argv);
-    } catch (Exception ex) {
-      System.exit(1);
-    }
-    System.exit(res);
-  }
-
-  private static ScmClient getScmClient(OzoneConfiguration ozoneConf)
-      throws IOException {
-    long version = RPC.getProtocolVersion(
-        StorageContainerLocationProtocolPB.class);
-    InetSocketAddress scmAddress =
-        OzoneClientUtils.getScmAddressForClients(ozoneConf);
-    int containerSizeGB = ozoneConf.getInt(OZONE_SCM_CONTAINER_SIZE_GB,
-        OZONE_SCM_CONTAINER_SIZE_DEFAULT);
-    ContainerOperationClient.setContainerSizeB(containerSizeGB*OzoneConsts.GB);
-
-    RPC.setProtocolEngine(ozoneConf, StorageContainerLocationProtocolPB.class,
-        ProtobufRpcEngine.class);
-    StorageContainerLocationProtocolClientSideTranslatorPB client =
-        new StorageContainerLocationProtocolClientSideTranslatorPB(
-            RPC.getProxy(StorageContainerLocationProtocolPB.class, version,
-                scmAddress, UserGroupInformation.getCurrentUser(), ozoneConf,
-                NetUtils.getDefaultSocketFactory(ozoneConf),
-                Client.getRpcTimeout(ozoneConf)));
-    ScmClient storageClient = new ContainerOperationClient(
-        client, new XceiverClientManager(ozoneConf));
-    return storageClient;
-  }
-
-  /**
-   * Adds ALL the options that hdfs scm command supports. Given the hierarchy
-   * of commands, the options are added in a cascading manner, e.g.:
-   * {@link SCMCLI} asks {@link ContainerCommandHandler} to add it's options,
-   * which then asks it's sub command, such as
-   * {@link org.apache.hadoop.ozone.scm.cli.container.CreateContainerHandler}
-   * to add it's own options.
-   *
-   * We need to do this because {@link BasicParser} need to take all the options
-   * when paring args.
-   * @return ALL the options supported by this CLI.
-   */
-  @Override
-  protected Options getOptions() {
-    Options newOptions = new Options();
-    // add the options
-    addTopLevelOptions(newOptions);
-    ContainerCommandHandler.addOptions(newOptions);
-    // TODO : add pool, node and pipeline commands.
-    addHelpOption(newOptions);
-    return newOptions;
-  }
-
-  private static void addTopLevelOptions(Options options) {
-    Option containerOps = new Option(
-        CONTAINER_CMD, false, "Container related options");
-    options.addOption(containerOps);
-    // TODO : add pool, node and pipeline commands.
-  }
-
-  private static void addHelpOption(Options options) {
-    Option helpOp = new Option(HELP_OP, false, "display help message");
-    options.addOption(helpOp);
-  }
-
-  @Override
-  protected void displayHelp() {
-    HelpFormatter helpFormatter = new HelpFormatter();
-    Options topLevelOptions = new Options();
-    addTopLevelOptions(topLevelOptions);
-    helpFormatter.printHelp(CMD_WIDTH, "hdfs scmcli <commands> [<options>]",
-        "where <commands> can be one of the following",
-        topLevelOptions, "");
-  }
-
-  @Override
-  public int run(String[] args) throws Exception {
-    CommandLine cmd = parseArgs(args, options);
-    if (cmd == null) {
-      err.println("Unrecognized options:" + Arrays.asList(args));
-      displayHelp();
-      return UNRECOGNIZED_CMD;
-    }
-    return dispatch(cmd, options);
-  }
-
-  /**
-   * This function parses all command line arguments
-   * and returns the appropriate values.
-   *
-   * @param argv - Argv from main
-   *
-   * @return CommandLine
-   */
-  @Override
-  protected CommandLine parseArgs(String[] argv, Options opts)
-      throws ParseException {
-    try {
-      BasicParser parser = new BasicParser();
-      return parser.parse(opts, argv);
-    } catch (ParseException ex) {
-      err.println(ex.getMessage());
-    }
-    return null;
-  }
-
-  @Override
-  protected int dispatch(CommandLine cmd, Options opts)
-      throws IOException, OzoneException, URISyntaxException {
-    OzoneCommandHandler handler = null;
-    try {
-      if (cmd.hasOption(CONTAINER_CMD)) {
-        handler = new ContainerCommandHandler(scmClient);
-      }
-
-      if (handler == null) {
-        if (cmd.hasOption(HELP_OP)) {
-          displayHelp();
-          return SUCCESS;
-        } else {
-          displayHelp();
-          err.println("Unrecognized command: " + Arrays.asList(cmd.getArgs()));
-          return UNRECOGNIZED_CMD;
-        }
-      } else {
-        // Redirect stdout and stderr if necessary.
-        handler.setOut(this.out);
-        handler.setErr(this.err);
-        handler.execute(cmd);
-        return SUCCESS;
-      }
-    } catch (IOException ioe) {
-      err.println("Error executing command:" + ioe);
-      return EXECUTION_ERROR;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
deleted file mode 100644
index 6152c55..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
+++ /dev/null
@@ -1,720 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.DFSUtilClient;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.OzoneAclInfo;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.BucketInfo;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.VolumeInfo;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.VolumeList;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.Pipeline;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.HashSet;
-import java.util.Set;
-
-import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_DB;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_SUFFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.KSM_DB_NAME;
-import static org.apache.hadoop.ozone.OzoneConsts.KSM_USER_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.KSM_BUCKET_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.KSM_VOLUME_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
-import static org.apache.hadoop.ozone.OzoneConsts.OPEN_CONTAINERS_DB;
-
-/**
- * This is the CLI that can be use to convert an ozone metadata DB into
- * a sqlite DB file.
- *
- * NOTE: user should use this CLI in an offline fashion. Namely, this should not
- * be used to convert a DB that is currently being used by Ozone. Instead,
- * this should be used to debug and diagnosis closed DB instances.
- *
- */
-public class SQLCLI  extends Configured implements Tool {
-
-  private Options options;
-  private BasicParser parser;
-  private final Charset encoding = Charset.forName("UTF-8");
-  private final OzoneConfiguration conf;
-
-  // for container.db
-  private static final String CREATE_CONTAINER_INFO =
-      "CREATE TABLE containerInfo (" +
-          "containerName TEXT PRIMARY KEY NOT NULL, " +
-          "leaderUUID TEXT NOT NULL)";
-  private static final String CREATE_CONTAINER_MEMBERS =
-      "CREATE TABLE containerMembers (" +
-          "containerName TEXT NOT NULL, " +
-          "datanodeUUID TEXT NOT NULL," +
-          "PRIMARY KEY(containerName, datanodeUUID));";
-  private static final String CREATE_DATANODE_INFO =
-      "CREATE TABLE datanodeInfo (" +
-          "hostName TEXT NOT NULL, " +
-          "datanodeUUId TEXT PRIMARY KEY NOT NULL," +
-          "ipAddr TEXT, " +
-          "xferPort INTEGER," +
-          "infoPort INTEGER," +
-          "ipcPort INTEGER," +
-          "infoSecurePort INTEGER," +
-          "containerPort INTEGER NOT NULL);";
-  private static final String INSERT_CONTAINER_INFO =
-      "INSERT INTO containerInfo (containerName, leaderUUID) " +
-          "VALUES (\"%s\", \"%s\")";
-  private static final String INSERT_DATANODE_INFO =
-      "INSERT INTO datanodeInfo (hostname, datanodeUUid, ipAddr, xferPort, " +
-          "infoPort, ipcPort, infoSecurePort, containerPort) " +
-          "VALUES (\"%s\", \"%s\", \"%s\", %d, %d, %d, %d, %d)";
-  private static final String INSERT_CONTAINER_MEMBERS =
-      "INSERT INTO containerMembers (containerName, datanodeUUID) " +
-          "VALUES (\"%s\", \"%s\")";
-  // for block.db
-  private static final String CREATE_BLOCK_CONTAINER =
-      "CREATE TABLE blockContainer (" +
-          "blockKey TEXT PRIMARY KEY NOT NULL, " +
-          "containerName TEXT NOT NULL)";
-  private static final String INSERT_BLOCK_CONTAINER =
-      "INSERT INTO blockContainer (blockKey, containerName) " +
-          "VALUES (\"%s\", \"%s\")";
-  // for nodepool.db
-  private static final String CREATE_NODE_POOL =
-      "CREATE TABLE nodePool (" +
-          "datanodeUUID TEXT NOT NULL," +
-          "poolName TEXT NOT NULL," +
-          "PRIMARY KEY(datanodeUUID, poolName))";
-  private static final String INSERT_NODE_POOL =
-      "INSERT INTO nodePool (datanodeUUID, poolName) " +
-          "VALUES (\"%s\", \"%s\")";
-  // and reuse CREATE_DATANODE_INFO and INSERT_DATANODE_INFO
-  // for openContainer.db
-  private static final String CREATE_OPEN_CONTAINER =
-      "CREATE TABLE openContainer (" +
-          "containerName TEXT PRIMARY KEY NOT NULL, " +
-          "containerUsed INTEGER NOT NULL)";
-  private static final String INSERT_OPEN_CONTAINER =
-      "INSERT INTO openContainer (containerName, containerUsed) " +
-          "VALUES (\"%s\", \"%s\")";
-
-  // for ksm.db
-  private static final String CREATE_VOLUME_LIST =
-      "CREATE TABLE volumeList (" +
-          "userName TEXT NOT NULL," +
-          "volumeName TEXT NOT NULL," +
-          "PRIMARY KEY (userName, volumeName))";
-  private static final String INSERT_VOLUME_LIST =
-      "INSERT INTO volumeList (userName, volumeName) " +
-          "VALUES (\"%s\", \"%s\")";
-
-  private static final String CREATE_VOLUME_INFO =
-      "CREATE TABLE volumeInfo (" +
-          "adminName TEXT NOT NULL," +
-          "ownerName TEXT NOT NULL," +
-          "volumeName TEXT NOT NULL," +
-          "PRIMARY KEY (adminName, ownerName, volumeName))";
-  private static final String INSERT_VOLUME_INFO =
-      "INSERT INTO volumeInfo (adminName, ownerName, volumeName) " +
-          "VALUES (\"%s\", \"%s\", \"%s\")";
-
-  private static final String CREATE_ACL_INFO =
-      "CREATE TABLE aclInfo (" +
-          "adminName TEXT NOT NULL," +
-          "ownerName TEXT NOT NULL," +
-          "volumeName TEXT NOT NULL," +
-          "type TEXT NOT NULL," +
-          "userName TEXT NOT NULL," +
-          "rights TEXT NOT NULL," +
-          "FOREIGN KEY (adminName, ownerName, volumeName, userName, type)" +
-          "REFERENCES " +
-          "volumeInfo(adminName, ownerName, volumeName, userName, type)" +
-          "PRIMARY KEY (adminName, ownerName, volumeName, userName, type))";
-  private static final String INSERT_ACL_INFO =
-      "INSERT INTO aclInfo (adminName, ownerName, volumeName, type, " +
-          "userName, rights) " +
-          "VALUES (\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\")";
-
-  private static final String CREATE_BUCKET_INFO =
-      "CREATE TABLE bucketInfo (" +
-          "volumeName TEXT NOT NULL," +
-          "bucketName TEXT NOT NULL," +
-          "versionEnabled BOOLEAN NOT NULL," +
-          "storageType TEXT," +
-          "PRIMARY KEY (volumeName, bucketName))";
-  private static final String INSERT_BUCKET_INFO =
-      "INSERT INTO bucketInfo(volumeName, bucketName, " +
-          "versionEnabled, storageType)" +
-          "VALUES (\"%s\", \"%s\", \"%s\", \"%s\")";
-
-  private static final String CREATE_KEY_INFO =
-      "CREATE TABLE keyInfo (" +
-          "volumeName TEXT NOT NULL," +
-          "bucketName TEXT NOT NULL," +
-          "keyName TEXT NOT NULL," +
-          "dataSize INTEGER," +
-          "blockKey TEXT NOT NULL," +
-          "containerName TEXT NOT NULL," +
-          "PRIMARY KEY (volumeName, bucketName, keyName))";
-  private static final String INSERT_KEY_INFO =
-      "INSERT INTO keyInfo (volumeName, bucketName, keyName, dataSize, " +
-          "blockKey, containerName)" +
-          "VALUES (\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\")";
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(SQLCLI.class);
-
-  public SQLCLI(OzoneConfiguration conf) {
-    this.options = getOptions();
-    this.parser = new BasicParser();
-    this.conf = conf;
-  }
-
-  @SuppressWarnings("static-access")
-  private Options getOptions() {
-    Options allOptions = new Options();
-    Option helpOpt = OptionBuilder
-        .hasArg(false)
-        .withLongOpt("help")
-        .withDescription("display help message")
-        .create("h");
-    allOptions.addOption(helpOpt);
-
-    Option dbPathOption = OptionBuilder
-        .withArgName("DB path")
-        .withLongOpt("dbPath")
-        .hasArgs(1)
-        .withDescription("specify DB path")
-        .create("p");
-    allOptions.addOption(dbPathOption);
-
-    Option outPathOption = OptionBuilder
-        .withArgName("output path")
-        .withLongOpt("outPath")
-        .hasArgs(1)
-        .withDescription("specify output DB file path")
-        .create("o");
-    allOptions.addOption(outPathOption);
-
-    return allOptions;
-  }
-
-  public void displayHelp() {
-    HelpFormatter helpFormatter = new HelpFormatter();
-    Options allOpts = getOptions();
-    helpFormatter.printHelp("hdfs oz_debug -p <DB path>"
-        + " -o <Output DB file path>", allOpts);
-  }
-
-  @Override
-  public int run(String[] args) throws Exception {
-    CommandLine commandLine = parseArgs(args);
-    if (commandLine.hasOption("help")) {
-      displayHelp();
-      return 0;
-    }
-    if (!commandLine.hasOption("p") || !commandLine.hasOption("o")) {
-      displayHelp();
-      return -1;
-    }
-    String value = commandLine.getOptionValue("p");
-    LOG.info("DB path {}", value);
-    // the value is supposed to be an absolute path to a container file
-    Path dbPath = Paths.get(value);
-    if (!Files.exists(dbPath)) {
-      LOG.error("DB path not exist:{}", dbPath);
-    }
-    Path parentPath = dbPath.getParent();
-    Path dbName = dbPath.getFileName();
-    if (parentPath == null || dbName == null) {
-      LOG.error("Error processing db path {}", dbPath);
-      return -1;
-    }
-
-    value = commandLine.getOptionValue("o");
-    Path outPath = Paths.get(value);
-    if (outPath == null || outPath.getParent() == null) {
-      LOG.error("Error processing output path {}", outPath);
-      return -1;
-    }
-
-    if (outPath.toFile().isDirectory()) {
-      LOG.error("The db output path should be a file instead of a directory");
-      return -1;
-    }
-
-    Path outParentPath = outPath.getParent();
-    if (outParentPath != null) {
-      if (!Files.exists(outParentPath)) {
-        Files.createDirectories(outParentPath);
-      }
-    }
-    LOG.info("Parent path [{}] db name [{}]", parentPath, dbName);
-    if (dbName.toString().endsWith(CONTAINER_DB_SUFFIX)) {
-      LOG.info("Converting container DB");
-      convertContainerDB(dbPath, outPath);
-    } else if (dbName.toString().equals(BLOCK_DB)) {
-      LOG.info("Converting block DB");
-      convertBlockDB(dbPath, outPath);
-    } else if (dbName.toString().equals(NODEPOOL_DB)) {
-      LOG.info("Converting node pool DB");
-      convertNodePoolDB(dbPath, outPath);
-    } else if (dbName.toString().equals(OPEN_CONTAINERS_DB)) {
-      LOG.info("Converting open container DB");
-      convertOpenContainerDB(dbPath, outPath);
-    } else if (dbName.toString().equals(KSM_DB_NAME)) {
-      LOG.info("Converting ksm DB");
-      convertKSMDB(dbPath, outPath);
-    } else {
-      LOG.error("Unrecognized db name {}", dbName);
-    }
-    return 0;
-  }
-
-  private Connection connectDB(String dbPath) throws Exception {
-    Class.forName("org.sqlite.JDBC");
-    String connectPath =
-        String.format("jdbc:sqlite:%s", dbPath);
-    return DriverManager.getConnection(connectPath);
-  }
-
-  private void executeSQL(Connection conn, String sql) throws SQLException {
-    try (Statement stmt = conn.createStatement()) {
-      stmt.executeUpdate(sql);
-    }
-  }
-
-  /**
-   * Convert ksm.db to sqlite db file. With following schema.
-   * (* for primary key)
-   *
-   * 1. for key type USER, it contains a username and a list volumes
-   * volumeList
-   * --------------------------------
-   *   userName*     |  volumeName*
-   * --------------------------------
-   *
-   * 2. for key type VOLUME:
-   *
-   * volumeInfo
-   * ----------------------------------------------
-   * adminName | ownerName* | volumeName* | aclID
-   * ----------------------------------------------
-   *
-   * aclInfo
-   * ----------------------------------------------
-   * aclEntryID* | type* | userName* | rights
-   * ----------------------------------------------
-   *
-   * 3. for key type BUCKET
-   * bucketInfo
-   * --------------------------------------------------------
-   * volumeName* | bucketName* | versionEnabled | storageType
-   * --------------------------------------------------------
-   *
-   * TODO : the following table will be changed when key partition is added.
-   * Only has the minimum entries for test purpose now.
-   * 4. for key type KEY
-   * -----------------------------------------------
-   * volumeName* | bucketName* | keyName* | dataSize
-   * -----------------------------------------------
-   *
-   *
-   *
-   * @param dbPath
-   * @param outPath
-   * @throws Exception
-   */
-  private void convertKSMDB(Path dbPath, Path outPath) throws Exception {
-    LOG.info("Create tables for sql ksm db.");
-    File dbFile = dbPath.toFile();
-    try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
-        .setConf(conf).setDbFile(dbFile).build();
-         Connection conn = connectDB(outPath.toString())) {
-      executeSQL(conn, CREATE_VOLUME_LIST);
-      executeSQL(conn, CREATE_VOLUME_INFO);
-      executeSQL(conn, CREATE_ACL_INFO);
-      executeSQL(conn, CREATE_BUCKET_INFO);
-      executeSQL(conn, CREATE_KEY_INFO);
-
-      dbStore.iterate(null, (key, value) -> {
-        String keyString = DFSUtilClient.bytes2String(key);
-        KeyType type = getKeyType(keyString);
-        try {
-          insertKSMDB(conn, type, keyString, value);
-        } catch (IOException | SQLException ex) {
-          LOG.error("Exception inserting key {} type {}", keyString, type, ex);
-        }
-        return true;
-      });
-    }
-  }
-
-  private void insertKSMDB(Connection conn, KeyType type, String keyName,
-      byte[] value) throws IOException, SQLException {
-    switch (type) {
-    case USER:
-      VolumeList volumeList = VolumeList.parseFrom(value);
-      for (String volumeName : volumeList.getVolumeNamesList()) {
-        String insertVolumeList =
-            String.format(INSERT_VOLUME_LIST, keyName, volumeName);
-        executeSQL(conn, insertVolumeList);
-      }
-      break;
-    case VOLUME:
-      VolumeInfo volumeInfo = VolumeInfo.parseFrom(value);
-      String adminName = volumeInfo.getAdminName();
-      String ownerName = volumeInfo.getOwnerName();
-      String volumeName = volumeInfo.getVolume();
-      String insertVolumeInfo =
-          String.format(INSERT_VOLUME_INFO, adminName, ownerName, volumeName);
-      executeSQL(conn, insertVolumeInfo);
-      for (OzoneAclInfo aclInfo : volumeInfo.getVolumeAclsList()) {
-        String insertAclInfo =
-            String.format(INSERT_ACL_INFO, adminName, ownerName, volumeName,
-                aclInfo.getType(), aclInfo.getName(), aclInfo.getRights());
-        executeSQL(conn, insertAclInfo);
-      }
-      break;
-    case BUCKET:
-      BucketInfo bucketInfo = BucketInfo.parseFrom(value);
-      String insertBucketInfo =
-          String.format(INSERT_BUCKET_INFO, bucketInfo.getVolumeName(),
-              bucketInfo.getBucketName(), bucketInfo.getIsVersionEnabled(),
-              bucketInfo.getStorageType());
-      executeSQL(conn, insertBucketInfo);
-      break;
-    case KEY:
-      KeyInfo keyInfo = KeyInfo.parseFrom(value);
-      // TODO : the two fields container name and block id are no longer used,
-      // need to revisit this later.
-      String insertKeyInfo =
-          String.format(INSERT_KEY_INFO, keyInfo.getVolumeName(),
-              keyInfo.getBucketName(), keyInfo.getKeyName(),
-              keyInfo.getDataSize(), "EMPTY",
-              "EMPTY");
-      executeSQL(conn, insertKeyInfo);
-      break;
-    default:
-      throw new IOException("Unknown key from ksm.db");
-    }
-  }
-
-  private KeyType getKeyType(String key) {
-    if (key.startsWith(KSM_USER_PREFIX)) {
-      return KeyType.USER;
-    } else if (key.startsWith(KSM_VOLUME_PREFIX)) {
-      return key.replaceFirst(KSM_VOLUME_PREFIX, "")
-          .contains(KSM_BUCKET_PREFIX) ? KeyType.BUCKET : KeyType.VOLUME;
-    }else {
-      return KeyType.KEY;
-    }
-  }
-
-  private enum KeyType {
-    USER,
-    VOLUME,
-    BUCKET,
-    KEY,
-    UNKNOWN
-  }
-
-  /**
-   * Convert container.db to sqlite. The schema of sql db:
-   * three tables, containerId, containerMachines, datanodeInfo
-   * (* for primary key)
-   *
-   * containerInfo:
-   * ----------------------------------------------
-   * container name* | container lead datanode uuid
-   * ----------------------------------------------
-   *
-   * containerMembers:
-   * --------------------------------
-   * container name* |  datanodeUUid*
-   * --------------------------------
-   *
-   * datanodeInfo:
-   * ---------------------------------------------------------
-   * hostname | datanodeUUid* | xferPort | infoPort | ipcPort
-   * ---------------------------------------------------------
-   *
-   * --------------------------------
-   * | infoSecurePort | containerPort
-   * --------------------------------
-   *
-   * @param dbPath path to container db.
-   * @param outPath path to output sqlite
-   * @throws IOException throws exception.
-   */
-  private void convertContainerDB(Path dbPath, Path outPath)
-      throws Exception {
-    LOG.info("Create tables for sql container db.");
-    File dbFile = dbPath.toFile();
-    try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
-        .setConf(conf).setDbFile(dbFile).build();
-        Connection conn = connectDB(outPath.toString())) {
-      executeSQL(conn, CREATE_CONTAINER_INFO);
-      executeSQL(conn, CREATE_CONTAINER_MEMBERS);
-      executeSQL(conn, CREATE_DATANODE_INFO);
-
-      HashSet<String> uuidChecked = new HashSet<>();
-      dbStore.iterate(null, (key, value) -> {
-        String containerName = new String(key, encoding);
-        ContainerInfo containerInfo = null;
-        containerInfo = ContainerInfo.fromProtobuf(
-            OzoneProtos.SCMContainerInfo.PARSER.parseFrom(value));
-        Preconditions.checkNotNull(containerInfo);
-        try {
-          //TODO: include container state to sqllite schema
-          insertContainerDB(conn, containerName,
-              containerInfo.getPipeline().getProtobufMessage(), uuidChecked);
-          return true;
-        } catch (SQLException e) {
-          throw new IOException(e);
-        }
-      });
-    }
-  }
-
-  /**
-   * Insert into the sqlite DB of container.db.
-   * @param conn the connection to the sqlite DB.
-   * @param containerName the name of the container.
-   * @param pipeline the actual container pipeline object.
-   * @param uuidChecked the uuid that has been already inserted.
-   * @throws SQLException throws exception.
-   */
-  private void insertContainerDB(Connection conn, String containerName,
-      Pipeline pipeline, Set<String> uuidChecked) throws SQLException {
-    LOG.info("Insert to sql container db, for container {}", containerName);
-    String insertContainerInfo = String.format(
-        INSERT_CONTAINER_INFO, containerName,
-        pipeline.getPipelineChannel().getLeaderID());
-    executeSQL(conn, insertContainerInfo);
-
-    for (HdfsProtos.DatanodeIDProto dnID :
-        pipeline.getPipelineChannel().getMembersList()) {
-      String uuid = dnID.getDatanodeUuid();
-      if (!uuidChecked.contains(uuid)) {
-        // we may also not use this checked set, but catch exception instead
-        // but this seems a bit cleaner.
-        String ipAddr = dnID.getIpAddr();
-        String hostName = dnID.getHostName();
-        int xferPort = dnID.hasXferPort() ? dnID.getXferPort() : 0;
-        int infoPort = dnID.hasInfoPort() ? dnID.getInfoPort() : 0;
-        int securePort =
-            dnID.hasInfoSecurePort() ? dnID.getInfoSecurePort() : 0;
-        int ipcPort = dnID.hasIpcPort() ? dnID.getIpcPort() : 0;
-        int containerPort = dnID.getContainerPort();
-        String insertMachineInfo = String.format(
-            INSERT_DATANODE_INFO, hostName, uuid, ipAddr, xferPort, infoPort,
-            ipcPort, securePort, containerPort);
-        executeSQL(conn, insertMachineInfo);
-        uuidChecked.add(uuid);
-      }
-      String insertContainerMembers = String.format(
-          INSERT_CONTAINER_MEMBERS, containerName, uuid);
-      executeSQL(conn, insertContainerMembers);
-    }
-    LOG.info("Insertion completed.");
-  }
-
-  /**
-   * Converts block.db to sqlite. This is rather simple db, the schema has only
-   * one table:
-   *
-   * blockContainer
-   * --------------------------
-   * blockKey*  | containerName
-   * --------------------------
-   *
-   * @param dbPath path to container db.
-   * @param outPath path to output sqlite
-   * @throws IOException throws exception.
-   */
-  private void convertBlockDB(Path dbPath, Path outPath) throws Exception {
-    LOG.info("Create tables for sql block db.");
-    File dbFile = dbPath.toFile();
-    try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
-        .setConf(conf).setDbFile(dbFile).build();
-        Connection conn = connectDB(outPath.toString())) {
-      executeSQL(conn, CREATE_BLOCK_CONTAINER);
-
-      dbStore.iterate(null, (key, value) -> {
-        String blockKey = DFSUtilClient.bytes2String(key);
-        String containerName = DFSUtilClient.bytes2String(value);
-        String insertBlockContainer = String.format(
-            INSERT_BLOCK_CONTAINER, blockKey, containerName);
-
-        try {
-          executeSQL(conn, insertBlockContainer);
-          return true;
-        } catch (SQLException e) {
-          throw new IOException(e);
-        }
-      });
-    }
-  }
-
-  /**
-   * Converts nodePool.db to sqlite. The schema of sql db:
-   * two tables, nodePool and datanodeInfo (the same datanode Info as for
-   * container.db).
-   *
-   * nodePool
-   * ---------------------------------------------------------
-   * datanodeUUID* | poolName*
-   * ---------------------------------------------------------
-   *
-   * datanodeInfo:
-   * ---------------------------------------------------------
-   * hostname | datanodeUUid* | xferPort | infoPort | ipcPort
-   * ---------------------------------------------------------
-   *
-   * --------------------------------
-   * | infoSecurePort | containerPort
-   * --------------------------------
-   *
-   * @param dbPath path to container db.
-   * @param outPath path to output sqlite
-   * @throws IOException throws exception.
-   */
-  private void convertNodePoolDB(Path dbPath, Path outPath) throws Exception {
-    LOG.info("Create table for sql node pool db.");
-    File dbFile = dbPath.toFile();
-    try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
-        .setConf(conf).setDbFile(dbFile).build();
-        Connection conn = connectDB(outPath.toString())) {
-      executeSQL(conn, CREATE_NODE_POOL);
-      executeSQL(conn, CREATE_DATANODE_INFO);
-
-      dbStore.iterate(null, (key, value) -> {
-        DatanodeID nodeId = DatanodeID
-            .getFromProtoBuf(HdfsProtos.DatanodeIDProto.PARSER.parseFrom(key));
-        String blockPool = DFSUtil.bytes2String(value);
-        try {
-          insertNodePoolDB(conn, blockPool, nodeId);
-          return true;
-        } catch (SQLException e) {
-          throw new IOException(e);
-        }
-      });
-    }
-  }
-
-  private void insertNodePoolDB(Connection conn, String blockPool,
-      DatanodeID datanodeID) throws SQLException {
-    String insertNodePool = String.format(INSERT_NODE_POOL,
-        datanodeID.getDatanodeUuid(), blockPool);
-    executeSQL(conn, insertNodePool);
-
-    String insertDatanodeID = String.format(INSERT_DATANODE_INFO,
-        datanodeID.getHostName(), datanodeID.getDatanodeUuid(),
-        datanodeID.getIpAddr(), datanodeID.getXferPort(),
-        datanodeID.getInfoPort(), datanodeID.getIpcPort(),
-        datanodeID.getInfoSecurePort(), datanodeID.getContainerPort());
-    executeSQL(conn, insertDatanodeID);
-  }
-
-  /**
-   * Convert openContainer.db to sqlite db file. This is rather simple db,
-   * the schema has only one table:
-   *
-   * openContainer
-   * -------------------------------
-   * containerName* | containerUsed
-   * -------------------------------
-   *
-   * @param dbPath path to container db.
-   * @param outPath path to output sqlite
-   * @throws IOException throws exception.
-   */
-  private void convertOpenContainerDB(Path dbPath, Path outPath)
-      throws Exception {
-    LOG.info("Create table for open container db.");
-    File dbFile = dbPath.toFile();
-    try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
-        .setConf(conf).setDbFile(dbFile).build();
-        Connection conn = connectDB(outPath.toString())) {
-      executeSQL(conn, CREATE_OPEN_CONTAINER);
-
-      dbStore.iterate(null, (key, value) -> {
-        String containerName = DFSUtil.bytes2String(key);
-        Long containerUsed =
-            Long.parseLong(DFSUtil.bytes2String(value));
-        String insertOpenContainer = String
-            .format(INSERT_OPEN_CONTAINER, containerName, containerUsed);
-        try {
-          executeSQL(conn, insertOpenContainer);
-          return true;
-        } catch (SQLException e) {
-          throw new IOException(e);
-        }
-      });
-    }
-  }
-
-  private CommandLine parseArgs(String[] argv)
-      throws ParseException {
-    return parser.parse(options, argv);
-  }
-
-  public static void main(String[] args) {
-    Tool shell = new SQLCLI(new OzoneConfiguration());
-    int res = 0;
-    try {
-      ToolRunner.run(shell, args);
-    } catch (Exception ex) {
-      LOG.error(ex.toString());
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Command execution failed", ex);
-      }
-      res = 1;
-    }
-    System.exit(res);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CloseContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CloseContainerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CloseContainerHandler.java
deleted file mode 100644
index 7a52305..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CloseContainerHandler.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli.container;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.ozone.scm.cli.OzoneCommandHandler;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.CMD_WIDTH;
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.HELP_OP;
-
-/**
- * The handler of close container command.
- */
-public class CloseContainerHandler extends OzoneCommandHandler {
-
-  public static final String CONTAINER_CLOSE = "close";
-  public static final String OPT_CONTAINER_NAME = "c";
-
-  @Override
-  public void execute(CommandLine cmd) throws IOException {
-    if (!cmd.hasOption(CONTAINER_CLOSE)) {
-      throw new IOException("Expecting container close");
-    }
-    if (!cmd.hasOption(OPT_CONTAINER_NAME)) {
-      displayHelp();
-      if (!cmd.hasOption(HELP_OP)) {
-        throw new IOException("Expecting container name");
-      } else {
-        return;
-      }
-    }
-    String containerName = cmd.getOptionValue(OPT_CONTAINER_NAME);
-
-    Pipeline pipeline = getScmClient().getContainer(containerName);
-    if (pipeline == null) {
-      throw new IOException("Cannot close an non-exist container "
-          + containerName);
-    }
-    logOut("Closing container : %s.", containerName);
-    getScmClient().closeContainer(pipeline);
-    logOut("Container closed.");
-  }
-
-  @Override
-  public void displayHelp() {
-    Options options = new Options();
-    addOptions(options);
-    HelpFormatter helpFormatter = new HelpFormatter();
-    helpFormatter.printHelp(CMD_WIDTH, "hdfs scm -container -close <option>",
-        "where <option> is", options, "");
-  }
-
-  public static void addOptions(Options options) {
-    Option containerNameOpt = new Option(OPT_CONTAINER_NAME,
-        true, "Specify container name");
-    options.addOption(containerNameOpt);
-  }
-
-  CloseContainerHandler(ScmClient client) {
-    super(client);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ContainerCommandHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ContainerCommandHandler.java
deleted file mode 100644
index 7838b1f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ContainerCommandHandler.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli.container;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.ozone.scm.cli.OzoneCommandHandler;
-import org.apache.hadoop.scm.client.ScmClient;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.CMD_WIDTH;
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.HELP_OP;
-import static org.apache.hadoop.ozone.scm.cli.container.CloseContainerHandler.CONTAINER_CLOSE;
-import static org.apache.hadoop.ozone.scm.cli.container
-    .CreateContainerHandler.CONTAINER_CREATE;
-import static org.apache.hadoop.ozone.scm.cli.container
-    .DeleteContainerHandler.CONTAINER_DELETE;
-import static org.apache.hadoop.ozone.scm.cli.container
-    .InfoContainerHandler.CONTAINER_INFO;
-import static org.apache.hadoop.ozone.scm.cli.container
-    .ListContainerHandler.CONTAINER_LIST;
-
-/**
- * The handler class of container-specific commands, e.g. addContainer.
- */
-public class ContainerCommandHandler extends OzoneCommandHandler {
-
-  public static final String CONTAINER_CMD = "container";
-
-  public ContainerCommandHandler(ScmClient scmClient) {
-    super(scmClient);
-  }
-
-  @Override
-  public void execute(CommandLine cmd) throws IOException {
-    // all container commands should contain -container option
-    if (!cmd.hasOption(CONTAINER_CMD)) {
-      throw new IOException("Expecting container cmd");
-    }
-    // check which each the sub command it is
-    OzoneCommandHandler handler = null;
-    if (cmd.hasOption(CONTAINER_CREATE)) {
-      handler = new CreateContainerHandler(getScmClient());
-    } else if (cmd.hasOption(CONTAINER_DELETE)) {
-      handler = new DeleteContainerHandler(getScmClient());
-    } else if (cmd.hasOption(CONTAINER_INFO)) {
-      handler = new InfoContainerHandler(getScmClient());
-    } else if (cmd.hasOption(CONTAINER_LIST)) {
-      handler = new ListContainerHandler(getScmClient());
-    } else if (cmd.hasOption(CONTAINER_CLOSE)) {
-      handler = new CloseContainerHandler(getScmClient());
-    }
-
-    // execute the sub command, throw exception if no sub command found
-    // unless -help option is given.
-    if (handler != null) {
-      handler.setOut(this.out);
-      handler.setErr(this.err);
-      handler.execute(cmd);
-    } else {
-      displayHelp();
-      if (!cmd.hasOption(HELP_OP)) {
-        throw new IOException("Unrecognized command "
-            + Arrays.asList(cmd.getArgs()));
-      }
-    }
-  }
-
-  @Override
-  public void displayHelp() {
-    Options options = new Options();
-    addCommandsOption(options);
-    HelpFormatter helpFormatter = new HelpFormatter();
-    helpFormatter.printHelp(CMD_WIDTH,
-        "hdfs scm -container <commands> <options>",
-        "where <commands> can be one of the following", options, "");
-  }
-
-  private static void addCommandsOption(Options options) {
-    Option createContainer =
-        new Option(CONTAINER_CREATE, false, "Create container");
-    Option infoContainer =
-        new Option(CONTAINER_INFO, false, "Info container");
-    Option deleteContainer =
-        new Option(CONTAINER_DELETE, false, "Delete container");
-    Option listContainer =
-        new Option(CONTAINER_LIST, false, "List container");
-    Option closeContainer =
-        new Option(CONTAINER_CLOSE, false, "Close container");
-
-    options.addOption(createContainer);
-    options.addOption(deleteContainer);
-    options.addOption(infoContainer);
-    options.addOption(listContainer);
-    options.addOption(closeContainer);
-    // Every new option should add it's option here.
-  }
-
-  public static void addOptions(Options options) {
-    addCommandsOption(options);
-    // for create container options.
-    CreateContainerHandler.addOptions(options);
-    DeleteContainerHandler.addOptions(options);
-    InfoContainerHandler.addOptions(options);
-    ListContainerHandler.addOptions(options);
-    CloseContainerHandler.addOptions(options);
-    // Every new option should add it's option here.
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CreateContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CreateContainerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CreateContainerHandler.java
deleted file mode 100644
index 569c4a3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CreateContainerHandler.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli.container;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.ozone.scm.cli.OzoneCommandHandler;
-import org.apache.hadoop.scm.client.ScmClient;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.CMD_WIDTH;
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.HELP_OP;
-
-/**
- * This is the handler that process container creation command.
- */
-public class CreateContainerHandler extends OzoneCommandHandler {
-
-  public static final String CONTAINER_CREATE = "create";
-  public static final String OPT_CONTAINER_NAME = "c";
-  public static final String containerOwner = "OZONE";
-  // TODO Support an optional -p <pipelineID> option to create
-  // container on given datanodes.
-
-  public CreateContainerHandler(ScmClient scmClient) {
-    super(scmClient);
-  }
-
-  @Override
-  public void execute(CommandLine cmd) throws IOException {
-    if (!cmd.hasOption(CONTAINER_CREATE)) {
-      throw new IOException("Expecting container create");
-    }
-    if (!cmd.hasOption(OPT_CONTAINER_NAME)) {
-      displayHelp();
-      if (!cmd.hasOption(HELP_OP)) {
-        throw new IOException("Expecting container name");
-      } else {
-        return;
-      }
-    }
-    String containerName = cmd.getOptionValue(OPT_CONTAINER_NAME);
-
-    logOut("Creating container : %s.", containerName);
-    getScmClient().createContainer(containerName, containerOwner);
-    logOut("Container created.");
-  }
-
-  @Override
-  public void displayHelp() {
-    Options options = new Options();
-    addOptions(options);
-    HelpFormatter helpFormatter = new HelpFormatter();
-    helpFormatter.printHelp(CMD_WIDTH, "hdfs scm -container -create <option>",
-        "where <option> is", options, "");
-  }
-
-  public static void addOptions(Options options) {
-    Option containerNameOpt = new Option(OPT_CONTAINER_NAME,
-        true, "Specify container name");
-    options.addOption(containerNameOpt);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/DeleteContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/DeleteContainerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/DeleteContainerHandler.java
deleted file mode 100644
index 0782af3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/DeleteContainerHandler.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.cli.container;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.ozone.scm.cli.OzoneCommandHandler;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.CMD_WIDTH;
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.HELP_OP;
-
-/**
- * This is the handler that process delete container command.
- */
-public class DeleteContainerHandler extends OzoneCommandHandler {
-
-  protected static final String CONTAINER_DELETE = "delete";
-  protected static final String OPT_FORCE = "f";
-  protected static final String OPT_CONTAINER_NAME = "c";
-
-  public DeleteContainerHandler(ScmClient scmClient) {
-    super(scmClient);
-  }
-
-  @Override
-  public void execute(CommandLine cmd) throws IOException {
-    Preconditions.checkArgument(cmd.hasOption(CONTAINER_DELETE),
-        "Expecting command delete");
-    if (!cmd.hasOption(OPT_CONTAINER_NAME)) {
-      displayHelp();
-      if (!cmd.hasOption(HELP_OP)) {
-        throw new IOException("Expecting container name");
-      } else {
-        return;
-      }
-    }
-
-    String containerName = cmd.getOptionValue(OPT_CONTAINER_NAME);
-
-    Pipeline pipeline = getScmClient().getContainer(containerName);
-    if (pipeline == null) {
-      throw new IOException("Cannot delete an non-exist container "
-          + containerName);
-    }
-
-    logOut("Deleting container : %s.", containerName);
-    getScmClient().deleteContainer(pipeline, cmd.hasOption(OPT_FORCE));
-    logOut("Container %s deleted.", containerName);
-  }
-
-  @Override
-  public void displayHelp() {
-    Options options = new Options();
-    addOptions(options);
-    HelpFormatter helpFormatter = new HelpFormatter();
-    helpFormatter.printHelp(CMD_WIDTH, "hdfs scm -container -delete <option>",
-        "where <option> is", options, "");
-  }
-
-  public static void addOptions(Options options) {
-    Option forceOpt = new Option(OPT_FORCE,
-        false,
-        "forcibly delete a container");
-    options.addOption(forceOpt);
-    Option containerNameOpt = new Option(OPT_CONTAINER_NAME,
-        true, "Specify container name");
-    options.addOption(containerNameOpt);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/InfoContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/InfoContainerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/InfoContainerHandler.java
deleted file mode 100644
index f54a2b8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/InfoContainerHandler.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli.container;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerData;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.scm.cli.OzoneCommandHandler;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.CMD_WIDTH;
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.HELP_OP;
-
-/**
- * This is the handler that process container info command.
- */
-public class InfoContainerHandler extends OzoneCommandHandler {
-
-  public static final String CONTAINER_INFO = "info";
-  protected static final String OPT_CONTAINER_NAME = "c";
-
-  /**
-   * Constructs a handler object.
-   *
-   * @param scmClient scm client.
-   */
-  public InfoContainerHandler(ScmClient scmClient) {
-    super(scmClient);
-  }
-
-  @Override
-  public void execute(CommandLine cmd) throws IOException {
-    if (!cmd.hasOption(CONTAINER_INFO)) {
-      throw new IOException("Expecting container info");
-    }
-    if (!cmd.hasOption(OPT_CONTAINER_NAME)) {
-      displayHelp();
-      if (!cmd.hasOption(HELP_OP)) {
-        throw new IOException("Expecting container name");
-      } else {
-        return;
-      }
-    }
-    String containerName = cmd.getOptionValue(OPT_CONTAINER_NAME);
-    Pipeline pipeline = getScmClient().getContainer(containerName);
-    Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
-
-    ContainerData containerData =
-        getScmClient().readContainer(pipeline);
-
-    // Print container report info.
-    logOut("Container Name: %s",
-        containerData.getName());
-    String openStatus =
-        containerData.getState() == OzoneProtos.LifeCycleState.OPEN ? "OPEN" :
-            "CLOSED";
-    logOut("Container State: %s", openStatus);
-    if (!containerData.getHash().isEmpty()) {
-      logOut("Container Hash: %s", containerData.getHash());
-    }
-    logOut("Container DB Path: %s", containerData.getDbPath());
-    logOut("Container Path: %s", containerData.getContainerPath());
-
-    // Output meta data.
-    String metadataStr = containerData.getMetadataList().stream().map(
-        p -> p.getKey() + ":" + p.getValue()).collect(Collectors.joining(", "));
-    logOut("Container Metadata: {%s}", metadataStr);
-
-    // Print pipeline of an existing container.
-    logOut("LeaderID: %s", pipeline.getLeader().getHostName());
-    String machinesStr = pipeline.getMachines().stream().map(
-        DatanodeID::getHostName).collect(Collectors.joining(","));
-    logOut("Datanodes: [%s]", machinesStr);
-  }
-
-  @Override
-  public void displayHelp() {
-    Options options = new Options();
-    addOptions(options);
-    HelpFormatter helpFormatter = new HelpFormatter();
-    helpFormatter.printHelp(CMD_WIDTH, "hdfs scm -container -info <option>",
-        "where <option> is", options, "");
-  }
-
-  public static void addOptions(Options options) {
-    Option containerNameOpt = new Option(OPT_CONTAINER_NAME,
-        true, "Specify container name");
-    options.addOption(containerNameOpt);
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[04/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.js.map
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.js.map b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.js.map
deleted file mode 100644
index 594da5a3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.js.map
+++ /dev/null
@@ -1 +0,0 @@
-{"version":3,"file":"nv.d3.min.js","sources":["../src/core.js","../src/dom.js","../src/interactiveLayer.js","../src/tooltip.js","../src/utils.js","../src/models/axis.js","../src/models/boxPlot.js","../src/models/boxPlotChart.js","../src/models/bullet.js","../src/models/bulletChart.js","../src/models/candlestickBar.js","../src/models/cumulativeLineChart.js","../src/models/discreteBar.js","../src/models/discreteBarChart.js","../src/models/distribution.js","../src/models/focus.js","../src/models/forceDirectedGraph.js","../src/models/furiousLegend.js","../src/models/historicalBar.js","../src/models/historicalBarChart.js","../src/models/legend.js","../src/models/line.js","../src/models/lineChart.js","../src/models/linePlusBarChart.js","../src/models/multiBar.js","../src/models/multiBarChart.js","../src/models/multiBarHorizontal.js","../src/models/multiBarHorizontalChart.js","../src/models/multiChart.js","../src/models/ohlcBar.js","../src/models/parallelCoordinates.js","../src/models/para
 llelCoordinatesChart.js","../src/models/pie.js","../src/models/pieChart.js","../src/models/sankey.js","../src/models/sankeyChart.js","../src/models/scatter.js","../src/models/scatterChart.js","../src/models/sparkline.js","../src/models/sparklinePlus.js","../src/models/stackedArea.js","../src/models/stackedAreaChart.js","../src/models/sunburst.js","../src/models/sunburstChart.js"],"names":["nv","dev","tooltip","utils","models","charts","logs","dom","d3","require","dispatch","Function","prototype","bind","oThis","this","TypeError","aArgs","Array","slice","call","arguments","fToBind","fNOP","fBound","apply","concat","on","e","startTime","Date","endTime","totalTime","log","window","console","length","deprecated","name","info","warn","render","step","active","render_start","renderLoop","chart","graph","i","queue","generate","callback","splice","setTimeout","render_end","addGraph","obj","push","module","exports","write","undefined","fastdom","mutate","read","measure","interactiveGuideline
 ","layer","selection","each","data","mouseHandler","d3mouse","mouse","mouseX","mouseY","subtractMargin","mouseOutAnyReason","isMSIE","event","offsetX","offsetY","target","tagName","className","baseVal","match","margin","left","top","type","availableWidth","availableHeight","relatedTarget","ownerSVGElement","nvPointerEventsClass","elementMouseout","renderGuideLine","hidden","scaleIsOrdinal","xScale","rangeBands","pointXValue","elementIndex","bisect","range","rangeBand","domain","invert","elementMousemove","elementDblclick","elementClick","elementMouseDown","elementMouseUp","container","select","width","height","wrap","selectAll","wrapEnter","enter","append","attr","svgContainer","guideLine","x","showGuideLine","line","NaNtoZero","String","d","exit","remove","scale","linear","ActiveXObject","duration","hideDelay","_","interactiveBisect","values","searchVal","xAccessor","_xAccessor","_cmp","v","bisector","index","max","currentValue","nextIndex","min","nextValue","Math","abs","nearestVa
 lueIndex","threshold","yDistMax","Infinity","indexToHighlight","forEach","delta","initTooltip","node","document","body","id","classes","style","classed","nvtooltip","enabled","dataSeriesExists","newContent","contentGenerator","innerHTML","positionTooltip","floor","random","gravity","distance","snapDistance","lastPosition","headerEnabled","valueFormatter","headerFormatter","keyFormatter","table","createElement","theadEnter","html","value","tbodyEnter","trowEnter","p","series","highlight","color","total","key","filter","percent","format","opacityScale","opacity","outerHTML","footer","position","pos","clientX","clientY","getComputedStyle","transform","client","getBoundingClientRect","isArray","isObject","calcGravityOffset","tmp","offsetHeight","offsetWidth","clientWidth","documentElement","clientHeight","gravityOffset","interrupt","transition","delay","old_translate","new_translate","round","translateInterpolator","interpolateString","is_hidden","styleTween","options","optionsFunc","_o
 ptions","Object","create","get","set","chartContainer","fixedTop","offset","point","y","initOptions","windowSize","size","innerWidth","innerHeight","compatMode","a","isFunction","isDate","toString","isNumber","isNaN","windowResize","handler","addEventListener","clear","removeEventListener","getColor","defaultColor","color_scale","ordinal","category20","customTheme","dictionary","getKey","defaultColors","defIndex","pjax","links","content","load","href","fragment","parentNode","replaceChild","history","pushState","textContent","preventDefault","state","calcApproxTextWidth","svgTextElem","text","fontSize","parseInt","replace","textLength","n","watchTransition","renderWatch","args","_duration","renderStack","self","model","__rendered","m","arg","renderEnd","indexOf","reset","pop","every","deepExtend","dst","sources","source","srcObj","_setState","_getState","init","changed","getter","fn","setter","update","_set","settings","JSON","stringify","change","map","calcTicksX","numTicks","numVa
 lues","stream_len","calcTicksY","initOption","_calls","_overrides","ops","getOwnPropertyNames","calls","inheritOptionsD3","d3_source","oplist","_d3options","unshift","rebind","arrayUnique","sort","item","symbolMap","symbol","t","s","svg","symbolTypes","functor","inheritOptions","inherited","_inherited","d3ops","initSVG","nvd3-svg","sanitizeHeight","sanitizeWidth","bottom","right","noData","opt","noDataText","wrapTicks","word","words","split","reverse","lineNumber","lineHeight","dy","parseFloat","tspan","join","getComputedTextLength","arrayEquals","array1","array2","l","axis","g","ticks","orient","scale0","fmt","tickFormat","axisLabel","axisLabelText","xLabelMargin","axisMaxMin","w","isOrdinal","showMaxMin","tickPadding","axisLabelDistance","maxTextWidth","textHeight","xTicks","rotateLabelsRule","rotateLabels","box","sin","PI","staggerLabels","rotateYLabel","maxMinRange","err","copy","boxPlot","xDomain","getX","xRange","yData","yDomain","yMin","yMax","q1","getQ1","q3","getQ3","wl","g
 etWl","wh","getWh","olItems","getOlItems","getOlValue","yScale","yRange","xScale0","yScale0","boxplots","boxEnter","j","hover","f","box_width","maxBoxWidth","box_left","box_right","endpoint","elementMouseover","getQ2","outliers","getOlColor","getOlLabel","label","Q1","Q2","Q3","whisker_low","whisker_high","q2","itemColor","outlierValue","outlierLabel","outlierColor","boxPlotChart","boxplot","showXAxis","xAxis","showYAxis","yAxis","beforeUpdate","clamp","gEnter","defsEnter","rightAlignYAxis","barsWrap","datum","disabled","tickSize","evt","tooltipContent","bullet","sortLabels","labels","lz","b","iA","iB","descending","rangez","ranges","markerz","markers","markerLinez","markerLines","measurez","measures","rangeLabelz","rangeLabels","markerLabelz","markerLabels","markerLineLabelz","markerLineLabels","measureLabelz","measureLabels","x1","extent","merge","forceX","__chart__","il","rangeClassNames","legacyRangeClassNames","w1","xp1","h3","markerData","marker","markerLinesData","defaultRang
 eLabels","bulletChart","x0","title","subtitle","bulletWrap","tick","tickEnter","tickUpdate","timer","flush","candlestickBar","barWidth","padData","getLow","forceY","getHigh","chartClick","clipEdge","tickGroups","getOpen","getClose","getY","open","close","high","low","interactive","highlightPoint","pointIndex","isHoverOver","clearHighlights","cumulativeLineChart","lines","dragStart","dragMove","dx","updateZero","dragEnd","stateChange","indexLine","oldDuration","stateSetter","stateGetter","defaultState","indexDrag","behavior","drag","rescaleY","seriesDomains","initialDomain","completeDomain","indexify","interactivePointerEvents","showLegend","legend","marginTop","showControls","controlsData","controls","rightAlign","tempDisabled","useInteractiveGuideline","interactiveLayer","display","linesWrap","seriesIndex","avgLineData","average","avgLines","getAvgLineY","yVal","_ticks","newState","singlePoint","pointXLocation","allData","yValue","domainExtent","xValue","idx","indexifyYGetter","ind
 exValue","noErrorCheck","updateState","useVoronoi","discreteBar","seriesData","y0","showValues","groups","bars","barsEnter","element","stopPropagation","elementDblClick","valueFormat","rectClass","discreteBarChart","discretebar","wrapLabels","distribution","naxis","distWrap","dist","getData","focus","resizePath","updateBrushBG","brush","empty","brushExtent","brushBG","leftWidth","rightWidth","onBrush","shouldDispatch","contentWrap","syncBrushing","brushBGenter","gBrush","pointActive","interpolate","xTickFormat","yTickFormat","forceDirectedGraph","nodes","nodeFieldSet","Set","keys","add","force","layout","linkStrength","friction","linkDistance","linkDist","charge","theta","alpha","start","link","sqrt","radius","py","px","nodeColor","field","linkExtras","nodeExtras","furiousLegend","setTextColor","vers","expanded","disengaged","setBGColor","seriesShape","seriesEnter","property","seriesCheckbox","seriesText","legendMouseover","legendMouseout","legendClick","radioButtonMode","userDisabl
 ed","engaged","legendDblclick","versPadding","align","seriesWidths","legendText","maxKeyLength","trimmedKey","substring","nodeTextLength","Error","padding","seriesPerRow","legendWidth","columnWidths","k","reduce","prev","cur","array","xPositions","curX","ceil","xpos","ypos","newxpos","maxwidth","historicalBar","rval","historicalBarChart","bar_model","transitionDuration","tooltipHide","ohlcBarChart","ohlcBar","candlestickBarChart","setBGOpacity","insert","seriesBG","scatter","scatterWrap","strokeWidth","fillOpacity","areaPaths","isArea","area","defined","y1","linePaths","pointSize","pointDomain","lineChart","updateXAxis","updateYAxis","focusLinesWrap","disableTooltip","focusEnable","focusEnter","legendPosition","currentValues","pointYValue","defaultValueFormatter","yPos","x2Axis","y2Axis","focusHeight","focusShowAxisX","focusShowAxisY","focusMargin","lineWithFocusChart","linePlusBarChart","availableHeight2","x2","availableHeight1","bar","focusBarsWrap","dataBars","allDisabled","dataL
 ines","dataLine","switchYAxisOrder","y1Axis","y2","barsOpacity","linesOpacity","y1Opacity","y2Opacity","margin2","y3","lines2","bars2","y4","series1","series2","contextEnter","legendXPosition","originalKey","legendRightAxisHint","legendLeftAxisHint","bars2Wrap","lines2Wrap","y3Axis","y4Axis","getBarsAxis","main","getLinesAxis","multiBar","nonStackableCount","hideable","stacked","parsed","stack","stackOffset","nonStackable","nonStackableSeries","posBase","negBase","groupSpacing","exitTransition","last_datalength","barColor","rgb","darker","barSelection","multiBarChart","multibar","controlWidth","controlLabels","grouped","getTranslate","staggerUp","staggerDown","totalInBetweenTicks","reduceXTicks","multiBarHorizontal","valuePadding","getYerr","xerr","mid","path","yerr","showBarLabels","yErr","multiBarHorizontalChart","multiChart","mouseover_line","yaxis","yAxis2","yAxis1","mouseover_scatter","mouseover_stack","stack1","mouseover_bar","bars1","serieIndex","dataLines1","dataLines2","dat
 aScatters1","dataScatters2","dataBars1","dataBars2","dataStack1","dataStack2","color_array","lines1","scatters1","scatters2","stack2","lines1Wrap","scatters1Wrap","bars1Wrap","stack1Wrap","scatters2Wrap","stack2Wrap","extraValue1","aVal","extraValue2","yScale1","yDomain1","yScale2","yDomain2","stackedArea","parallelCoordinates","enabledDimensions","displayMissingValuesline","axisWithUndefinedValues","newscale","missingValuesline","missingValueslineText","restoreBrush","visible","filters","brushDomain","dimension","hasOnlyNaN","oldDomainMaxValue","hasNaN","dimensions","updateTicks","brushstart","displayBrush","actives","dimensionNames","extents","foreground","isActive","brushend","hasActiveBrush","brushEnd","currentTicks","tickValues","dragging","__origin__","background","dimensionPosition","currentPosition","dimensionsOrder","newData","val","dataValues","dimensionData","rangePoints","onlyUndefinedValues","tension","lineTension","axisDrag","lineData","undefinedValuesLabel","dimension
 sEnter","formerActive","activeChanged","dimensionFormats","parallelCoordinatesChart","originalPosition","parallelCoordinatesWrap","isSorted","nanValue","str","tp","dim","dd","pie","arcTween","endAngle","startAngle","donut","innerRadius","_current","arcs","arcsRadiusOuter","arcsRadiusInner","arcsRadius","outer","inner","donutRatio","growOnHover","g_pie","arcsOver","arc","outerRadius","arcOver","cornerRadius","padAngle","titleOffset","slices","pieLabels","ae","attrTween","showLabels","labelsArc","labelsOutside","group","labelSunbeamLayout","rotateAngle","centroid","labelLocationHash","avgHeight","avgWidth","createHashKey","coordinates","getSlicePercentage","center","labelThreshold","hashKey","labelType","pieLabelsOutside","donutLabelsOutside","labelFormat","pieChart","pieWrap","showTooltipPercent","sankey","computeNodeLinks","sourceLinks","targetLinks","computeNodeValues","sum","computeNodeBreadths","nextNodes","remainingNodes","nodeWidth","sinksRight","moveSinksRight","scaleNodeBread
 ths","kx","computeNodeDepths","iterations","initializeNodeDepth","ky","nodesByBreadth","nodePadding","relaxLeftToRight","weightedSource","sy","breadth","relaxRightToLeft","weightedTarget","ty","resolveCollisions","ascendingDepth","nest","sortKeys","ascending","entries","computeLinkDepths","ascendingSourceDepth","ascendingTargetDepth","relayout","xi","interpolateNumber","curvature","x3","linkPath","sankeyChart","dragmove","testData","isDataValid","dataAvailable","error","showError","linkTitle","origin","appendChild","nodeFillColor","nodeStrokeColor","nodeTitle","units","formatNumber","message","nodeStyle","fillColor","strokeColor","getCache","_cache","delCache","getDiffs","cache","diffs","hasOwnProperty","updateInteractiveLayer","needsUpdate","vertices","groupIndex","pX","pY","pointArray","bounds","geom","polygon","voronoi","clip","pointPaths","vPointPaths","showVoronoi","clipVoronoi","pointClips","clipRadius","mouseEventCallback","el","mDispatch","scrollTop","pageYOffset","scrollLef
 t","pageXOffset","relativePos","logScale","sizeDomain","getSize","padDataOuter","z","forceSize","sizeRange","_sizeRange_def","z0","scaleDiff","width0","height0","sizeDiff","pointBorderColor","points","getShape","titles","interactiveUpdateDelay","clearTimeout","timeoutID","shape","notActive","pointScale","pointRange","forcePoint","pointShape","scatterChart","showDistX","distX","showDistY","distY","regWrap","regLine","intercept","slope","sparkline","paths","result","yValues","maxPoint","lastIndexOf","minPoint","currentPoint","showMinMaxPoints","showCurrentPoint","animate","sparklinePlus","updateValueLine","paused","hoverValue","hoverEnter","sparklineHover","getClosestIndex","closestIndex","sparklineWrap","showLastValue","valueWrap","rightAlignValue","alignValue","dataRaw","aseries","dataFiltered","order","out","zeroArea","areaMouseover","pageX","pageY","areaMouseout","areaClick","d3_stackedOffset_stackPercent","stackData","o","stackedAreaChart","currentFormat","oldYTickFormat","percen
 tFormatter","stackedWrap","xAxisHeight","legendTop","metaKey","stream","stack_percent","controlOptions","requiredTop","valueSum","allNullValues","tooltipValue","showTotalInTooltip","stackedY0","stackedY","totalLabel","oldValueFormatter","stackedAreaWithFocusChart","sunburst","rotationToAvoidUpsideDown","centerAngle","computeCenterAngle","computeNodePercentage","labelThresholdMatched","arcTweenZoom","xd","yd","yr","arcTweenUpdate","ipo","dx0","dy0","updatePrevPosition","prevPositions","pP","storeRetrievePrevPositions","zoomClick","depth","arcText","getBBox","rotation","partition","modes","mode","cG","cGE","groupColorByParent","children","parent","count","d1","d2","sunburstChart"],"mappings":";;AAAA,YAEA,GAAIA,KAGJA,GAAGC,KAAM,EACTD,EAAGE,QAAUF,EAAGE,YAChBF,EAAGG,MAAQH,EAAGG,UACdH,EAAGI,OAASJ,EAAGI,WACfJ,EAAGK,UACHL,EAAGM,QACHN,EAAGO,OAGoB,mBAAb,SAAgD,mBAAd,UAA2C,mBAAR,MAC3EC,GAAKC,QAAO,OAGhBT,EAAGU,SAAWF,GAAGE,SAAQ,eAAiB,cAOrCC,SAASC,UAAUC,OACpBF,SAASC,UAAUC,KAAO,SAAUC,GAChC,GAAoB,kB
 AATC,MAEP,KAAM,IAAIC,WAAS,uEAGvB,IAAIC,GAAQC,MAAMN,UAAUO,MAAMC,KAAKC,UAAW,GAC9CC,EAAUP,KACVQ,EAAO,aACPC,EAAS,WACL,MAAOF,GAAQG,MAAMV,eAAgBQ,IAAQT,EACnCC,KACAD,EACNG,EAAMS,OAAOR,MAAMN,UAAUO,MAAMC,KAAKC,aAKpD,OAFAE,GAAKX,UAAYG,KAAKH,UACtBY,EAAOZ,UAAY,GAAIW,GAChBC,IAKXxB,EAAGC,MACHD,EAAGU,SAASiB,GAAE,eAAiB,SAASC,GACpC5B,EAAGM,KAAKuB,WAAa,GAAIC,QAG7B9B,EAAGU,SAASiB,GAAE,aAAe,SAASC,GAClC5B,EAAGM,KAAKyB,SAAW,GAAID,MACvB9B,EAAGM,KAAK0B,UAAYhC,EAAGM,KAAKyB,QAAU/B,EAAGM,KAAKuB,UAC9C7B,EAAGiC,IAAG,QAAUjC,EAAGM,KAAK0B,cAQhChC,EAAGiC,IAAM,WACL,GAAIjC,EAAGC,KAAOiC,OAAOC,SAAWA,QAAQF,KAAOE,QAAQF,IAAIR,MACvDU,QAAQF,IAAIR,MAAMU,QAASd,eAC1B,IAAIrB,EAAGC,KAAOiC,OAAOC,SAAiC,kBAAfA,SAAQF,KAAqBtB,SAASC,UAAUC,KAAM,CAC9F,GAAIoB,GAAMtB,SAASC,UAAUC,KAAKO,KAAKe,QAAQF,IAAKE,QACpDF,GAAIR,MAAMU,QAASd,WAEvB,MAAOA,WAAUA,UAAUe,OAAS,IAIxCpC,EAAGqC,WAAa,SAASC,EAAMC,GACvBJ,SAAWA,QAAQK,MACnBL,QAAQK,KAAI,kBAAqBF,EAAO,0BAA2BC,GAAQ,KAOnFvC,EAAGyC,OAAS,SAAgBC,GAExBA,EAAOA,GAAQ,EAEf1C,EAAGyC,OAAOE,QAAS,EACnB3C,EAAGU,SAASkC,c
 AEZ,IAAIC,GAAa,WAGb,IAAK,GAFDC,GAAOC,EAEFC,EAAI,EAAON,EAAJM,IAAaD,EAAQ/C,EAAGyC,OAAOQ,MAAMD,IAAKA,IACtDF,EAAQC,EAAMG,iBACHH,GAAMI,gBAAkB,WAAYJ,EAAMI,SAASL,EAGlE9C,GAAGyC,OAAOQ,MAAMG,OAAO,EAAGJ,GAEtBhD,EAAGyC,OAAOQ,MAAMb,OAChBiB,WAAWR,IAGX7C,EAAGU,SAAS4C,aACZtD,EAAGyC,OAAOE,QAAS,GAI3BU,YAAWR,IAGf7C,EAAGyC,OAAOE,QAAS,EACnB3C,EAAGyC,OAAOQ,SAmBVjD,EAAGuD,SAAW,SAASC,SACRnC,WAAU,UAAa,YAC9BmC,GAAON,SAAU7B,UAAU,GAAI8B,SAAU9B,UAAU,KAGvDrB,EAAGyC,OAAOQ,MAAMQ,KAAKD,GAEhBxD,EAAGyC,OAAOE,QACX3C,EAAGyC,UAKY,mBAAb,SAAgD,mBAAd,WAC1CiB,OAAOC,QAAU3D,GAGI,mBAAb,UACRkC,OAAOlC,GAAKA,GClJdA,EAAGO,IAAIqD,MAAQ,SAAST,GACvB,MAAuBU,UAAnB3B,OAAO4B,QACHA,QAAQC,OAAOZ,GAEhBA,KASRnD,EAAGO,IAAIyD,KAAO,SAASb,GACtB,MAAuBU,UAAnB3B,OAAO4B,QACHA,QAAQG,QAAQd,GAEjBA,KCfRnD,EAAGkE,qBAAuB,WACtB,YAkBA,SAASC,GAAMC,GACXA,EAAUC,KAAK,SAASC,GAapB,QAASC,KACL,GAAIC,GAAUhE,GAAGiE,MAAM1D,MACnB2D,EAASF,EAAQ,GACjBG,EAASH,EAAQ,GACjBI,GAAiB,EACjBC,GAAoB,CAuCxB,IAtCIC,IAQAJ,EAASlE,GAAGuE,MAAMC,QAClBL,EAASnE,GAAGuE,MAAME,QAWa,QAA5BzE,GAAGu
 E,MAAMG,OAAOC,UACfP,GAAiB,GAGjBpE,GAAGuE,MAAMG,OAAOE,UAAUC,QAAQC,MAAK,eACvCT,GAAoB,IAKzBD,IACCF,GAAUa,EAAOC,KACjBb,GAAUY,EAAOE,KAMC,aAAlBjF,GAAGuE,MAAMW,MACG,EAAThB,GAAuB,EAATC,GACdD,EAASiB,GAAkBhB,EAASiB,GACnCpF,GAAGuE,MAAMc,eAA4DhC,SAA3CrD,GAAGuE,MAAMc,cAAcC,iBAClDjB,EACD,CAEF,GAAIC,GACItE,GAAGuE,MAAMc,eACqChC,SAA3CrD,GAAGuE,MAAMc,cAAcC,kBACejC,SAArCrD,GAAGuE,MAAMc,cAAcT,WACpB5E,GAAGuE,MAAMc,cAAcT,UAAUE,MAAMpF,EAAQ6F,uBAEtD,MASR,OANArF,GAASsF,iBACLtB,OAAQA,EACRC,OAAQA,IAEZR,EAAM8B,gBAAgB,UACtB/F,GAAQgG,QAAO,GAGfhG,EAAQgG,QAAO,EAInB,IAAIC,GAA8C,kBAAtBC,GAAOC,WAC/BC,EAAczC,MAGlB,IAAIsC,EAAgB,CAChB,GAAII,GAAe/F,GAAGgG,OAAOJ,EAAOK,QAAS/B,GAAU,CAEvD,MAAI0B,EAAOK,QAAQF,GAAgBH,EAAOM,aAAehC,GAUrD,MANAhE,GAASsF,iBACLtB,OAAQA,EACRC,OAAQA,IAEZR,EAAM8B,gBAAgB,UACtB/F,GAAQgG,QAAO,EARfI,GAAcF,EAAOO,SAASnG,GAAGgG,OAAOJ,EAAOK,QAAS/B,GAAU,OAatE4B,GAAcF,EAAOQ,OAAOlC,EAGhChE,GAASmG,kBACLnC,OAAQA,EACRC,OAAQA,EACR2B,YAAaA,IAIK,aAAlB9F,GAAGuE,MAAMW,MACThF,EAASoG,iBACLpC,OAAQA,EACRC,OAAQA,EACR2B,YAAaA,I
 AKC,UAAlB9F,GAAGuE,MAAMW,MACThF,EAASqG,cACLrC,OAAQA,EACRC,OAAQA,EACR2B,YAAaA,IAKC,cAAlB9F,GAAGuE,MAAMW,MACZhF,EAASsG,kBACRtC,OAAQA,EACRC,OAAQA,EACR2B,YAAaA,IAKO,YAAlB9F,GAAGuE,MAAMW,MACZhF,EAASuG,gBACRvC,OAAQA,EACRC,OAAQA,EACR2B,YAAaA,IAlJnB,GAAIY,GAAY1G,GAAG2G,OAAOpG,MACtB4E,EAAkByB,GAAS,IAAMxB,EAAmByB,GAAU,IAC9DC,EAAOJ,EAAUK,UAAS,qCACzBjD,MAAMA,IACPkD,EAAYF,EAAKG,QAChBC,OAAM,KAAMC,KAAI,QAAU,mCAC/BH,GAAUE,OAAM,KAAMC,KAAI,QAAO,2BAE5BC,IA+ILA,EACKjG,GAAE,YAAa4C,GACf5C,GAAE,YAAa4C,GAAc,GAC7B5C,GAAE,WAAa4C,GAAa,GAC5B5C,GAAE,YAAc4C,GAAa,GAC7B5C,GAAE,UAAY4C,GAAa,GAC3B5C,GAAE,WAAa4C,GACf5C,GAAE,QAAU4C,GAGjBJ,EAAM0D,UAAY,KAElB1D,EAAM8B,gBAAkB,SAAS6B,GACxBC,IACD5D,EAAM0D,WAAa1D,EAAM0D,UAAUF,KAAI,QAAWG,GACtD9H,EAAGO,IAAIqD,MAAM,WACT,GAAIoE,GAAOV,EAAKH,OAAM,4BACjBI,UAAS,QACTjD,KAAW,MAALwD,GAAc9H,EAAGG,MAAM8H,UAAUH,OAAUI,OACtDF,GAAKP,QACAC,OAAM,QACNC,KAAI,QAAU,gBACdA,KAAI,KAAO,SAASQ,GAAK,MAAOA,KAChCR,KAAI,KAAO,SAASQ,GAAK,MAAOA,KAChCR,KAAI,KAAO/B,GACX+B,KAAI,KAAM,GACfK,EAAKI,OAAOC,gBAnM5B,GAAI9
 C,IAAWC,KAAM,EAAGC,IAAK,GACrB2B,EAAQ,KACRC,EAAS,KACTjB,EAAS5F,GAAG8H,MAAMC,SAClB7H,EAAWF,GAAGE,SAAQ,mBAAqB,kBAAmB,eAAgB,kBAAmB,mBAAoB,kBACrHqH,GAAgB,EAChBH,EAAe,KACf1H,EAAUF,EAAGI,OAAOF,UACpB4E,EAAU5C,OAAOsG,aAyOzB,OAtOAtI,GACKuI,SAAS,GACTC,UAAU,GACVxC,QAAO,GA2LZ/B,EAAMzD,SAAWA,EACjByD,EAAMjE,QAAUA,EAEhBiE,EAAMoB,OAAS,SAASoD,GACpB,MAAKtH,WAAUe,QACfmD,EAAOE,IAA4B,mBAAZkD,GAAElD,IAAwBkD,EAAElD,IAASF,EAAOE,IACnEF,EAAOC,KAA4B,mBAAZmD,GAAEnD,KAAwBmD,EAAEnD,KAASD,EAAOC,KAC5DrB,GAHuBoB,GAMlCpB,EAAMiD,MAAQ,SAASuB,GACnB,MAAKtH,WAAUe,QACfgF,EAAQuB,EACDxE,GAFuBiD,GAKlCjD,EAAMkD,OAAS,SAASsB,GACpB,MAAKtH,WAAUe,QACfiF,EAASsB,EACFxE,GAFuBkD,GAKlClD,EAAMiC,OAAS,SAASuC,GACpB,MAAKtH,WAAUe,QACfgE,EAASuC,EACFxE,GAFuBiC,GAKlCjC,EAAM4D,cAAgB,SAASY,GAC3B,MAAKtH,WAAUe,QACf2F,EAAgBY,EACTxE,GAFuB4D,GAKlC5D,EAAMyD,aAAe,SAASe,GAC1B,MAAKtH,WAAUe,QACfwF,EAAee,EACRxE,GAFuByD,GAK3BzD,GAgBXnE,EAAG4I,kBAAoB,SAAUC,EAAQC,EAAWC,GAChD,YACA,MAAOF,YAAkB3H,QACrB,MAAO,KAEX,IAAI8H,EAEAA,GADqB,kBAAdD,GACM,SAASZ,GAClB,MAAOA,GA
 AEL,GAGAiB,CAEjB,IAAIE,GAAO,SAASd,EAAGe,GAUnB,MAAOF,GAAWb,GAAKe,GAGvB1C,EAAShG,GAAG2I,SAASF,GAAMzD,KAC3B4D,EAAQ5I,GAAG6I,KAAK,EAAG7C,EAAOqC,EAAOC,GAAa,IAC9CQ,EAAeN,EAAWH,EAAOO,GAMrC,IAJ4B,mBAAjBE,KACPA,EAAeF,GAGfE,IAAiBR,EACjB,MAAOM,EAGX,IAAIG,GAAY/I,GAAGgJ,KAAKJ,EAAM,EAAGP,EAAOzG,OAAS,IAC7CqH,EAAYT,EAAWH,EAAOU,GAMlC,OAJyB,mBAAdE,KACPA,EAAYF,GAGZG,KAAKC,IAAIF,EAAYX,IAAcY,KAAKC,IAAIL,EAAeR,GACpDM,EAEAG,GASfvJ,EAAG4J,kBAAoB,SAAUf,EAAQC,EAAWe,GAChD,YACA,IAAIC,GAAWC,EAAAA,EAAUC,EAAmB,IAQ5C,OAPAnB,GAAOoB,QAAQ,SAAS9B,EAAEnF,GACtB,GAAIkH,GAAQR,KAAKC,IAAIb,EAAYX,EACvB,OAALA,GAAsB2B,GAATI,GAA6BL,EAARK,IACnCJ,EAAWI,EACXF,EAAmBhH,KAGpBgH,GCvUXhK,EAAGI,OAAOF,QAAU,WAChB,YAyPA,SAASiK,KACL,IAAKjK,IAAYA,EAAQkK,OAAQ,CAG7B,GAAI9F,IAAQ,EACZpE,GAAUM,GAAG2G,OAAOkD,SAASC,MAAMnD,OAAM,IAAKoD,GAAIjG,KAAKA,GAEvDpE,EAAQuH,QAAQC,OAAM,OACdC,KAAI,QAAU,cAAgB6C,EAAUA,EAAU,eAClD7C,KAAI,KAAO4C,GACXE,MAAK,MAAQ,GAAGA,MAAK,OAAS,GAC9BA,MAAK,UAAY,GACjBA,MAAK,WAAa,SAClBlD,UAAS,sBAAuBmD,QAAQ3E,GAAsB,GAC9D2E,QAAQ3E,GAAsB,GA
 EtC7F,EAAQkI,OAAOC,UAKvB,QAASsC,KACL,MAAKC,IACAC,EAAiBvG,IAEtBtE,EAAGO,IAAIqD,MAAM,WACTuG,GAIA,IAAIW,GAAaC,EAAiBzG,EAC9BwG,KACA5K,EAAQkK,OAAOY,UAAYF,GAG/BG,MAGGN,GAhBP,OAjQJ,GAAIJ,GAAK,aAAeb,KAAKwB,MAAsB,IAAhBxB,KAAKyB,UAChC7G,EAAO,KACP8G,EAAU,IACVC,EAAW,GACXC,EAAe,EACfd,EAAU,KACVtE,GAAS,EACTwC,EAAY,IACZxI,EAAU,KACVqL,GAAiB/F,KAAM,KAAMC,IAAK,MAClCmF,GAAU,EACVnC,EAAW,IACX+C,GAAgB,EAChBzF,EAAuB,yBAI3B0F,EAAiB,SAAStD,EAAGnF,GAC7B,MAAOmF,IAIPuD,EAAkB,SAASvD,GAC3B,MAAOA,IAGPwD,EAAe,SAASxD,EAAGnF,GAC3B,MAAOmF,IAKP4C,EAAmB,SAAS5C,GAC5B,GAAU,OAANA,EACA,MAAO,EAGX,IAAIyD,GAAQpL,GAAG2G,OAAOkD,SAASwB,cAAa,SAC5C,IAAIL,EAAe,CACf,GAAIM,GAAaF,EAAMrE,UAAS,SAC3BjD,MAAM6D,IACNV,QAAQC,OAAM,QAEnBoE,GAAWpE,OAAM,MACZA,OAAM,MACNC,KAAI,UAAY,GAChBD,OAAM,UACNgD,QAAO,WAAY,GACnBqB,KAAKL,EAAgBvD,EAAE6D,QAGhC,GAAIC,GAAaL,EAAMrE,UAAS,SAC3BjD,MAAM6D,IACNV,QAAQC,OAAM,SAEfwE,EAAYD,EAAW1E,UAAS,MAC3BjD,KAAK,SAAS6H,GAAK,MAAOA,GAAEC,SAC5B3E,QACAC,OAAM,MACNgD,QAAO,YAAc,SAASyB,GAAK,MAAOA,GAAEE,WAErDH,GAAUxE,OAAM,MACXgD,QAA
 O,sBAAsB,GAC7BhD,OAAM,OACN+C,MAAK,mBAAqB,SAAS0B,GAAK,MAAOA,GAAEG,QAEtDJ,EAAUxE,OAAM,MACXgD,QAAO,OAAO,GACdA,QAAO,QAAS,SAASyB,GAAK,QAASA,EAAEI,QACzCR,KAAK,SAASI,EAAGnJ,GAAK,MAAO2I,GAAaQ,EAAEK,IAAKxJ,KAEtDkJ,EAAUxE,OAAM,MACXgD,QAAO,SAAS,GAChBqB,KAAK,SAASI,EAAGnJ,GAAK,MAAOyI,GAAeU,EAAEH,MAAOhJ,KAE1DkJ,EAAUO,OAAO,SAAUN,EAAEnJ,GAAK,MAAqBa,UAAdsI,EAAEO,UAAyBhF,OAAM,MACrEgD,QAAO,WAAY,GACnBqB,KAAK,SAASI,EAAGnJ,GAAK,MAAO,IAAMxC,GAAGmM,OAAM,KAAMR,EAAEO,SAAW,MAEpER,EAAU3E,UAAS,MAAOlD,KAAK,SAAS8H,GACpC,GAAIA,EAAEE,UAAW,CACb,GAAIO,GAAepM,GAAG8H,MAAMC,SAAS5B,QAAQ,EAAE,IAAIF,OAAK,OAAS0F,EAAEG,QAC/DO,EAAU,EACdrM,IAAG2G,OAAOpG,MACL0J,MAAK,sBAAwBmC,EAAaC,IAC1CpC,MAAK,mBAAqBmC,EAAaC,MAKpD,IAAId,GAAOH,EAAMxB,OAAO0C,SAGxB,OAFiBjJ,UAAbsE,EAAE4E,SACFhB,GAAQ,uBAAyB5D,EAAE4E,OAAS,UACzChB,GAYPiB,EAAW,WACX,GAAIC,IACAzH,KAAmB,OAAbhF,GAAGuE,MAAiBvE,GAAGuE,MAAMmI,QAAU,EAC7CzH,IAAkB,OAAbjF,GAAGuE,MAAiBvE,GAAGuE,MAAMoI,QAAU,EAGhD,IAAgD,QAA7CC,iBAAiB/C,SAASC,MAAM+C,UAAqB,CAGpD,GAAIC,GAASjD,SAASC,KAAKiD,uBAC3BN,GAAIz
 H,MAAQ8H,EAAO9H,KACnByH,EAAIxH,KAAO6H,EAAO7H,IAGtB,MAAOwH,IAGPpC,EAAmB,SAAS1C,GAC5B,GAAIA,GAAKA,EAAEiE,OAAQ,CACf,GAAIpM,EAAGG,MAAMqN,QAAQrF,EAAEiE,QACnB,OAAO,CAGX,IAAIpM,EAAGG,MAAMsN,SAAStF,EAAEiE,QAEpB,MADAjE,GAAEiE,QAAUjE,EAAEiE,SACP,EAGf,OAAO,GAKPsB,EAAoB,SAAST,GAC7B,GAIIzH,GAAMC,EAAKkI,EAJXtG,EAASnH,EAAQkK,OAAOwD,aACxBxG,EAAQlH,EAAQkK,OAAOyD,YACvBC,EAAczD,SAAS0D,gBAAgBD,YACvCE,EAAe3D,SAAS0D,gBAAgBC,YAI5C,QAAQ5C,GACJ,IAAK,IACD5F,GAAS4B,EAAQiE,EACjB5F,IAAS4B,EAAS,GACf4F,EAAIzH,KAAOA,EAAO,IAAGA,EAAO6F,IAC3BsC,EAAMV,EAAIxH,IAAMA,GAAO,IAAGA,GAAOkI,IACjCA,EAAMV,EAAIxH,IAAMA,EAAM4B,GAAU2G,IAAcvI,GAAOkI,EAAMK,EAC/D,MACJ,KAAK,IACDxI,EAAO6F,EACP5F,IAAS4B,EAAS,GACd4F,EAAIzH,KAAOA,EAAO4B,EAAQ0G,IAAatI,GAAS4B,EAAQiE,IACvDsC,EAAMV,EAAIxH,IAAMA,GAAO,IAAGA,GAAOkI,IACjCA,EAAMV,EAAIxH,IAAMA,EAAM4B,GAAU2G,IAAcvI,GAAOkI,EAAMK,EAChE,MACJ,KAAK,IACDxI,IAAU4B,EAAQ,GAAK,EACvB3B,EAAM4F,EACF4B,EAAIxH,IAAMA,EAAM4B,EAAS2G,IAAcvI,GAAQ4B,EAASgE,IACvDsC,EAAMV,EAAIzH,KAAOA,GAAQ,IAAGA,GAAQmI,IACpCA,EAAMV,EAAIzH,
 KAAOA,EAAO4B,GAAS0G,IAAatI,GAAQmI,EAAMG,EACjE,MACJ,KAAK,IACDtI,IAAU4B,EAAQ,GAClB3B,GAAQ4B,EAASgE,EACb4B,EAAIxH,IAAMA,EAAM,IAAGA,EAAM4F,IACxBsC,EAAMV,EAAIzH,KAAOA,GAAQ,IAAGA,GAAQmI,IACpCA,EAAMV,EAAIzH,KAAOA,EAAO4B,GAAS0G,IAAatI,GAAQmI,EAAMG,EACjE,MACJ,KAAK,SACDtI,IAAU4B,EAAQ,GAClB3B,IAAS4B,EAAS,EAClB,MACJ,SACI7B,EAAO,EACPC,EAAM,EAId,OAASD,KAAQA,EAAMC,IAAOA,IAM9BwF,EAAkB,WAClBjL,EAAGO,IAAIyD,KAAK,WACR,GAAIiJ,GAAMD,IACNiB,EAAgBP,EAAkBT,GAClCzH,EAAOyH,EAAIzH,KAAOyI,EAAczI,KAChCC,EAAMwH,EAAIxH,IAAMwI,EAAcxI,GAGlC,IAAIS,EACAhG,EACKgO,YACAC,aACAC,MAAM1F,GACND,SAAS,GACTgC,MAAK,UAAY,OACnB,CAEH,GAAI4D,GAAgB,aAAe9C,EAAa/F,KAAO,OAAS+F,EAAa9F,IAAM,MAC/E6I,EAAgB,aAAe5E,KAAK6E,MAAM/I,GAAQ,OAASkE,KAAK6E,MAAM9I,GAAO,MAC7E+I,EAAwBhO,GAAGiO,kBAAkBJ,EAAeC,GAC5DI,EAAYxO,EAAQuK,MAAK,WAAc,EAE3CvK,GACKgO,YACAC,aACA1F,SAASiG,EAAY,EAAIjG,GAEzBkG,WAAU,YAAc,SAAUxG,GAC/B,MAAOqG,IACR,aAEFG,WAAU,oBAAsB,SAAUxG,GACvC,MAAOqG,KAEV/D,MAAK,gBAAkB6D,GACvB7D,MAAK,UAAY,GAG1Bc,EAAa/F,KAAOA,EACpB+F,EAAa9F,IAAMA,IAuG3B,OAzDA
 kF,GAAU5E,qBAAuBA,EACjC4E,EAAUiE,QAAU5O,EAAGG,MAAM0O,YAAYhO,KAAK8J,GAE9CA,EAAUmE,SAAWC,OAAOC,WAExBvG,UAAWwG,IAAK,WAAW,MAAOxG,IAAYyG,IAAK,SAASvG,GAAGF,EAASE,IACxEyC,SAAU6D,IAAK,WAAW,MAAO7D,IAAW8D,IAAK,SAASvG,GAAGyC,EAAQzC,IACrE0C,UAAW4D,IAAK,WAAW,MAAO5D,IAAY6D,IAAK,SAASvG,GAAG0C,EAAS1C,IACxE2C,cAAe2D,IAAK,WAAW,MAAO3D,IAAgB4D,IAAK,SAASvG,GAAG2C,EAAa3C,IACpF6B,SAAUyE,IAAK,WAAW,MAAOzE,IAAW0E,IAAK,SAASvG,GAAG6B,EAAQ7B,IACrEiC,SAAUqE,IAAK,WAAW,MAAOrE,IAAWsE,IAAK,SAASvG,GAAGiC,EAAQjC,IACrED,WAAYuG,IAAK,WAAW,MAAOvG,IAAawG,IAAK,SAASvG,GAAGD,EAAUC,IAC3EoC,kBAAmBkE,IAAK,WAAW,MAAOlE,IAAoBmE,IAAK,SAASvG,GAAGoC,EAAiBpC,IAChG8C,gBAAiBwD,IAAK,WAAW,MAAOxD,IAAkByD,IAAK,SAASvG,GAAG8C,EAAe9C,IAC1F+C,iBAAkBuD,IAAK,WAAW,MAAOvD,IAAmBwD,IAAK,SAASvG,GAAG+C,EAAgB/C,IAC7FgD,cAAesD,IAAK,WAAW,MAAOtD,IAAgBuD,IAAK,SAASvG,GAAGgD,EAAahD,IACpF6C,eAAgByD,IAAK,WAAW,MAAOzD,IAAiB0D,IAAK,SAASvG,GAAG6C,EAAc7C,IACvFqE,UAAWiC,IAAK,WAAW,MAAOjC,IAAYkC,IAAK,SAASvG,GAAGqE,EAASrE,IAGxEwG,gBAAiBF,IAAK,WAAW,MAAO5E,UAASC,MAAQ4E,IAA
 K,SAASvG,GAEnE3I,EAAGqC,WAAU,iBAAmB,iCAEpC+M,UAAWH,IAAK,WAAW,MAAO,OAAQC,IAAK,SAASvG,GAEpD3I,EAAGqC,WAAU,WAAa,iCAE9BgN,QAASJ,IAAK,WAAW,OAAQzJ,KAAM,EAAGC,IAAK,IAAMyJ,IAAK,SAASvG,GAE/D3I,EAAGqC,WAAU,SAAW,0CAI5B6D,QAAS+I,IAAK,WAAW,MAAO/I,IAAUgJ,IAAK,SAASvG,GAChDzC,GAAUyC,IACVzC,IAAWyC,EACXgC,OAGRrG,MAAO2K,IAAK,WAAW,MAAO3K,IAAQ4K,IAAK,SAASvG,GAE5CA,EAAE2G,QACF3G,EAAEqD,MAAQrD,EAAE2G,MAAMxH,EAClBa,EAAEyD,OAASzD,EAAEyD,WACbzD,EAAEyD,OAAOJ,MAAQrD,EAAE2G,MAAMC,EACzB5G,EAAEyD,OAAOE,MAAQ3D,EAAE2G,MAAMhD,OAAS3D,EAAEyD,OAAOE,OAE/ChI,EAAOqE,IAIXyB,MAAO6E,IAAK,WAAW,MAAO/O,GAAQkK,QAAU8E,IAAK,SAASvG,MAC9D4B,IAAK0E,IAAK,WAAW,MAAO1E,IAAM2E,IAAK,SAASvG,QAGpD3I,EAAGG,MAAMqP,YAAY7E,GACdA,GC7VX3K,EAAGG,MAAMsP,WAAa,WAElB,GAAIC,IAAQtI,MAAO,IAAKC,OAAQ,IAGhC,OAAInF,QAAOyN,YAAczN,OAAO0N,aAC5BF,EAAKtI,MAAQlF,OAAOyN,WACpBD,EAAKrI,OAASnF,OAAO0N,YACd,GAIY,cAAnBvF,SAASwF,YACTxF,SAAS0D,iBACT1D,SAAS0D,gBAAgBF,aAEzB6B,EAAKtI,MAAQiD,SAAS0D,gBAAgBF,YACtC6B,EAAKrI,OAASgD,SAAS0D,gBAAgBH,aAChC,GAIPvD,SAASC,MAAQD,SAASC,KAAK
 uD,aAC/B6B,EAAKtI,MAAQiD,SAASC,KAAKuD,YAC3B6B,EAAKrI,OAASgD,SAASC,KAAKsD,aACrB,GAGJ,GAOX5N,EAAGG,MAAMqN,QAAUtM,MAAMsM,QACzBxN,EAAGG,MAAMsN,SAAW,SAASqC,GACzB,MAAa,QAANA,GAA2B,gBAANA,IAEhC9P,EAAGG,MAAM4P,WAAa,SAASD,GAC3B,MAAoB,kBAANA,IAElB9P,EAAGG,MAAM6P,OAAS,SAASF,GACvB,MAA4B,kBAArBG,SAAS7O,KAAK0O,IAEzB9P,EAAGG,MAAM+P,SAAW,SAASJ,GACzB,OAAQK,MAAML,IAAmB,gBAANA,IAO/B9P,EAAGG,MAAMiQ,aAAe,SAASC,GAO7B,MANInO,QAAOoO,iBACPpO,OAAOoO,iBAAgB,SAAWD,GAElCrQ,EAAGiC,IAAG,gDAAkDoO,IAIxDlN,SAAUkN,EACVE,MAAO,WACHrO,OAAOsO,oBAAmB,SAAWH,MAYjDrQ,EAAGG,MAAMsQ,SAAW,SAASnE,GAEzB,GAAczI,SAAVyI,EACA,MAAOtM,GAAGG,MAAMuQ,cAGb,IAAG1Q,EAAGG,MAAMqN,QAAQlB,GAAQ,CAC/B,GAAIqE,GAAcnQ,GAAG8H,MAAMsI,UAAUnK,MAAM6F,EAC3C,OAAO,UAASnE,EAAGnF,GACf,GAAIwJ,GAAY3I,SAANb,EAAkBmF,EAAInF,CAChC,OAAOmF,GAAEmE,OAASqE,EAAYnE,IAOlC,MAAOF,IASftM,EAAGG,MAAMuQ,aAAe,WAEpB,MAAO1Q,GAAGG,MAAMsQ,SAASjQ,GAAG8H,MAAMuI,aAAapK,UAQnDzG,EAAGG,MAAM2Q,YAAc,SAASC,EAAYC,EAAQC,GAEhDD,EAASA,GAAU,SAAS5E,GAAU,MAAOA,GAAOI,KACpDyE,EAAgBA,GAAiBzQ,GAAG8H,MAAMu
 I,aAAapK,OAGvD,IAAIyK,GAAWD,EAAc7O,MAE7B,OAAO,UAASgK,EAAQhD,GACpB,GAAIoD,GAAMwE,EAAO5E,EACjB,OAAIpM,GAAGG,MAAM4P,WAAWgB,EAAWvE,IACxBuE,EAAWvE,KACS3I,SAApBkN,EAAWvE,GACXuE,EAAWvE,IAGb0E,IAEDA,EAAWD,EAAc7O,QAE7B8O,GAAsB,EACfD,EAAcC,MAWjClR,EAAGG,MAAMgR,KAAO,SAASC,EAAOC,GAE5B,GAAIC,GAAO,SAASC,GAChB/Q,GAAGuL,KAAKwF,EAAM,SAASC,GACnB,GAAItM,GAAS1E,GAAG2G,OAAOkK,GAASjH,MAChClF,GAAOuM,WAAWC,aACdlR,GAAG2G,OAAOqK,GAAUrK,OAAOkK,GAASjH,OACpClF,GACJlF,EAAGG,MAAMgR,KAAKC,EAAOC,KAI7B7Q,IAAG+G,UAAU6J,GAAOzP,GAAE,QAAU,WAC5BgQ,QAAQC,UAAU7Q,KAAKwQ,KAAMxQ,KAAK8Q,YAAa9Q,KAAKwQ,MACpDD,EAAKvQ,KAAKwQ,MACV/Q,GAAGuE,MAAM+M,mBAGbtR,GAAG2G,OAAOjF,QAAQP,GAAE,WAAa,WACzBnB,GAAGuE,MAAMgN,OACTT,EAAK9Q,GAAGuE,MAAMgN,UAW1B/R,EAAGG,MAAM6R,oBAAsB,SAAUC,GACrC,GAAIjS,EAAGG,MAAM4P,WAAWkC,EAAYxH,QAAUzK,EAAGG,MAAM4P,WAAWkC,EAAYC,MAAO,CACjF,GAAIC,GAAWC,SAASH,EAAYxH,MAAK,aAAc4H,QAAO,KAAI,IAAO,IACrEC,EAAaL,EAAYC,OAAO9P,MACpC,OAAOpC,GAAGG,MAAM8H,UAAUqK,EAAaH,EAAW,IAEtD,MAAO,IAOXnS,EAAGG,MAAM8H,UAAY,SAASsK,GAC1B,OAAKvS,EAAGG,MAA
 M+P,SAASqC,IAChBpC,MAAMoC,IACA,OAANA,GACAA,IAAMxI,EAAAA,GACNwI,MAAOxI,EAAAA,GAEH,EAEJwI,GAMX/R,GAAG4D,UAAUxD,UAAU4R,gBAAkB,SAASC,GAC9C,GAAIC,IAAQ3R,MAAMW,UAAUP,MAAMC,KAAKC,UAAW,GAClD,OAAOoR,GAAYtE,WAAW1M,MAAMgR,EAAaC,IAOrD1S,EAAGG,MAAMsS,YAAc,SAAS/R,EAAU+H,GACtC,KAAM1H,eAAgBf,GAAGG,MAAMsS,aAC3B,MAAO,IAAIzS,GAAGG,MAAMsS,YAAY/R,EAAU+H,EAG9C,IAAIkK,GAAyB9O,SAAb4E,EAAyBA,EAAW,IAChDmK,KACAC,EAAO9R,IAEXA,MAAKX,OAAS,SAASA,GAevB,MAdIA,MAAYe,MAAMC,KAAKC,UAAW,GAClCjB,EAAO6J,QAAQ,SAAS6I,GACpBA,EAAMC,YAAa,EACnB,SAAUC,GACNA,EAAEtS,SAASiB,GAAE,YAAc,SAASsR,GAChCD,EAAED,YAAa,EACfF,EAAKK,UAAS,YAEnBJ,GAECF,EAAYO,QAAQL,GAAS,GAC7BF,EAAYnP,KAAKqP,KAGtB/R,MAGPA,KAAKqS,MAAQ,SAAS3K,GACD5E,SAAb4E,IACAkK,EAAYlK,GAEhBmK,MAGJ7R,KAAKoN,WAAa,SAAS/J,EAAWsO,EAAMjK,GAcxC,GAbAiK,EAAOrR,UAAUe,OAAS,KAAOjB,MAAMC,KAAKC,UAAW,MAGnDoH,EADAiK,EAAKtQ,OAAS,EACHsQ,EAAKW,MAESxP,SAAd8O,EAA0BA,EAAY,IAErDvO,EAAU2O,YAAa,EAEnBH,EAAYO,QAAQ/O,GAAa,GACjCwO,EAAYnP,KAAKW,GAGJ,IAAbqE,EAIA,MAHArE,GAAU2O,YAAa,EACvB3O,EAAUgK,MAAQ,WAAa,MAAOrN
 ,OACtCqD,EAAUqE,SAAW,WAAa,MAAO1H,OAClCqD,CAEkB,KAArBA,EAAUhC,OACVgC,EAAU2O,YAAa,EAChB3O,EAAUkP,MAAO,SAASnL,GAAI,OAAQA,EAAE/F,SAC/CgC,EAAU2O,YAAa,EAEvB3O,EAAU2O,YAAa,CAG3B,IAAIR,GAAI,CACR,OAAOnO,GACF+J,aACA1F,SAASA,GACTpE,KAAK,aAAckO,IACnBlO,KAAI,MAAQ,SAAS8D,EAAGnF,GACT,MAANuP,IACFnO,EAAU2O,YAAa,EACvBF,EAAKK,UAAUzR,MAAMV,KAAM2R,OAM/C3R,KAAKmS,UAAY,WACTN,EAAYU,MAAO,SAASnL,GAAI,MAAOA,GAAE4K,eACzCH,EAAY3I,QAAS,SAAS9B,GAAIA,EAAE4K,YAAa,IACjDrS,EAASwS,UAAUzR,MAAMV,KAAMM,cAY3CrB,EAAGG,MAAMoT,WAAa,SAASC,GAC3B,GAAIC,GAAUpS,UAAUe,OAAS,KAAOjB,MAAMC,KAAKC,UAAW,KAC9DoS,GAAQxJ,QAAQ,SAASyJ,GACrB,IAAK,GAAIlH,KAAOkH,GAAQ,CACpB,GAAIlG,GAAUxN,EAAGG,MAAMqN,QAAQgG,EAAIhH,IAC/BiB,EAAWzN,EAAGG,MAAMsN,SAAS+F,EAAIhH,IACjCmH,EAAS3T,EAAGG,MAAMsN,SAASiG,EAAOlH,GAElCiB,KAAaD,GAAWmG,EACxB3T,EAAGG,MAAMoT,WAAWC,EAAIhH,GAAMkH,EAAOlH,IAErCgH,EAAIhH,GAAOkH,EAAOlH,OAUlCxM,EAAGG,MAAM4R,MAAQ,WACb,KAAMhR,eAAgBf,GAAGG,MAAM4R,OAC3B,MAAO,IAAI/R,GAAGG,MAAM4R,KAExB,IAAIA,MAEA6B,EAAY,aACZC,EAAY,WAAY,UACxBC,EAAO,KACPC,EAAU,IAEd
 hT,MAAKL,SAAWF,GAAGE,SAAQ,SAAW,OAEtCK,KAAKL,SAASiB,GAAE,MAAQ,SAASoQ,GAC7B6B,EAAU7B,GAAO,KAGrBhR,KAAKiT,OAAS,SAASC,GAEnB,MADAJ,GAAYI,EACLlT,MAGXA,KAAKmT,OAAS,SAASD,EAAI9Q,GAUvB,MATKA,KACDA,EAAW,cAEfyQ,EAAY,SAAS7B,EAAOoC,GACxBF,EAAGlC,GACCoC,GACAhR,KAGDpC,MAGXA,KAAK+S,KAAO,SAAS/B,GACjB+B,EAAOA,MACP9T,EAAGG,MAAMoT,WAAWO,EAAM/B,GAG9B,IAAIqC,GAAO,WACP,GAAIC,GAAWR,GAEf,IAAIS,KAAKC,UAAUF,KAAcC,KAAKC,UAAUxC,GAC5C,OAAO,CAGX,KAAK,GAAIvF,KAAO6H,GACOxQ,SAAfkO,EAAMvF,KACNuF,EAAMvF,OAEVuF,EAAMvF,GAAO6H,EAAS7H,GACtBuH,GAAU,CAEd,QAAO,EAGXhT,MAAKoT,OAAS,WACNL,IACAF,EAAUE,GAAM,GAChBA,EAAO,MAEPM,EAAKhT,KAAKL,OACVA,KAAKL,SAAS8T,OAAOzC,KAkBjC/R,EAAGG,MAAM0O,YAAc,SAAS6D,GAQ5B,MAPIA,IACAlS,GAAGiU,IAAI/B,GAAMzI,QAAO,SAAWuC,EAAIR,GAC3BhM,EAAGG,MAAM4P,WAAWhP,KAAKyL,KACzBzL,KAAKyL,GAAKR,IAEfnL,KAAKE,OAELA,MAWXf,EAAGG,MAAMuU,WAAa,SAASC,EAAUrQ,GAErC,GAAIsQ,GAAY,EACZ5R,EAAI,CACR,KAAKA,EAAGA,EAAIsB,EAAKlC,OAAQY,GAAK,EAAG,CAC7B,GAAI6R,GAAavQ,EAAKtB,IAAMsB,EAAKtB,GAAG6F,OAASvE,EAAKtB,GAAG6F,OAAOzG,OAAS,CACrEwS,GAAY
 C,EAAaD,EAAYC,EAAaD,EAWtD,MATA5U,GAAGiC,IAAG,8BAAgC0S,GACtC3U,EAAGiC,IAAG,gCAAkC2S,GAExCD,EAAWA,EAAWC,EAAYD,EAAWC,EAAY,EAAID,EAE7DA,EAAsB,EAAXA,EAAe,EAAIA,EAE9BA,EAAWjL,KAAKwB,MAAMyJ,GACtB3U,EAAGiC,IAAG,8BAAgC0S,GAC/BA,GAOX3U,EAAGG,MAAM2U,WAAa,SAASH,EAAUrQ,GAErC,MAAOtE,GAAGG,MAAMuU,WAAWC,EAAUrQ,IAYzCtE,EAAGG,MAAM4U,WAAa,SAASjS,EAAOR,GAE9BQ,EAAMkS,QAAUlS,EAAMkS,OAAO1S,GAC7BQ,EAAMR,GAAQQ,EAAMkS,OAAO1S,IAE3BQ,EAAMR,GAAQ,SAAUqG,GACpB,MAAKtH,WAAUe,QACfU,EAAMmS,WAAW3S,IAAQ,EACzBQ,EAAMgM,SAASxM,GAAQqG,EAChB7F,GAHuBA,EAAMgM,SAASxM,IAOjDQ,EAAK,IAAOR,GAAQ,SAASqG,GACzB,MAAKtH,WAAUe,QACVU,EAAMmS,WAAW3S,KAClBQ,EAAMgM,SAASxM,GAAQqG,GAEpB7F,GAJuBA,EAAMgM,SAASxM,MAazDtC,EAAGG,MAAMqP,YAAc,SAAS1M,GAC5BA,EAAMmS,WAAanS,EAAMmS,cACzB,IAAIC,GAAMnG,OAAOoG,oBAAoBrS,EAAMgM,cACvCsG,EAAQrG,OAAOoG,oBAAoBrS,EAAMkS,WAC7CE,GAAMA,EAAIxT,OAAO0T,EACjB,KAAK,GAAIpS,KAAKkS,GACVlV,EAAGG,MAAM4U,WAAWjS,EAAOoS,EAAIlS,KAUvChD,EAAGG,MAAMkV,iBAAmB,SAASnQ,EAAQoQ,EAAWC,GACpDrQ,EAAOsQ,WAAaD,EAAO7T,OAAOwD,EAAOsQ,gBACzCD,EAAOE,QAAQ
 H,GACfC,EAAOE,QAAQvQ,GACf1E,GAAGkV,OAAOjU,MAAMV,KAAMwU,IAO1BvV,EAAGG,MAAMwV,YAAc,SAAS7F,GAC5B,MAAOA,GAAE8F,OAAOnJ,OAAO,SAASoJ,EAAM5I,GAClC,OAAQA,GAAO4I,GAAQ/F,EAAE7C,EAAM,MAUvCjN,EAAGG,MAAM2V,UAAYtV,GAAGiU,MAMxBzU,EAAGG,MAAM4V,OAAS,WAGd,QAASA,GAAO5N,EAAEnF,GACd,GAAIgT,GAAItQ,EAAKtE,KAAKL,KAAKoH,EAAEnF,GACrBiT,EAAIvG,EAAKtO,KAAKL,KAAKoH,EAAEnF,EACzB,OAAsC,KAAlCxC,GAAG0V,IAAIC,YAAYhD,QAAQ6C,GACpBxV,GAAG0V,IAAIH,SAASrQ,KAAKsQ,GAAGtG,KAAKuG,KAE7BjW,EAAGG,MAAM2V,UAAU7G,IAAI+G,GAAGC,GARzC,GAAIvQ,GACAgK,EAAO,EAoBX,OAVAqG,GAAOrQ,KAAO,SAASiD,GACnB,MAAKtH,WAAUe,QACfsD,EAAOlF,GAAG4V,QAAQzN,GACXoN,GAFuBrQ,GAIlCqQ,EAAOrG,KAAO,SAAS/G,GACnB,MAAKtH,WAAUe,QACfsN,EAAOlP,GAAG4V,QAAQzN,GACXoN,GAFuBrG,GAI3BqG,GAUX/V,EAAGG,MAAMkW,eAAiB,SAASnR,EAAQwO,GAEvC,GAAIwB,GAAMnG,OAAOoG,oBAAoBzB,EAAO5E,cACxCsG,EAAQrG,OAAOoG,oBAAoBzB,EAAOsB,YAC1CsB,EAAY5C,EAAO6C,eACnBC,EAAQ9C,EAAO8B,eACf9C,EAAOwC,EAAIxT,OAAO0T,GAAO1T,OAAO4U,GAAW5U,OAAO8U,EACtD9D,GAAK+C,QAAQ/B,GACbhB,EAAK+C,QAAQvQ,GACb1E,GAAGkV,OAAOjU,MAAMV,KAAM2R,GA
 EtBxN,EAAOqR,WAAavW,EAAGG,MAAMwV,YAAYT,EAAIxT,OAAO0T,GAAO1T,OAAO4U,GAAW5U,OAAOwT,GAAKxT,OAAOwD,EAAOqR,iBACvGrR,EAAOsQ,WAAaxV,EAAGG,MAAMwV,YAAYa,EAAM9U,OAAOwD,EAAOsQ,kBAOjExV,EAAGG,MAAMsW,QAAU,SAASP,GACxBA,EAAIxL,SAAOgM,YAAa,KAO5B1W,EAAGG,MAAMwW,eAAiB,SAAStP,EAAQH,GACvC,MAAQG,IAAU+K,SAASlL,EAAUuD,MAAK,UAAY,KAAO,KAOjEzK,EAAGG,MAAMyW,cAAgB,SAASxP,EAAOF,GACrC,MAAQE,IAASgL,SAASlL,EAAUuD,MAAK,SAAW,KAAO,KAO/DzK,EAAGG,MAAMyF,gBAAkB,SAASyB,EAAQH,EAAW3B,GACnD,MAAOmE,MAAKL,IAAI,EAAErJ,EAAGG,MAAMwW,eAAetP,EAAQH,GAAa3B,EAAOE,IAAMF,EAAOsR,SAMvF7W,EAAGG,MAAMwF,eAAiB,SAASyB,EAAOF,EAAW3B,GACjD,MAAOmE,MAAKL,IAAI,EAAErJ,EAAGG,MAAMyW,cAAcxP,EAAOF,GAAa3B,EAAOC,KAAOD,EAAOuR,QAMtF9W,EAAGG,MAAM4W,OAAS,SAASjU,EAAOoE,GAC9B,GAAI8P,GAAMlU,EAAM8L,UACZrJ,EAASyR,EAAIzR,SACbwR,EAASC,EAAID,SACbzS,EAAkB,MAAVyS,GAAkB,uBAA0BA,GACpD1P,EAASrH,EAAGG,MAAMyF,gBAAgB,KAAMsB,EAAW3B,GACnD6B,EAAQpH,EAAGG,MAAMwF,eAAe,KAAMuB,EAAW3B,GACjDuC,EAAIvC,EAAOC,KAAO4B,EAAM,EACxBmI,EAAIhK,EAAOE,IAAM4B,EAAO,CAG5BH,GAAUK,UAAS,KAAMc,QAEzB,IAA
 I4O,GAAa/P,EAAUK,UAAS,cAAejD,KAAKA,EAExD2S,GAAWxP,QAAQC,OAAM,QACpBC,KAAI,QAAU,kBACdA,KAAI,KAAO,SACX8C,MAAK,cAAgB,UAE1BwM,EACKtP,KAAI,IAAMG,GACVH,KAAI,IAAM4H,GACV2C,KAAK,SAAS8D,GAAI,MAAOA,MAMlChW,EAAGG,MAAM+W,UAAY,SAAUhF,EAAM9K,GACjC8K,EAAK7N,KAAK,WAUN,IATA,GAEI8S,GAFAjF,EAAO1R,GAAG2G,OAAOpG,MACjBqW,EAAQlF,EAAKA,OAAOmF,MAAK,OAAQC,UAEjCtP,KACAuP,EAAa,EACbC,EAAa,IACbjI,EAAI2C,EAAKvK,KAAI,KACb8P,EAAKC,WAAWxF,EAAKvK,KAAI,OACzBgQ,EAAQzF,EAAKA,KAAK,MAAMxK,OAAM,SAAUC,KAAI,IAAM,GAAGA,KAAI,IAAM4H,GAAG5H,KAAI,KAAO8P,EAAK,MAC/EN,EAAOC,EAAM/D,OAChBrL,EAAKvE,KAAK0T,GACVQ,EAAMzF,KAAKlK,EAAK4P,KAAI,MAChBD,EAAMvN,OAAOyN,wBAA0BzQ,IACvCY,EAAKqL,MACLsE,EAAMzF,KAAKlK,EAAK4P,KAAI,MACpB5P,GAAQmP,GACRQ,EAAQzF,EAAKxK,OAAM,SAAUC,KAAI,IAAM,GAAGA,KAAI,IAAM4H,GAAG5H,KAAI,OAAS4P,EAAaC,EAAaC,EAAK,MAAMvF,KAAKiF,OAS9HnX,EAAGG,MAAM2X,YAAc,SAAUC,EAAQC,GACrC,GAAID,IAAWC,EACX,OAAO,CAEX,KAAKD,IAAWC,EACZ,OAAO,CAGX,IAAID,EAAO3V,QAAU4V,EAAO5V,OACxB,OAAO,CAEX,KAAK,GAAIY,GAAI,EACTiV,EAAIF,EAAO3V,OAAY6V,EAAJjV,EAAOA,IAE1B,GAA
 I+U,EAAO/U,YAAc9B,QAAS8W,EAAOhV,YAAc9B,QAEnD,IAAKlB,EAAG8X,YAAYC,EAAO/U,GAAIgV,EAAOhV,IAClC,OAAO,MACR,IAAI+U,EAAO/U,IAAMgV,EAAOhV,GAE3B,OAAO,CAGf,QAAO,GCpsBXhD,EAAGI,OAAO8X,KAAO,WACb,YAqCA,SAASpV,GAAMsB,GAgTX,MA/SAqO,GAAYW,QACZhP,EAAUC,KAAK,SAASC,GACpB,GAAI4C,GAAY1G,GAAG2G,OAAOpG,KAC1Bf,GAAGG,MAAMsW,QAAQvP,EAGjB,IAAII,GAAOJ,EAAUK,UAAS,qBAAsBjD,MAAMA,IACtDkD,EAAYF,EAAKG,QAAQC,OAAM,KAAMC,KAAI,QAAU,wBAEnDwQ,GADS3Q,EAAUE,OAAM,KACrBJ,EAAKH,OAAM,KAEL,QAAViR,EACAF,EAAKE,MAAMA,IACW,OAAjBF,EAAKG,UAAsC,UAAjBH,EAAKG,WACpCH,EAAKE,MAAM1O,KAAKC,IAAIrB,EAAM7B,QAAQ,GAAK6B,EAAM7B,QAAQ,IAAM,KAG/D0R,EAAE3F,gBAAgBC,EAAa,QAAQrR,KAAK8W,GAE5CI,EAASA,GAAUJ,EAAK5P,OAExB,IAAIiQ,GAAML,EAAKM,YACJ,OAAPD,IACAA,EAAMD,EAAOE,aAGjB,IAAIC,GAAYN,EAAE5Q,UAAS,qBACtBjD,MAAMoU,GAAiB,MAC5BD,GAAUrQ,OAAOC,SAGAxE,SAAbsO,GACAgG,EAAE5Q,UAAS,KAAMJ,OAAM,QAASsD,MAAK,YAAc0H,EAGvD,IAAIwG,GACAC,EACAC,CACJ,QAAQX,EAAKG,UACT,IAAK,MACDI,EAAUhR,QAAQC,OAAM,QAASC,KAAI,QAAU,gBACjDkR,EAAI,EACyB,IAAzBvQ,EAAM7B,QAAQrE,OAChByW,EAAIC,EAA+B,EAAnBx
 Q,EAAM7B,QAAQ,GAAS6B,EAAM5B,YAAc,EACzB,IAAzB4B,EAAM7B,QAAQrE,OACvByW,EAAIC,EAAYxQ,EAAM7B,QAAQ,GAAK6B,EAAM7B,QAAQ,GAAK6B,EAAM5B,YAAc4B,EAAM7B,QAAQ,GAC9E6B,EAAM7B,QAAQrE,OAAS,IACjCyW,EAAIvQ,EAAM7B,QAAQ6B,EAAM7B,QAAQrE,OAAO,IAAIkG,EAAM7B,QAAQ,GAAG6B,EAAM7B,QAAQ,KAE1EgS,EACK9Q,KAAI,cAAgB,UACpBA,KAAI,IAAM,GACVA,KAAI,IAAMkR,EAAE,GACbE,IACAH,EAAatR,EAAKC,UAAS,mBACtBjD,KAAKgE,EAAM3B,UAChBiS,EAAWnR,QAAQC,OAAM,KAAMC,KAAI,QAAS,SAASQ,EAAEnF,GAC/C,OAAO,gBAAe,kBAA0B,GAALA,EAAS,eAAa,gBAAmB4U,KAAI,OAC7FlQ,OAAM,QACTkR,EAAWxQ,OAAOC,SAClBuQ,EACKjR,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,aAAehD,EAAGG,MAAM8H,UAAUK,EAAMH,IAAM,QAExDhB,OAAM,QACNQ,KAAI,KAAO,UACXA,KAAI,KAAOuQ,EAAKc,eAChBrR,KAAI,cAAgB,UACpBuK,KAAK,SAAS/J,EAAEnF,GACb,GAAIkG,GAAIqP,EAAIpQ,EACZ,QAAO,GAAMe,GAAG5D,MAAK,OAAU,GAAK4D,IAE5C0P,EAAWpG,gBAAgBC,EAAa,eACnC9K,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,aAAehD,EAAGG,MAAM8H,UAAUK,EAAM7B,QAAQzD,IAAM,QAGzE,MACJ,KAAK,SACD2V,EAAeM,EAAoB,EACnC,IAAIC,GAAe,GACfC,EAAa,EACbC,EAASjB,EAAE5Q,UAAS,KAAMJ,OAAM,QAChCkS,EAAm
 B,EACvB,IAAIC,EAAa,IAAK,CAElBF,EAAOzR,KAAI,YAAc,IAEzByR,EAAO/U,KAAK,SAAS8D,EAAEnF,GACnB,GAAIuW,GAAMxY,KAAKwM,wBACXnG,EAAQmS,EAAInS,KAChB+R,GAAaI,EAAIlS,OACdD,EAAQ8R,IAAcA,EAAe9R,KAE5CiS,EAAmB,UAAYC,EAAe,OAASH,EAAW,EAAIjB,EAAKc,eAAiB,GAE5F,IAAIQ,GAAM9P,KAAKC,IAAID,KAAK8P,IAAIF,EAAa5P,KAAK+P,GAAG,KACjDd,IAAgBa,EAAMA,EAAIN,EAAeA,GAAc,GAEvDE,EACKzR,KAAI,YAAc0R,GAClB5O,MAAK,cAAgB6O,EAAa,IAAM,EAAI,QAAU,WAEvDI,GACAN,EACKzR,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,gBAAkBA,EAAI,GAAK,EAAI,IAAM,MAAQ,MAG5DoW,EAAOzR,KAAI,YAAc,iBAGjC8Q,GAAUhR,QAAQC,OAAM,QAASC,KAAI,QAAU,gBAC/CkR,EAAI,EACyB,IAAzBvQ,EAAM7B,QAAQrE,OACdyW,EAAIC,EAA+B,EAAnBxQ,EAAM7B,QAAQ,GAAS6B,EAAM5B,YAAc,EAC3B,IAAzB4B,EAAM7B,QAAQrE,OACrByW,EAAIC,EAAYxQ,EAAM7B,QAAQ,GAAK6B,EAAM7B,QAAQ,GAAK6B,EAAM5B,YAAc4B,EAAM7B,QAAQ,GAChF6B,EAAM7B,QAAQrE,OAAS,IAC/ByW,EAAIvQ,EAAM7B,QAAQ6B,EAAM7B,QAAQrE,OAAO,IAAIkG,EAAM7B,QAAQ,GAAG6B,EAAM7B,QAAQ,KAE9EgS,EACK9Q,KAAI,cAAgB,UACpBA,KAAI,IAAMgR,GACVhR,KAAI,IAAMkR,EAAE,GACbE,IAEAH,EAAatR,EAAKC,UAAS,mBAEtBjD,MAAMgE,E
 AAM3B,SAAS,GAAI2B,EAAM3B,SAAS2B,EAAM3B,SAASvE,OAAS,KACrEwW,EAAWnR,QAAQC,OAAM,KAAMC,KAAI,QAAS,SAASQ,EAAEnF,GAC/C,OAAO,gBAAe,kBAA0B,GAALA,EAAS,eAAa,gBAAmB4U,KAAI,OAC7FlQ,OAAM,QACTkR,EAAWxQ,OAAOC,SAClBuQ,EACKjR,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,aAAehD,EAAGG,MAAM8H,UAAWK,EAAMH,IAAM2Q,EAAYxQ,EAAM5B,YAAc,EAAI,IAAO,QAEpGS,OAAM,QACNQ,KAAI,KAAO,SACXA,KAAI,IAAMuQ,EAAKc,eACfrR,KAAI,YAAc0R,GAClB5O,MAAK,cAAgB6O,EAAgBA,EAAa,IAAM,EAAI,QAAU,MAAS,UAC/EpH,KAAK,SAAS/J,EAAEnF,GACb,GAAIkG,GAAIqP,EAAIpQ,EACZ,QAAO,GAAMe,GAAG5D,MAAK,OAAU,GAAK4D,IAE5C0P,EAAWpG,gBAAgBC,EAAa,kBACnC9K,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,aAAehD,EAAGG,MAAM8H,UAAWK,EAAMH,IAAM2Q,EAAYxQ,EAAM5B,YAAc,EAAI,IAAO,QAI7G,MACJ,KAAK,QACD+R,EAAUhR,QAAQC,OAAM,QAASC,KAAI,QAAU,gBAC/C8Q,EACKhO,MAAK,cAAgBkP,EAAe,SAAW,SAC/ChS,KAAI,YAAcgS,EAAe,aAAe,IAChDhS,KAAI,IAAMgS,GAAiBjQ,KAAKL,IAAI9D,EAAOuR,MAAO1P,GAAS,IAAM6R,GAAqB,GAAM,KAC5FtR,KAAI,IAAMgS,EAAgBnZ,GAAG6I,IAAIf,EAAM7B,SAAW,EAAKyR,EAAKc,eAC7DD,IACAH,EAAatR,EAAKC,UAAS,mBACtBjD,KAAKgE,EAAM3B,UAChBiS
 ,EAAWnR,QAAQC,OAAM,KAAMC,KAAI,QAAS,SAASQ,EAAEnF,GAC/C,OAAO,gBAAe,kBAA0B,GAALA,EAAS,eAAa,gBAAmB4U,KAAI,OAC7FlQ,OAAM,QACJ+C,MAAK,UAAY,GACtBmO,EAAWxQ,OAAOC,SAClBuQ,EACKjR,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,eAAiBhD,EAAGG,MAAM8H,UAAUK,EAAMH,IAAM,MAE1DhB,OAAM,QACNQ,KAAI,KAAO,SACXA,KAAI,IAAM,GACVA,KAAI,IAAMuQ,EAAKc,eACfvO,MAAK,cAAgB,SACrByH,KAAK,SAAS/J,EAAGnF,GACd,GAAIkG,GAAIqP,EAAIpQ,EACZ,QAAO,GAAMe,GAAG5D,MAAK,OAAU,GAAK4D,IAE5C0P,EAAWpG,gBAAgBC,EAAa,iBACnC9K,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,eAAiBhD,EAAGG,MAAM8H,UAAUK,EAAM7B,QAAQzD,IAAM,MAElEmE,OAAM,QACNsD,MAAK,UAAY,GAE1B,MACJ,KAAK,OASDgO,EAAUhR,QAAQC,OAAM,QAASC,KAAI,QAAU,gBAC/C8Q,EACKhO,MAAK,cAAgBkP,EAAe,SAAW,OAC/ChS,KAAI,YAAcgS,EAAe,cAAgB,IACjDhS,KAAI,IAAMgS,GAAiBjQ,KAAKL,IAAI9D,EAAOC,KAAM4B,GAAS,IAAM6R,GAAqB,GAAM,KAC3FtR,KAAI,IAAMgS,GAAiBnZ,GAAG6I,IAAIf,EAAM7B,SAAW,GAAMyR,EAAKc,eAC/DD,IACAH,EAAatR,EAAKC,UAAS,mBACtBjD,KAAKgE,EAAM3B,UAChBiS,EAAWnR,QAAQC,OAAM,KAAMC,KAAI,QAAS,SAASQ,EAAEnF,GAC/C,OAAO,gBAAe,kBAA0B,GAALA,EAAS,eAAa,gBAAmB4U,KA
 AI,OAC7FlQ,OAAM,QACJ+C,MAAK,UAAY,GACtBmO,EAAWxQ,OAAOC,SAClBuQ,EACKjR,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,eAAiBhD,EAAGG,MAAM8H,UAAUqQ,EAAOnQ,IAAM,MAE3DhB,OAAM,QACNQ,KAAI,KAAO,SACXA,KAAI,IAAM,GACVA,KAAI,KAAOuQ,EAAKc,eAChBrR,KAAI,cAAgB,OACpBuK,KAAK,SAAS/J,EAAEnF,GACb,GAAIkG,GAAIqP,EAAIpQ,EACZ,QAAO,GAAMe,GAAG5D,MAAK,OAAU,GAAK4D,IAE5C0P,EAAWpG,gBAAgBC,EAAa,iBACnC9K,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,eAAiBhD,EAAGG,MAAM8H,UAAUK,EAAM7B,QAAQzD,IAAM,MAElEmE,OAAM,QACNsD,MAAK,UAAY,IA2BlC,GAvBAgO,EAAUvG,KAAK,SAAS/J,GAAK,MAAOA,MAEhC4Q,GAAiC,SAAlBb,EAAKG,UAAyC,UAAlBH,EAAKG,WAEhDF,EAAE5Q,UAAS,KACNlD,KAAK,SAAS8D,EAAEnF,GACbxC,GAAG2G,OAAOpG,MAAMoG,OAAM,QAASQ,KAAI,UAAY,IAC3CW,EAAMH,GAAKG,EAAM7B,QAAQ,GAAK,IAAM6B,EAAMH,GAAKG,EAAM7B,QAAQ,GAAK,OAC9D0B,EAAI,OAAa,OAAJA,IACb3H,GAAG2G,OAAOpG,MAAM4G,KAAI,UAAY,GAEpCnH,GAAG2G,OAAOpG,MAAMoG,OAAM,QAASQ,KAAI,UAAY,MAKvDW,EAAM3B,SAAS,IAAM2B,EAAM3B,SAAS,IAA2B,GAArB2B,EAAM3B,SAAS,IACzDW,EAAKC,UAAS,mBAAoBkD,MAAK,UAAY,SAAUtC,EAAGnF,GAC5D,MAAQA,GAAQ,EAAJ,KAKpB+V,IAAiC,QAAlBb
 ,EAAKG,UAAwC,WAAlBH,EAAKG,UAAwB,CACvE,GAAIuB,KACJtS,GAAKC,UAAS,mBACTlD,KAAK,SAAS8D,EAAEnF,GACb,IACQA,EACA4W,EAAYnW,KAAK6E,EAAMH,GAAKpH,KAAKwM,wBAAwBnG,MAAQ,GAEjEwS,EAAYnW,KAAK6E,EAAMH,GAAKpH,KAAKwM,wBAAwBnG,MAAQ,GACxE,MAAOyS,GACA7W,EACA4W,EAAYnW,KAAK6E,EAAMH,GAAK,GAE5ByR,EAAYnW,KAAK6E,EAAMH,GAAK,MAI5CgQ,EAAE5Q,UAAS,KAAMlD,KAAK,SAAS8D,EAAGnF,IAC1BsF,EAAMH,GAAKyR,EAAY,IAAMtR,EAAMH,GAAKyR,EAAY,MAChDzR,EAAI,OAAa,OAAJA,EACb3H,GAAG2G,OAAOpG,MAAMsH,SAEhB7H,GAAG2G,OAAOpG,MAAMoG,OAAM,QAASkB,YAM/C8P,EAAE5Q,UAAS,SACNkF,OAAO,SAAUtE,GAMd,OAAQuP,WAAWhO,KAAK6E,MAAU,IAAJpG,GAAc,MAAmBtE,SAANsE,IAE5DuC,QAAO,QAAS,GAGrB4N,EAAShQ,EAAMwR,SAInBrH,EAAYS,UAAS,kBACdpQ,EA/UX,GAAIoV,GAAO1X,GAAG0V,IAAIgC,OACd5P,EAAQ9H,GAAG8H,MAAMC,SAEjBhD,GAAUE,IAAK,EAAGqR,MAAO,EAAGD,OAAQ,EAAGrR,KAAM,GAC3C4B,EAAQ,GACRC,EAAS,GACTqR,EAAgB,KAChBK,GAAa,EACbO,EAAe,EACfK,GAAe,EACfD,GAAgB,EAChBZ,GAAY,EACZV,EAAQ,KACRa,EAAoB,EACpB9G,EAAWtO,OACX4E,EAAW,IACX/H,EAAWF,GAAGE,SAAQ,YAE5BwX,GACK5P,MAAMA,GACN+P,OAAM,UACNG,WAAW,SAASrQ,GAAK,MAAOA,
 IAOrC,IAAImQ,GACA7F,EAAczS,EAAGG,MAAMsS,YAAY/R,EAAU+H,EAkWjD,OAxCA3F,GAAMoV,KAAOA,EACbpV,EAAMpC,SAAWA,EAEjBoC,EAAM8L,QAAU5O,EAAGG,MAAM0O,YAAYhO,KAAKiC,GAC1CA,EAAMgM,SAAWC,OAAOC,WAEpBiK,mBAAoBhK,IAAK,WAAW,MAAOgK,IAAqB/J,IAAK,SAASvG,GAAGsQ,EAAkBtQ,IACnG+Q,eAAoBzK,IAAK,WAAW,MAAOyK,IAAiBxK,IAAK,SAASvG,GAAG+Q,EAAc/Q,IAC3F2Q,cAAoBrK,IAAK,WAAW,MAAOqK,IAAgBpK,IAAK,SAASvG,GAAG2Q,EAAa3Q,IACzFgR,cAAoB1K,IAAK,WAAW,MAAO0K,IAAgBzK,IAAK,SAASvG,GAAGgR,EAAahR,IACzFoQ,YAAoB9J,IAAK,WAAW,MAAO8J,IAAc7J,IAAK,SAASvG,GAAGoQ,EAAWpQ,IACrF8P,WAAoBxJ,IAAK,WAAW,MAAOyJ,IAAiBxJ,IAAK,SAASvG,GAAG+P,EAAc/P,IAC3FtB,QAAoB4H,IAAK,WAAW,MAAO5H,IAAU6H,IAAK,SAASvG,GAAGtB,EAAOsB,IAC7EyP,OAAoBnJ,IAAK,WAAW,MAAOmJ,IAASlJ,IAAK,SAASvG,GAAGyP,EAAMzP,IAC3EvB,OAAoB6H,IAAK,WAAW,MAAO7H,IAAS8H,IAAK,SAASvG,GAAGvB,EAAMuB,IAC3EwJ,UAAoBlD,IAAK,WAAW,MAAOkD,IAAYjD,IAAK,SAASvG,GAAGwJ,EAASxJ,IAGjFpD,QAAS0J,IAAK,WAAW,MAAO1J,IAAU2J,IAAK,SAASvG,GACpDpD,EAAOE,IAAmB5B,SAAV8E,EAAElD,IAAuBkD,EAAElD,IAASF,EAAOE,IAC3DF,EAAOuR,MAAqBjT,SAAZ8E,EAAEmO,MAA
 uBnO,EAAEmO,MAASvR,EAAOuR,MAC3DvR,EAAOsR,OAAsBhT,SAAb8E,EAAEkO,OAAuBlO,EAAEkO,OAAStR,EAAOsR,OAC3DtR,EAAOC,KAAoB3B,SAAX8E,EAAEnD,KAAuBmD,EAAEnD,KAASD,EAAOC,OAE/DiD,UAAWwG,IAAK,WAAW,MAAOxG,IAAYyG,IAAK,SAASvG,GACxDF,EAASE,EACT8J,EAAYW,MAAM3K,KAEtBH,OAAQ2G,IAAK,WAAW,MAAO3G,IAAS4G,IAAK,SAASvG,GAClDL,EAAQK,EACRuP,EAAK5P,MAAMA,GACXwQ,EAAwC,kBAArBxQ,GAAMjC,WACzBrG,EAAGG,MAAMkV,iBAAiBvS,EAAOwF,GAAO,SAAW,QAAS,YAAa,mBAIjFtI,EAAGG,MAAMqP,YAAY1M,GACrB9C,EAAGG,MAAMkV,iBAAiBvS,EAAOoV,GAAM,SAAW,aAAc,gBAAiB,WAAY,cAAe,eAC5GlY,EAAGG,MAAMkV,iBAAiBvS,EAAOwF,GAAO,SAAW,QAAS,YAAa,eAElExF,GCtYX9C,EAAGI,OAAO2Z,QAAU,WAChB,YAsCA,SAASjX,GAAMsB,GA+NX,MA9NAqO,GAAYW,QACZhP,EAAUC,KAAK,SAASC,GACpB,GAAIqB,GAAiByB,EAAQ7B,EAAOC,KAAOD,EAAOuR,MAC9ClR,EAAkByB,EAAS9B,EAAOE,IAAMF,EAAOsR,MAEnD3P,GAAY1G,GAAG2G,OAAOpG,MACtBf,EAAGG,MAAMsW,QAAQvP,GAGjBd,EAAOO,OAAOqT,GAAW1V,EAAKmQ,IAAI,SAAStM,EAAEnF,GAAK,MAAOiX,GAAK9R,EAAEnF,MAC3DqD,WAAW6T,IAAW,EAAGvU,GAAiB,GAG/C,IAAIwU,KACJ,KAAKC,EAAS,CAEV,GAAiBC,GAAMC,EAAnBzR,IACJvE,GAAK2F,QAAQ
 ,SAAU9B,EAAGnF,GACtB,GAAIuX,GAAKC,EAAMrS,GAAIsS,EAAKC,EAAMvS,GAAIwS,EAAKC,EAAMzS,GAAI0S,EAAKC,EAAM3S,GACxD4S,EAAUC,EAAW7S,EACrB4S,IACAA,EAAQ9Q,QAAQ,SAAUrI,EAAGoB,GACzB6F,EAAOpF,KAAKwX,EAAWrZ,EAAGoB,EAAGa,WAGjC8W,GAAM9R,EAAOpF,KAAKkX,GAClBJ,GAAM1R,EAAOpF,KAAK8W,GAClBE,GAAM5R,EAAOpF,KAAKgX,GAClBI,GAAMhS,EAAOpF,KAAKoX,KAE1BR,EAAO7Z,GAAGgJ,IAAIX,GACdyR,EAAO9Z,GAAG6I,IAAIR,GACdsR,GAAUE,EAAMC,GAGpBY,EAAOvU,OAAOyT,GAAWD,GACzBe,EAAOzU,MAAM0U,IAAWvV,EAAiB,IAGzCwV,EAAUA,GAAWhV,EACrBiV,EAAUA,GAAWH,EAAOpB,OAAOrT,OAAOyU,EAAO,GAAGA,EAAO,IAG3D,IAAI5T,GAAOJ,EAAUK,UAAS,aAAcjD,MAAMA,GAClCgD,GAAKG,QAAQC,OAAM,KAAMC,KAAI,QAAU,eACvDL,GAAKK,KAAI,YAAc,aAAepC,EAAOC,KAAO,IAAMD,EAAOE,IAAM,IAEvE,IAAI6V,GAAWhU,EAAKC,UAAS,eAAgBjD,KAAK,SAAS6D,GAAK,MAAOA,KACnEoT,EAAWD,EAAS7T,QAAQC,OAAM,KAAM+C,MAAK,iBAAmB,MAAMA,MAAK,eAAiB,KAChG6Q,GACK3T,KAAI,QAAU,cACdA,KAAI,YAAc,SAASQ,EAAEnF,EAAEwY,GAAK,MAAO,cAAgBpV,EAAO6T,EAAK9R,EAAEnF,IAA2B,IAArBoD,EAAOM,aAAsB,SAC5GgE,QAAO,QAAU,SAASvC,GAAK,MAAOA,GAAEsT,QAC7CH,EACK9I,gBAAgBC,EAAa,
 wBAC7BhI,MAAK,iBAAmB,GACxBA,MAAK,eAAiB,KACtB2D,MAAM,SAASjG,EAAEnF,GAAK,MAAOA,GAAIyF,EAAWnE,EAAKlC,SACjDuF,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,cAAgBoD,EAAO6T,EAAK9R,EAAEnF,IAA2B,IAArBoD,EAAOM,aAAsB,SAEhF4U,EAASlT,OAAOC,SAKhBkT,EAASlX,KAAK,SAAS8D,EAAEnF,GACrB,GAAIuW,GAAM/Y,GAAG2G,OAAOpG,OACnB6Z,EAAOE,GAAO7Q,QAAQ,SAAUyR,GAC7B,GAAa7X,SAAT6X,EAAEvT,IAA6B,OAATuT,EAAEvT,GAAa,CACrC,GAAIqE,GAAOkP,IAAMd,EAAS,MAAQ,MAClCrB,GAAI7R,OAAM,QACP+C,MAAK,SAAWgG,EAAStI,IAAMmE,EAAMnE,EAAEnF,IACvC2E,KAAI,QAAU,iCAAmC6E,GACpD+M,EAAI7R,OAAM,QACP+C,MAAK,SAAWgG,EAAStI,IAAMmE,EAAMnE,EAAEnF,IACvC2E,KAAI,QAAU,8BAAgC6E,OAK7D,IAAImP,GAAY,WAAa,MAAwB,QAAhBC,EAA4C,GAArBxV,EAAOM,YAAoBgD,KAAKF,IAAI,GAAyB,GAArBpD,EAAOM,cACvGmV,EAAY,WAAa,MAA4B,IAArBzV,EAAOM,YAAqBiV,IAAY,GACxEG,EAAY,WAAa,MAA4B,IAArB1V,EAAOM,YAAqBiV,IAAY,IAG3Ef,EAAOE,GAAO7Q,QAAQ,SAAUyR,GAC7B,GAAIlP,GAAOkP,IAAMd,EAAS,MAAQ,OAC9BmB,EAAYL,IAAMd,EAASJ,EAAQE,CACvCY,GAASnU,OAAM,sCAAyCqF,GACrDgG,gBAAgBC,EAAa,wBAC3B9K,KAAI,KAA4B,IAArBvB,EAAOM,aAClBiB,KAAI,KAAO,SAASQ,E
 AAEnF,GAAK,MAAOkY,GAAOQ,EAAEvT,MAC3CR,KAAI,KAA4B,IAArBvB,EAAOM,aAClBiB,KAAI,KAAO,SAASQ,EAAEnF,GAAK,MAAOkY,GAAOa,EAAS5T,MACvDmT,EAASnU,OAAM,mCAAsCqF,GAClDgG,gBAAgBC,EAAa,wBAC3B9K,KAAI,KAAOkU,GACXlU,KAAI,KAAO,SAASQ,EAAEnF,GAAK,MAAOkY,GAAOQ,EAAEvT,MAC3CR,KAAI,KAAOmU,GACXnU,KAAI,KAAO,SAASQ,EAAEnF,GAAK,MAAOkY,GAAOQ,EAAEvT,SAGnDyS,EAAOE,GAAO7Q,QAAQ,SAAUyR,GAC7B,GAAIlP,GAAOkP,IAAMd,EAAS,MAAQ,MAClCW,GAAShU,UAAS,eAAkBiF,GACjC7K,GAAE,YAAc,SAASwG,EAAEnF,EAAEwY,GAC1Bhb,GAAG2G,OAAOpG,MAAM2J,QAAO,SAAU,GACjChK,EAASsb,kBACL5P,QAAUI,IAAKkP,EAAEvT,GAAImE,MAAOmE,EAAStI,IAAMmE,EAAMnE,EAAEqT,IACnD5Z,EAAGpB,GAAGuE,UAGbpD,GAAE,WAAa,SAASwG,EAAEnF,EAAEwY,GACzBhb,GAAG2G,OAAOpG,MAAM2J,QAAO,SAAU,GACjChK,EAASsF,iBACLoG,QAAUI,IAAKkP,EAAEvT,GAAImE,MAAOmE,EAAStI,IAAMmE,EAAMnE,EAAEqT,IACnD5Z,EAAGpB,GAAGuE,UAGbpD,GAAE,YAAc,SAASwG,EAAEnF,GACxBtC,EAASmG,kBAAkBjF,EAAGpB,GAAGuE,YAK3CwW,EAAS7T,OAAM,QACVC,KAAI,QAAU,kBAEdhG,GAAE,YAAc,SAASwG,EAAEnF,GACxBxC,GAAG2G,OAAOpG,MAAM2J,QAAO,SAAU,GACjChK,EAASsb,kBACLxP,IAAKyN,EAAK9R,
 GACV6D,MAAOiO,EAAK9R,GACZiE,SACMI,IAAK,KAAMR,MAAO0O,EAAMvS,GAAImE,MAAOmE,EAAStI,IAAMmE,EAAMnE,EAAEnF,KAC1DwJ,IAAK,KAAMR,MAAOiQ,EAAM9T,GAAImE,MAAOmE,EAAStI,IAAMmE,EAAMnE,EAAEnF,KAC1DwJ,IAAK,KAAMR,MAAOwO,EAAMrS,GAAImE,MAAOmE,EAAStI,IAAMmE,EAAMnE,EAAEnF,KAEhEsB,KAAM6D,EACNiB,MAAOpG,EACPpB,EAAGpB,GAAGuE,UAGbpD,GAAE,WAAa,SAASwG,EAAEnF,GACvBxC,GAAG2G,OAAOpG,MAAM2J,QAAO,SAAU,GACjChK,EAASsF,iBACLwG,IAAKyN,EAAK9R,GACV6D,MAAOiO,EAAK9R,GACZiE,SACMI,IAAK,KAAMR,MAAO0O,EAAMvS,GAAImE,MAAOmE,EAAStI,IAAMmE,EAAMnE,EAAEnF,KAC1DwJ,IAAK,KAAMR,MAAOiQ,EAAM9T,GAAImE,MAAOmE,EAAStI,IAAMmE,EAAMnE,EAAEnF,KAC1DwJ,IAAK,KAAMR,MAAOwO,EAAMrS,GAAImE,MAAOmE,EAAStI,IAAMmE,EAAMnE,EAAEnF,KAEhEsB,KAAM6D,EACNiB,MAAOpG,EACPpB,EAAGpB,GAAGuE,UAGbpD,GAAE,YAAc,SAASwG,EAAEnF,GACxBtC,EAASmG,kBAAkBjF,EAAGpB,GAAGuE,UAIzCuW,EAASnU,OAAM,uBACZqL,gBAAgBC,EAAa,qBAC3B9K,KAAI,IAAM,SAASQ,EAAEnF,GAAK,MAAOkY,GAAOR,EAAMvS,MAC9CR,KAAI,QAAUgU,GACdhU,KAAI,IAAMkU,GACVlU,KAAI,SAAW,SAASQ,EAAEnF,GAAK,MAAO0G,MAAKC,IAAIuR,EAAOR,EAAMvS,IAAM+S,EAAOV,EA
 AMrS,MAAQ,IACvFsC,MAAK,OAAS,SAAStC,EAAEnF,GAAK,MAAOyN,GAAStI,IAAMmE,EAAMnE,EAAEnF,KAC5DyH,MAAK,SAAW,SAAStC,EAAEnF,GAAK,MAAOyN,GAAStI,IAAMmE,EAAMnE,EAAEnF,KAGnEuY,EAAS7T,OAAM,QAASC,KAAI,QAAU,qBAEtC2T,EAASnU,OAAM,0BACZqL,gBAAgBC,EAAa,6BAC3B9K,KAAI,KAAOkU,GACXlU,KAAI,KAAO,SAASQ,EAAEnF,GAAK,MAAOkY,GAAOe,EAAM9T,MAC/CR,KAAI,KAAOmU,GACXnU,KAAI,KAAO,SAASQ,EAAEnF,GAAK,MAAOkY,GAAOe,EAAM9T,KAGpD,IAAI+T,GAAWZ,EAAS/T,UAAS,uBAAwBjD,KAAK,SAAS6D,GACnE,MAAO6S,GAAW7S,QAEtB+T,GAASzU,QAAQC,OAAM,UAClB+C,MAAK,OAAS,SAAStC,EAAEnF,EAAEwY,GAAK,MAAOW,GAAWhU,EAAEnF,EAAEwY,IAAMlP,EAAMnE,EAAEqT,KACpE/Q,MAAK,SAAW,SAAStC,EAAEnF,EAAEwY,GAAK,MAAOW,GAAWhU,EAAEnF,EAAEwY,IAAMlP,EAAMnE,EAAEqT,KACtE/Q,MAAK,UAAY,KACjB9I,GAAE,YAAc,SAASwG,EAAEnF,EAAEwY,GAC1Bhb,GAAG2G,OAAOpG,MAAM2J,QAAO,SAAU,GACjChK,EAASsb,kBACL5P,QAAUI,IAAK4P,EAAWjU,EAAEnF,EAAEwY,GAAIlP,MAAO6P,EAAWhU,EAAEnF,EAAEwY,IAAMlP,EAAMnE,EAAEqT,IACtE5Z,EAAGpB,GAAGuE,UAGbpD,GAAE,WAAa,SAASwG,EAAEnF,EAAEwY,GACzBhb,GAAG2G,OAAOpG,MAAM2J,QAAO,SAAU,GACjChK,EAASsF,iBACLoG,QA
 AUI,IAAK4P,EAAWjU,EAAEnF,EAAEwY,GAAIlP,MAAO6P,EAAWhU,EAAEnF,EAAEwY,IAAMlP,EAAMnE,EAAEqT,IACtE5Z,EAAGpB,GAAGuE,UAGbpD,GAAE,YAAc,SAASwG,EAAEnF,GACxBtC,EAASmG,kBAAkBjF,EAAGpB,GAAGuE,UAEzCmX,EAASvU,KAAI,QAAU,sBACvBuU,EACG1J,gBAAgBC,EAAa,kCAC3B9K,KAAI,KAA4B,IAArBvB,EAAOM,aAClBiB,KAAI,KAAO,SAASQ,EAAEnF,EAAEwY,GAAK,MAAON,GAAOD,EAAW9S,EAAEnF,EAAEwY,MAC1D7T,KAAI,IAAM,KACfuU,EAAS9T,OAAOC,SAGhB+S,EAAUhV,EAAO0T,OACjBuB,EAAUH,EAAOpB,SAGrBrH,EAAYS,UAAS,wBACdpQ,EA/PX,GAmBIkX,GAASE,EACTE,EAASe,EASTC,EAASC,EA7BT9V,GAAUE,IAAK,EAAGqR,MAAO,EAAGD,OAAQ,EAAGrR,KAAM,GAC7C4B,EAAQ,IACRC,EAAS,IACTkD,EAAKb,KAAKwB,MAAsB,IAAhBxB,KAAKyB,UACrB/E,EAAS5F,GAAG8H,MAAMsI,UAClBsK,EAAS1a,GAAG8H,MAAMC,SAClB0R,EAAQ,SAAS9R,GAAK,MAAOA,GAAEkU,OAC/B7B,EAAQ,SAASrS,GAAK,MAAOA,GAAEU,OAAOyT,IACtCL,EAAQ,SAAS9T,GAAK,MAAOA,GAAEU,OAAO0T,IACtC7B,EAAQ,SAASvS,GAAK,MAAOA,GAAEU,OAAO2T,IACtC5B,EAAQ,SAASzS,GAAK,MAAOA,GAAEU,OAAO4T,aACtC3B,EAAQ,SAAS3S,GAAK,MAAOA,GAAEU,OAAO6T,cACtCjM,EAAW,SAAStI,GAAK,MAAOA,GAAEmE,OAClC0O,EAAc,SAAS7S,GAAK,MAAOA,
 GAAEU,OAAOqT,UAC5CjB,EAAa,SAAS9S,EAAGnF,EAAGwY,GAAK,MAAOrT,IACxCiU,EAAa,SAASjU,EAAGnF,EAAGwY,GAAK,MAAOrT,IACxCgU,EAAa,SAAShU,EAAGnF,EAAGwY,GAAK,MAAO3X,SACxCyI,EAAQtM,EAAGG,MAAMuQ,eACjBxJ,EAAY,KAGZxG,EAAWF,GAAGE,SAAQ,mBAAqB,kBAAmB,mBAAoB,aAClF+H,EAAW,IACXmT,EAAc,KAOdnJ,EAAczS,EAAGG,MAAMsS,YAAY/R,EAAU+H,EA8RjD,OAtDA3F,GAAMpC,SAAWA,EACjBoC,EAAM8L,QAAU5O,EAAGG,MAAM0O,YAAYhO,KAAKiC,GAE1CA,EAAMgM,SAAWC,OAAOC,WAEpB5H,OAAc6H,IAAK,WAAW,MAAO7H,IAAS8H,IAAK,SAASvG,GAAGvB,EAAMuB,IACrEtB,QAAc4H,IAAK,WAAW,MAAO5H,IAAU6H,IAAK,SAASvG,GAAGtB,EAAOsB,IACvEiT,aAAc3M,IAAK,WAAW,MAAO2M,IAAe1M,IAAK,SAASvG,GAAGiT,EAAYjT,IACjFb,GAAcmH,IAAK,WAAW,MAAOgL,IAAQ/K,IAAK,SAASvG,GAAGsR,EAAKtR,IACnE4R,IAAKtL,IAAK,WAAW,MAAOuL,IAAStL,IAAK,SAASvG,GAAG6R,EAAM7R,IAC5DgU,IAAK1N,IAAK,WAAW,MAAOgN,IAAS/M,IAAK,SAASvG,GAAGsT,EAAMtT,IAC5D8R,IAAKxL,IAAK,WAAW,MAAOyL,IAASxL,IAAK,SAASvG,GAAG+R,EAAM/R,IAC5DgS,IAAK1L,IAAK,WAAW,MAAO2L,IAAS1L,IAAK,SAASvG,GAAGiS,EAAMjS,IAC5DkS,IAAK5L,IAAK,WAAW,MAAO6L,IAAS5L,IAAK,SAASvG,GAAGmS,EAAMnS,IAC5DiU
 ,WAAe3N,IAAK,WAAW,MAAOwB,IAAYvB,IAAK,SAASvG,GAAG8H,EAAS9H,IAC5EuT,UAAejN,IAAK,WAAW,MAAO+L,IAAc9L,IAAK,SAASvG;AN3R1E,AM2R6EqS,EN3R3E,AM2RsFrS,CN3RrF,GAAG,AM4REkU,CN5RD,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EM4RK5N,IN5RC,AM4RI,WAAW,MAAOgM,IAAc/L,IAAK,SAASvG,GAAGsS,EAAWtS,IAChFmU,cAAe7N,IAAK,WAAW,MAAOmN,IAAclN,IAAK,SAASvG,GAAGyT,EAAWzT,IAChFoU,cAAe9N,IAAK,WAAW,MAAOkN,IAAcjN,IAAK,SAASvG,GAAGwT,EAAWxT,IAChFvC,QAAU6I,IAAK,WAAW,MAAO7I,IAAU8I,IAAK,SAASvG,GAAGvC,EAAOuC,IACnEuS,QAAUjM,IAAK,WAAW,MAAOiM,IAAUhM,IAAK,SAASvG,GAAGuS,EAAOvS,IACnEqR,SAAU/K,IAAK,WAAW,MAAO+K,IAAW9K,IAAK,SAASvG,GAAGqR,EAAQrR,IACrEyR,SAAUnL,IAAK,WAAW,MAAOmL,IAAWlL,IAAK,SAASvG,GAAGyR,EAAQzR,IACrEuR,QAAUjL,IAAK,WAAW,MAAOiL,IAAUhL,IAAK,SAASvG,GAAGuR,EAAOvR,IACnEwS,QAAUlM,IAAK,WAAW,MAAOkM,IAAUjM,IAAK,SAASvG,GAAGwS,EAAOxS,IACnE4B,IAAc0E,IAAK,WAAW,MAAO1E,IAAM2E,IAAK,SAASvG,GAAG4B,EAAG5B,IAE/D4G,GACIN,IAAK,WAED,MADA9M,SAAQK,KAAI,mFAGhB0M,IAAK,SAASvG,GACVxG,QAAQK,KAAI,iFAIpB+C,QAAS0J,IAAK,WAAW,MAAO1J,IAAU2J,IAAK,SAASvG,GACpDpD,EAAOE,IAA
 sB5B,SAAb8E,EAAElD,IAAuBkD,EAAElD,IAASF,EAAOE,IAC3DF,EAAOuR,MAAsBjT,SAAb8E,EAAEmO,MAAuBnO,EAAEmO,MAASvR,EAAOuR,MAC3DvR,EAAOsR,OAAsBhT,SAAb8E,EAAEkO,OAAuBlO,EAAEkO,OAAStR,EAAOsR,OAC3DtR,EAAOC,KAAsB3B,SAAb8E,EAAEnD,KAAuBmD,EAAEnD,KAASD,EAAOC,OAE/D8G,OAAS2C,IAAK,WAAW,MAAO3C,IAAS4C,IAAK,SAASvG,GACnD2D,EAAQtM,EAAGG,MAAMsQ,SAAS9H,KAE9BF,UAAWwG,IAAK,WAAW,MAAOxG,IAAYyG,IAAK,SAASvG,GACxDF,EAAWE,EACX8J,EAAYW,MAAM3K,OAI1BzI,EAAGG,MAAMqP,YAAY1M,GAEdA,GCnUX9C,EAAGI,OAAO4c,aAAe,WACrB,YA0CA,SAASla,GAAMsB,GAuHX,MAtHAqO,GAAYW,QACZX,EAAYrS,OAAO6c,GACfC,GAAWzK,EAAYrS,OAAO+c,GAC9BC,GAAW3K,EAAYrS,OAAOid,GAElCjZ,EAAUC,KAAK,SAASC,GACpB,GAAI4C,GAAY1G,GAAG2G,OAAOpG,KAC1Bf,GAAGG,MAAMsW,QAAQvP,EACjB,IAAIvB,IAAkByB,GAAUgL,SAASlL,EAAUuD,MAAK,WAAc,KAAOlF,EAAOC,KAAOD,EAAOuR,MAC9FlR,GAAmByB,GAAU+K,SAASlL,EAAUuD,MAAK,YAAe,KAAOlF,EAAOE,IAAMF,EAAOsR,MAUnG,IARA/T,EAAMqR,OAAS,WACXzT,EAAS4c,eACTpW,EAAUiH,aAAa1F,SAASA,GAAUrH,KAAK0B,IAEnDA,EAAMoE,UAAYnG,MAIbuD,IAASA,EAAKlC,OAAQ,CACvB,GAAI6U,GAAa/P,EAAUK,UAAS,cAAejD,MAAMyS
 ,GAYzD,OAVAE,GAAWxP,QAAQC,OAAM,QACpBC,KAAI,QAAU,kBACdA,KAAI,KAAO,SACX8C,MAAK,cAAgB,UAE1BwM,EACKtP,KAAI,IAAMpC,EAAOC,KAAOG,EAAiB,GACzCgC,KAAI,IAAMpC,EAAOE,IAAMG,EAAkB,GACzCsM,KAAK,SAAS/J,GAAK,MAAOA,KAExBrF,EAEPoE,EAAUK,UAAS,cAAec,SAItCP,EAAImV,EAAQ7W,SACZmJ,EAAI0N,EAAQ/B,SAASqC,OAAM,EAG3B,IAAIjW,GAAOJ,EAAUK,UAAS,gCAAiCjD,MAAMA,IACjEkZ,EAASlW,EAAKG,QAAQC,OAAM,KAAMC,KAAI,QAAU,mCAAmCD,OAAM,KACzF+V,EAAYD,EAAO9V,OAAM,QACzByQ,EAAI7Q,EAAKH,OAAM,IAEnBqW,GAAO9V,OAAM,KAAMC,KAAI,QAAU,gBACjC6V,EAAO9V,OAAM,KAAMC,KAAI,QAAU,gBAC5BD,OAAM,KAAMC,KAAI,QAAU,eAC1BD,OAAM,QAEX8V,EAAO9V,OAAM,KAAMC,KAAI,QAAU,eACjCwQ,EAAExQ,KAAI,YAAc,aAAepC,EAAOC,KAAO,IAAMD,EAAOE,IAAM,KAEhEiY,GACAvF,EAAEhR,OAAM,iBACHQ,KAAI,YAAc,aAAehC,EAAiB,OAI3DsX,EAAQ7V,MAAMzB,GAAgB0B,OAAOzB,EAErC,IAAI+X,GAAWxF,EAAEhR,OAAM,gBAClByW,MAAMtZ,EAAKmI,OAAO,SAAStE,GAAK,OAAQA,EAAE0V,WAc/C,IAZAF,EAASxP,aAAa/M,KAAK6b,GAE3BQ,EAAU/V,OAAM,YACXC,KAAI,KAAO,mBAAqBsV,EAAQ1S,MACxC7C,OAAM,QAEXyQ,EAAEhR,OAAM,oBAAuB8V,EAAQ1S,KAAO,SACzC5C,KAAI,QAAUG,EAAEpB,aAAeg
 T,EAAgB,EAAI,IACnD/R,KAAI,SAAW,IACfA,KAAI,KAAOG,EAAEpB,aAAegT,EAAgB,EAAI,IAGjDwD,EAAW,CACXC,EACK7U,MAAMR,GACNsQ,MAAOpY,EAAGG,MAAMuU,WAAW/O,EAAe,IAAKrB,IAC/CwZ,UAAUlY,EAAiB,GAEhCuS,EAAEhR,OAAM,iBAAkBQ,KAAI,YAAc,eAAiB4H,EAAE9I,QAAQ,GAAK,KAC5E0R,EAAEhR,OAAM,iBAAkB/F,KAAK+b,EAE/B,IAAI/D,GAASjB,EAAEhR,OAAM,iBAAkBI,UAAS,IAC5CmS,IACAN,EACK7R,UAAS,QACTI,KAAI,YAAc,SAASQ,EAAEnF,EAAEwY,GAAK,MAAO,gBAAkBA,EAAI,IAAM,EAAI,IAAM,MAAQ,MAIlG4B,IACAC,EACK/U,MAAMiH,GACN6I,MAAO1O,KAAKwB,MAAMtF,EAAgB,KAClCkY,UAAWnY,EAAgB,GAEhCwS,EAAEhR,OAAM,iBAAkB/F,KAAKic,IAInClF,EAAEhR,OAAM,qBACHQ,KAAI,KAAM,GACVA,KAAI,KAAMhC,GACVgC,KAAI,KAAO4H,EAAE,IACb5H,KAAI,KAAO4H,EAAE,MAQtBkD,EAAYS,UAAS,8BACdpQ,EA3JX,GAaIgF,GAAGyH,EAbH0N,EAAUjd,EAAGI,OAAO2Z,UACpBoD,EAAQnd,EAAGI,OAAO8X,OAClBmF,EAAQrd,EAAGI,OAAO8X,OAElB3S,GAAUE,IAAK,GAAIqR,MAAO,GAAID,OAAQ,GAAIrR,KAAM,IAChD4B,EAAQ,KACRC,EAAS,KACTiF,EAAQtM,EAAGG,MAAMsQ,WACjByM,GAAY,EACZE,GAAY,EACZM,GAAkB,EAClBhE,GAAgB,EAChBxZ,EAAUF,EAAGI,OAAOF,UAEpB6W,EAAS,qBACTrW,EAAWF,GAAGE,SAAQ,eAAiB
 ,aACvC+H,EAAW,GAEf0U,GACK9E,OAAM,UACNU,YAAW,GACXP,WAAW,SAASrQ,GAAK,MAAOA,KAErCkV,EACKhF,OAAM,EAAqB,QAAU,QACrCG,WAAWhY,GAAGmM,OAAM,SAGzBzM,EAAQuI,SAAS,EAMjB,IAAIgK,GAAczS,EAAGG,MAAMsS,YAAY/R,EAAU+H,EA6LjD,OA7DAwU,GAAQvc,SAASiB,GAAE,2BAA6B,SAASoc,GACrD7d,EAAQoE,KAAKyZ,GAAK7X,QAAO,KAG7B+W,EAAQvc,SAASiB,GAAE,0BAA4B,SAASoc,GACpD7d,EAAQoE,KAAKyZ,GAAK7X,QAAO,KAG7B+W,EAAQvc,SAASiB,GAAE,2BAA6B,SAASoc,GACrD7d,MAOJ4C,EAAMpC,SAAWA,EACjBoC,EAAMma,QAAUA,EAChBna,EAAMqa,MAAQA,EACdra,EAAMua,MAAQA,EACdva,EAAM5C,QAAUA,EAEhB4C,EAAM8L,QAAU5O,EAAGG,MAAM0O,YAAYhO,KAAKiC,GAE1CA,EAAMgM,SAAWC,OAAOC,WAEpB5H,OAAa6H,IAAK,WAAW,MAAO7H,IAAS8H,IAAK,SAASvG,GAAGvB,EAAMuB,IACpEtB,QAAa4H,IAAK,WAAW,MAAO5H,IAAU6H,IAAK,SAASvG,GAAGtB,EAAOsB,IACtE+Q,eAAgBzK,IAAK,WAAW,MAAOyK,IAAiBxK,IAAK,SAASvG,GAAG+Q,EAAc/Q,IACvFuU,WAAYjO,IAAK,WAAW,MAAOiO,IAAahO,IAAK,SAASvG,GAAGuU,EAAUvU,IAC3EyU,WAAYnO,IAAK,WAAW,MAAOmO,IAAalO,IAAK,SAASvG,GAAGyU,EAAUzU,IAC3EqV,gBAAoB/O,IAAK,WAAW,MAAO/O,IAAWgP,IAAK,SAASvG,GAAGzI,EAAQyI,IAC/EoO,QAAY9H,IAAK,WAA
 W,MAAO8H,IAAU7H,IAAK,SAASvG,GAAGoO,EAAOpO,IAGrEpD,QAAS0J,IAAK,WAAW,MAAO1J,IAAU2J,IAAK,SAASvG,GACpDpD,EAAOE,IAAsB5B,SAAb8E,EAAElD,IAAuBkD,EAAElD,IAASF,EAAOE,IAC3DF,EAAOuR,MAAsBjT,SAAb8E,EAAEmO,MAAuBnO,EAAEmO,MAASvR,EAAOuR,MAC3DvR,EAAOsR,OAAsBhT,SAAb8E,EAAEkO,OAAuBlO,EAAEkO,OAAStR,EAAOsR,OAC3DtR,EAAOC,KAAsB3B,SAAb8E,EAAEnD,KAAuBmD,EAAEnD,KAASD,EAAOC,OAE/DiD,UAAWwG,IAAK,WAAW,MAAOxG,IAAYyG,IAAK,SAASvG,GACxDF,EAAWE,EACX8J,EAAYW,MAAM3K,GAClBwU,EAAQxU,SAASA,GACjB0U,EAAM1U,SAASA,GACf4U,EAAM5U,SAASA,KAEnB6D,OAAS2C,IAAK,WAAW,MAAO3C,IAAS4C,IAAK,SAASvG,GACnD2D,EAAQtM,EAAGG,MAAMsQ,SAAS9H,GAC1BsU,EAAQ3Q,MAAMA,KAElBoR,iBAAkBzO,IAAK,WAAW,MAAOyO,IAAmBxO,IAAK,SAASvG,GACtE+U,EAAkB/U,EAClB0U,EAAMhF,OAAQ,EAAM,QAAU,YAItCrY,EAAGG,MAAMkW,eAAevT,EAAOma,GAC/Bjd,EAAGG,MAAMqP,YAAY1M,GAEdA,GCjOX9C,EAAGI,OAAO6d,OAAS,WACf,YA6BA,SAASC,GAAWC,EAAQtV,GACxB,GAAIuV,GAAKD,EAAOhd,OAChBgd,GAAOvI,KAAK,SAAS9F,EAAGuO,GACpB,GAAIC,GAAKF,EAAGjL,QAAQrD,GAChByO,EAAKH,EAAGjL,QAAQkL,EACpB,OAAO7d,IAAGge,WAAW3V,EAAOyV,GAAKzV,EAAO0V,M
 AIhD,QAASzb,GAAMsB,GAkOX,MAjOAA,GAAUC,KAAK,SAAS8D,EAAGnF,GACvB,GAAI2C,GAAiByB,EAAQ7B,EAAOC,KAAOD,EAAOuR,MAC9ClR,EAAkByB,EAAS9B,EAAOE,IAAMF,EAAOsR,MAEnD3P,GAAY1G,GAAG2G,OAAOpG,MACtBf,EAAGG,MAAMsW,QAAQvP,EAEjB,IAAIuX,GAASC,EAAOtd,KAAKL,KAAMoH,EAAGnF,GAAG7B,QACjCwd,EAAUC,EAAQxd,KAAKL,KAAMoH,EAAGnF,GAAG7B,QACnC0d,EAAcC,EAAY1d,KAAKL,KAAMoH,EAAGnF,GAAG7B,QAC3C4d,EAAWC,EAAS5d,KAAKL,KAAMoH,EAAGnF,GAAG7B,QACrC8d,EAAcC,EAAY9d,KAAKL,KAAMoH,EAAGnF,GAAG7B,QAC3Cge,EAAeC,EAAahe,KAAKL,KAAMoH,EAAGnF,GAAG7B,QAC7Cke,EAAmBC,EAAiBle,KAAKL,KAAMoH,EAAGnF,GAAG7B,QACrDoe,EAAgBC,EAAcpe,KAAKL,KAAMoH,EAAGnF,GAAG7B,OAGnD+c,GAAWe,EAAaR,GACxBP,EAAWiB,EAAcR,GACzBT,EAAWmB,EAAkBR,GAC7BX,EAAWqB,EAAeR,GAG1BN,EAAO7I,KAAKpV,GAAGge,YACfG,EAAQ/I,KAAKpV,GAAGge,YAChBK,EAAYjJ,KAAKpV,GAAGge,YACpBO,EAASnJ,KAAKpV,GAAGge,WAIjB,IAAIiB,GAAKjf,GAAG8H,MAAMC,SACb5B,OAAQnG,GAAGkf,OAAOlf,GAAGmf,OAAOC,EAAQnB,MACpChY,MAAM6Q,GAAW3R,EAAgB,IAAM,EAAGA,GAGtC5E,MAAK8e,WAAarf,GAAG8H,MAAMC,SAC/B5B,QAAQ,EAAGoD,EAAAA,IACXtD,MAAMgZ,EAAGhZ,QAGd1F,MA
 AK8e,UAAYJ,CAYjB,KAAI,GALAnY,IALW9G,GAAGgJ,IAAIiV,GACPje,GAAG6I,IAAIoV,GACPA,EAAO,GAGXvX,EAAUK,UAAS,uBAAwBjD,MAAM6D,KACxDX,EAAYF,EAAKG,QAAQC,OAAM,KAAMC,KAAI,QAAU,0BACnD6V,EAAShW,EAAUE,OAAM,KACzByQ,EAAI7Q,EAAKH,OAAM,KAEXnE,EAAE,EAAE8c,EAAGrB,EAAOrc,OAAU0d,EAAF9c,EAAMA,IAAC,CACjC,GAAI+c,GAAkB,oBAAoB/c,CAClC,IAALA,IACC+c,EAAkBA,EAAkB,YAAYC,EAAsBhd,IAE1Ewa,EAAO9V,OAAM,QAASC,KAAI,QAAUoY,GAGxCvC,EAAO9V,OAAM,QAASC,KAAI,QAAU,cAEpCL,EAAKK,KAAI,YAAc,aAAepC,EAAOC,KAAO,IAAMD,EAAOE,IAAM,IAOvE,KAAI,GAJAwa,GAAK,SAAS9X,GAAK,MAAOuB,MAAKC,IAAI8V,EAAGtX,GAAKsX,EAAG,KAE9CS,EAAM,SAAS/X,GAAK,MAAesX,GAAJ,EAAJtX,EAAWA,EAAQ,IAE1CnF,EAAE,EAAE8c,EAAGrB,EAAOrc,OAAU0d,EAAF9c,EAAMA,IAAC,CACjC,GAAIyD,GAAQgY,EAAOzb,EACnBmV,GAAEhR,OAAM,gBAAiBnE,GACpB4a,MAAMnX,GACNkB,KAAI,SAAW/B,GACfuI,aACA1F,SAASA,GACTd,KAAI,QAAUsY,EAAGxZ,IACjBkB,KAAI,IAAMuY,EAAIzZ,IAGvB0R,EAAEhR,OAAM,mBACHsD,MAAK,OAAS6B,GACd3E,KAAI,SAAW/B,EAAkB,GACjC+B,KAAI,IAAM/B,EAAkB,GAC5BjE,GAAE,YAAc,WACbjB,EAASsb,kBACLhQ,MAAO+S,EAAS,GAChB1C,MAAOkD,EAAc,IAAM,U
 AC3BjT,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAGnC9I,GAAE,YAAc,WACbjB,EAASmG,kBACLmF,MAAO+S,EAAS,GAChB1C,MAAOkD,EAAc,IAAM,UAC3BjT,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAGnC9I,GAAE,WAAa,WACZjB,EAASsF,iBACLgG,MAAO+S,EAAS,GAChB1C,MAAOkD,EAAc,IAAM,UAC3BjT,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAGnC0D,aACA1F,SAASA,GACTd,KAAI,QAAqB,EAAXoX,EACXU,EAAG,GAAKA,EAAGV,EAAS,IAClBU,EAAGV,EAAS,IAAMU,EAAG,IAC1B9X,KAAI,IAAMuY,EAAInB,GAEnB,IAAIoB,GAAMva,EAAkB,EAExBwa,EAAazB,EAAQlK,IAAK,SAAS4L,EAAQjX,GAC3C,OAAQ4C,MAAOqU,EAAQhE,MAAO8C,EAAa/V,KAE/CoU,GACGjW,UAAS,0BACTjD,KAAK8b,GACL3Y,QACAC,OAAM,QACNC,KAAI,QAAU,qBACdA,KAAI,IAAM,MAAQwY,EAAK,IAAMA,EAAK,KAAQA,EAAM,KAAQA,EAAM,KAAQA,EAAM,KAC5Exe,GAAE,YAAc,SAASwG,GACxBzH,EAASsb,kBACPhQ,MAAO7D,EAAE6D,MACTqQ,MAAOlU,EAAEkU,OAAS,WAClB/P,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,QAC5BwC,KAAMwS,EAAGtX,EAAE6D,OAAQpG,EAAgB,OAItCjE,GAAE,YAAc,SAASwG,GACtBzH,EAASmG,kBACLmF,MAAO7D,EAAE6D,MACTqQ,MAAOlU,EAAEkU,OAAS,WAClB/P,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAGnC9I,GAAE,WAAa,SAASwG,EAAGnF,GACxBtC,EA
 ASsF,iBACLgG,MAAO7D,EAAE6D,MACTqQ,MAAOlU,EAAEkU,OAAS,WAClB/P,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAItC0N,EAAE5Q,UAAS,0BACRjD,KAAK8b,GACLjS,aACA1F,SAASA,GACTd,KAAI,YAAc,SAASQ,GAAK,MAAO,aAAesX,EAAGtX,EAAE6D,OAAS,IAAOpG,EAAkB,EAAK,KAErG,IAAI0a,GAAkBzB,EAAYpK,IAAK,SAAS4L,EAAQjX,GACpD,OAAQ4C,MAAOqU,EAAQhE,MAAOgD,EAAiBjW,KAEnDoU,GACGjW,UAAS,sBACTjD,KAAKgc,GACL7Y,QACAC,OAAM,QACNC,KAAI,SAAW,IACfA,KAAI,QAAU,iBACdA,KAAI,KAAO,SAASQ,GAAK,MAAOsX,GAAGtX,EAAE6D,SACrCrE,KAAI,KAAO,KACXA,KAAI,KAAO,SAASQ,GAAK,MAAOsX,GAAGtX,EAAE6D,SACrCrE,KAAI,KAAO/B,EAAkB,GAC7BjE,GAAE,YAAc,SAASwG,GACxBzH,EAASsb,kBACPhQ,MAAO7D,EAAE6D,MACTqQ,MAAOlU,EAAEkU,OAAS,WAClB/P,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,QAC5BwC,KAAMwS,EAAGtX,EAAE6D,OAAQpG,EAAgB,OAItCjE,GAAE,YAAc,SAASwG,GACtBzH,EAASmG,kBACLmF,MAAO7D,EAAE6D,MACTqQ,MAAOlU,EAAEkU,OAAS,WAClB/P,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAGnC9I,GAAE,WAAa,SAASwG,EAAGnF,GACxBtC,EAASsF,iBACLgG,MAAO7D,EAAE6D,MACTqQ,MAAOlU,EAAEkU,OAAS,WAClB/P,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAItC0N,EAAE5Q,UAAS,s
 BACRjD,KAAKgc,GACLnS,aACA1F,SAASA,GACTd,KAAI,KAAO,SAASQ,GAAK,MAAOsX,GAAGtX,EAAE6D,SACrCrE,KAAI,KAAO,SAASQ,GAAK,MAAOsX,GAAGtX,EAAE6D,SAExC1E,EAAKC,UAAS,aACT5F,GAAE,YAAc,SAASwG,EAAEnF,GACxB,GAAIqZ,GAAQ4C,EAAYjc,IAAMud,EAAmBvd,EACjDtC,GAASsb,kBACLhQ,MAAO7D,EACPkU,MAAOA,EACP/P,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAGnC9I,GAAE,YAAc,WACbjB,EAASmG,kBACLmF,MAAO+S,EAAS,GAChB1C,MAAOkD,EAAc,IAAM,WAC3BjT,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAGnC9I,GAAE,WAAa,SAASwG,EAAEnF,GACvB,GAAIqZ,GAAQ4C,EAAYjc,IAAMud,EAAmBvd,EACjDtC,GAASsF,iBACLgG,MAAO7D,EACPkU,MAAOA,EACP/P,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,cAKrC3H,EAlQX,GAAIyC,IAAUE,IAAK,EAAGqR,MAAO,EAAGD,OAAQ,EAAGrR,KAAM,GAC3C6S,EAAS,OACTf,GAAU,EACVoH,EAAS,SAASvW,GAAK,MAAOA,GAAEuW,QAChCE,EAAU,SAASzW,GAAK,MAAOA,GAAEyW,QAAUzW,EAAEyW,YAC7CE,EAAc,SAAS3W,GAAK,MAAOA,GAAE2W,YAAc3W,EAAE2W,aAAe,IACpEE,EAAW,SAAS7W,GAAK,MAAOA,GAAE6W,UAClCE,EAAc,SAAS/W,GAAK,MAAOA,GAAE+W,YAAc/W,EAAE+W,gBACrDE,EAAe,SAASjX,GAAK,MAAOA,GAAEiX,aAAejX,EAAEiX,iBACvDE,EAAmB,SAASnX,GAAK,MAAOA,GAAEmX,i
 BAAmBnX,EAAEmX,qBAC/DE,EAAgB,SAASrX,GAAK,MAAOA,GAAEqX,cAAgBrX,EAAEqX,kBACzDI,GAAU,GACVxY,EAAQ,IACRC,EAAS,GACTH,EAAY,KACZsR,EAAa,KACblM,EAAQtM,EAAGG,MAAMsQ,UAAQ,YACzB/P,EAAWF,GAAGE,SAAQ,mBAAqB,kBAAmB,oBAC9D6f,GAAqB,UAAY,OAAQ,WACzCP,GAAwB,MAAQ,MAAO,OACvCvX,EAAW,GAoRjB,OA/BA3F,GAAMpC,SAAWA,EACjBoC,EAAM8L,QAAU5O,EAAGG,MAAM0O,YAAYhO,KAAKiC,GAE1CA,EAAMgM,SAAWC,OAAOC,WAEpB0P,QAAczP,IAAK,WAAW,MAAOyP,IAAUxP,IAAK,SAASvG,GAAG+V,EAAO/V,IACvEiW,SAAc3P,IAAK,WAAW,MAAO2P,IAAW1P,IAAK,SAASvG,GAAGiW,EAAQjW,IACzEqW,UAAW/P,IAAK,WAAW,MAAO+P,IAAY9P,IAAK,SAASvG,GAAGqW,EAASrW,IACxEiX,QAAc3Q,IAAK,WAAW,MAAO2Q,IAAU1Q,IAAK,SAASvG,GAAGiX,EAAOjX,IACvEvB,OAAW6H,IAAK,WAAW,MAAO7H,IAAS8H,IAAK,SAASvG,GAAGvB,EAAMuB,IAClEtB,QAAY4H,IAAK,WAAW,MAAO5H,IAAU6H,IAAK,SAASvG,GAAGtB,EAAOsB,IACrE6P,YAAgBvJ,IAAK,WAAW,MAAOuJ,IAActJ,IAAK,SAASvG,GAAG6P,EAAW7P,IACjFF,UAAcwG,IAAK,WAAW,MAAOxG,IAAYyG,IAAK,SAASvG,GAAGF,EAASE,IAG3EpD,QAAS0J,IAAK,WAAW,MAAO1J,IAAU2J,IAAK,SAASvG,GACpDpD,EAAOE,IAAsB5B,SAAb8E,EAAElD,IAAuBkD,EAAElD,IAASF,EAAOE,IA
 C3DF,EAAOuR,MAAsBjT,SAAb8E,EAAEmO,MAAuBnO,EAAEmO,MAASvR,EAAOuR,MAC3DvR,EAAOsR,OAAsBhT,SAAb8E,EAAEkO,OAAuBlO,EAAEkO,OAAStR,EAAOsR,OAC3DtR,EAAOC,KAAsB3B,SAAb8E,EAAEnD,KAAuBmD,EAAEnD,KAASD,EAAOC,OAE/D6S,QAASpJ,IAAK,WAAW,MAAOoJ,IAAUnJ,IAAK,SAASvG,GACpD0P,EAAS1P,EACT2O,EAAoB,SAAVe,GAA+B,UAAVA,IAEnC/L,OAAS2C,IAAK,WAAW,MAAO3C,IAAS4C,IAAK,SAASvG,GACnD2D,EAAQtM,EAAGG,MAAMsQ,SAAS9H,OAIlC3I,EAAGG,MAAMqP,YAAY1M,GACdA,GChTX9C,EAAGI,OAAOogB,YAAc,WACpB,YA2BA,SAAS1d,GAAMsB,GAuHX,MAtHAA,GAAUC,KAAK,SAAS8D,EAAGnF,GACvB,GAAIkE,GAAY1G,GAAG2G,OAAOpG,KAC1Bf,GAAGG,MAAMsW,QAAQvP,EAEjB,IAAIvB,GAAiB3F,EAAGG,MAAMwF,eAAeyB,EAAOF,EAAW3B,GAC3DK,EAAkByB,EAAS9B,EAAOE,IAAMF,EAAOsR,MAOnD,IAJA/T,EAAMqR,OAAS,WAAarR,EAAMsB,IAClCtB,EAAMoE,UAAYnG,MAGboH,IAAMuW,EAAOtd,KAAKL,KAAMoH,EAAGnF,GAE5B,MADAhD,GAAGG,MAAM4W,OAAOjU,EAAOoE,GAChBpE,CAEPoE,GAAUK,UAAS,cAAec,QAGtC,IAAIoW,GAASC,EAAOtd,KAAKL,KAAMoH,EAAGnF,GAAG7B,QAAQyU,KAAKpV,GAAGge,YACjDG,EAAUC,EAAQxd,KAAKL,KAAMoH,EAAGnF,GAAG7B,QAAQyU,KAAKpV,GAAGge,YACnDO,EAAWC,EAAS5d,KAAKL
 ,KAAMoH,EAAGnF,GAAG7B,QAAQyU,KAAKpV,GAAGge,YAGrDlX,EAAOJ,EAAUK,UAAS,4BAA6BjD,MAAM6D,IAC7DX,EAAYF,EAAKG,QAAQC,OAAM,KAAMC,KAAI,QAAU,+BACnD6V,EAAShW,EAAUE,OAAM,KACzByQ,EAAI7Q,EAAKH,OAAM,IAEnBqW,GAAO9V,OAAM,KAAMC,KAAI,QAAU,iBACjC6V,EAAO9V,OAAM,KAAMC,KAAI,QAAU,aAEjCL,EAAKK,KAAI,YAAc,aAAepC,EAAOC,KAAO,IAAMD,EAAOE,IAAM,IAGvE,IAAIga,GAAKjf,GAAG8H,MAAMC,SACb5B,QAAQ,EAAG+C,KAAKL,IAAIoV,EAAO,GAAKE,EAAQ,IAAM,EAAII,EAAS,MAC3DtY,MAAM6Q,GAAW3R,EAAgB,IAAM,EAAGA,IAG3C8a,EAAK1f,KAAK8e,WAAarf,GAAG8H,MAAMC,SAC/B5B,QAAQ,EAAGoD,EAAAA,IACXtD,MAAMgZ,EAAGhZ,QAGd1F,MAAK8e,UAAYJ,CAEjB,IAGIiB,GAAQlD,EAAOrW,OAAM,cAAeO,OAAM,KACzCC,KAAI,cAAgB,OACpBA,KAAI,YAAc,iBAAmBN,EAAS9B,EAAOE,IAAMF,EAAOsR,QAAU,EAAI,IACrF6J,GAAMhZ,OAAM,QACPC,KAAI,QAAU,YACduK,KAAK,SAAS/J,GAAK,MAAOA,GAAEuY,QAEjCA,EAAMhZ,OAAM,QACPC,KAAI,QAAU,eACdA,KAAI,KAAO,OACXuK,KAAK,SAAS/J,GAAK,MAAOA,GAAEwY,WAEjC1C,EACK7W,MAAMzB,GACN0B,OAAOzB,EAEZ,IAAIgb,GAAazI,EAAEhR,OAAM,iBACzB3G,IAAG2N,WAAWyS,GAAYxf,KAAK6c,EAG/B,IAAItR,GAAS6L,GAAciH,EAAGjH,WAAY7S,EAAiB,KAGv
 Dkb,EAAO1I,EAAE5Q,UAAS,aACjBjD,KAAKmb,EAAGrH,MAAOA,EAAQA,EAASzS,EAAiB,IAAO,SAASwC,GAC9D,MAAOpH,MAAK8Q,aAAelF,EAAOxE,KAItC2Y,EAAYD,EAAKpZ,QAAQC,OAAM,KAC9BC,KAAI,QAAU,WACdA,KAAI,YAAc,SAASQ,GAAK,MAAO,aAAesY,EAAGtY,GAAK,QAC9DsC,MAAK,UAAY,KAEtBqW,GAAUpZ,OAAM,QACXC,KAAI,KAAO/B,GACX+B,KAAI,KAAyB,EAAlB/B,EAAsB,GAEtCkb,EAAUpZ,OAAM,QACXC,KAAI,cAAgB,UACpBA,KAAI,KAAO,OACXA,KAAI,IAAwB,EAAlB/B,EAAsB,GAChCsM,KAAKvF,EAGV,IAAIoU,GAAavgB,GAAG2N,WAAW0S,GAC1B1S,aACA1F,SAASwV,EAAOxV,YAChBd,KAAI,YAAc,SAASQ,GAAK,MAAO,aAAesX,EAAGtX,GAAK,QAC9DsC,MAAK,UAAY,EAEtBsW,GAAW5Z,OAAM,QACZQ,KAAI,KAAO/B,GACX+B,KAAI,KAAyB,EAAlB/B,EAAsB,GAEtCmb,EAAW5Z,OAAM,QACZQ,KAAI,IAAwB,EAAlB/B,EAAsB,GAGrCpF,GAAG2N,WAAW0S,EAAKzY,QACd+F,aACA1F,SAASwV,EAAOxV,YAChBd,KAAI,YAAc,SAASQ,GAAK,MAAO,aAAesX,EAAGtX,GAAK,QAC9DsC,MAAK,UAAY,MACjBpC,WAGT7H,GAAGwgB,MAAMC,QACFne,EA5IX,GAAImb,GAASje,EAAGI,OAAO6d,SACnB/d,EAAUF,EAAGI,OAAOF,UAEpBmY,EAAS,OACPf,GAAU,EACV/R,GAAUE,IAAK,EAAGqR,MAAO,GAAID,OAAQ,GAAIrR,KAAM,KAC/CkZ,EAAS,SAASvW,GAAK,MAAOA,GAAEuW,QA
 ChCE,EAAU,SAASzW,GAAK,MAAOA,GAAEyW,QAAUzW,EAAEyW,YAC7CI,EAAW,SAAS7W,GAAK,MAAOA,GAAE6W,UAClC5X,EAAQ,KACRC,EAAS,GACTmR,EAAa,KACbJ,EAAQ,KACRrB,EAAS,KACTrW,EAAWF,GAAGE,UA2LpB,OAxLAR,GACKuI,SAAS,GACT+C,eAAc,GAgInByS,EAAOvd,SAASiB,GAAE,2BAA6B,SAASoc,GACpDA,EAAW,QACPvR,IAAKuR,EAAI1B,MACTrQ,MAAO+R,EAAI/R,MACXM,MAAOyR,EAAIzR,OAEfpM,EAAQoE,KAAKyZ,GAAK7X,QAAO,KAG7B+X,EAAOvd,SAASiB,GAAE,0BAA4B,SAASoc,GACnD7d,EAAQgG,QAAO,KAGnB+X,EAAOvd,SAASiB,GAAE,2BAA6B,SAASoc,GACpD7d,MAOJ4C,EAAMmb,OAASA,EACfnb,EAAMpC,SAAWA,EACjBoC,EAAM5C,QAAUA,EAEhB4C,EAAM8L,QAAU5O,EAAGG,MAAM0O,YAAYhO,KAAKiC,GAE1CA,EAAMgM,SAAWC,OAAOC,WAEpB0P,QAAczP,IAAK,WAAW,MAAOyP,IAAUxP,IAAK,SAASvG,GAAG+V,EAAO/V,IACvEiW,SAAc3P,IAAK,WAAW,MAAO2P,IAAW1P,IAAK,SAASvG,GAAGiW,EAAQjW,IACzEqW,UAAW/P,IAAK,WAAW,MAAO+P,IAAY9P,IAAK,SAASvG,GAAGqW,EAASrW,IACxEvB,OAAW6H,IAAK,WAAW,MAAO7H,IAAS8H,IAAK,SAASvG,GAAGvB,EAAMuB,IAClEtB,QAAY4H,IAAK,WAAW,MAAO5H,IAAU6H,IAAK,SAASvG,GAAGtB,EAAOsB,IACrE6P,YAAgBvJ,IAAK,WAAW,MAAOuJ,IAActJ,IAAK,SAASvG,GAAG6P,EAAW7P,IACjFyP,O
 AAWnJ,IAAK,WAAW,MAAOmJ,IAASlJ,IAAK,SAASvG,GAAGyP,EAAMzP,IAClEoO,QAAY9H,IAAK,WAAW,MAAO8H,IAAU7H,IAAK,SAASvG,GAAGoO,EAAOpO,IAGrEpD,QAAS0J,IAAK,WAAW,MAAO1J,IAAU2J,IAAK,SAASvG,GACpDpD,EAAOE,IAAsB5B,SAAb8E,EAAElD,IAAuBkD,EAAElD,IAASF,EAAOE,IAC3DF,EAAOuR,MAAsBjT,SAAb8E,EAAEmO,MAAuBnO,EAAEmO,MAASvR,EAAOuR,MAC3DvR,EAAOsR,OAAsBhT,SAAb8E,EAAEkO,OAAuBlO,EAAEkO,OAAStR,EAAOsR,OAC3DtR,EAAOC,KAAsB3B,SAAb8E,EAAEnD,KAAuBmD,EAAEnD,KAASD,EAAOC,OAE/D6S,QAASpJ,IAAK,WAAW,MAAOoJ,IAAUnJ,IAAK,SAASvG,GACpD0P,EAAS1P,EACT2O,EAAoB,SAAVe,GAA+B,UAAVA,MAIvCrY,EAAGG,MAAMkW,eAAevT,EAAOmb,GAC/Bje,EAAGG,MAAMqP,YAAY1M,GAEdA,GCnNX9C,EAAGI,OAAO8gB,eAAiB,WACvB,YAoCA,SAASpe,GAAMsB,GA4HX,MA3HAA,GAAUC,KAAK,SAASC,GACpB4C,EAAY1G,GAAG2G,OAAOpG,KACtB,IAAI4E,GAAiB3F,EAAGG,MAAMwF,eAAeyB,EAAOF,EAAW3B,GAC3DK,EAAkB5F,EAAGG,MAAMyF,gBAAgByB,EAAQH,EAAW3B,EAElEvF,GAAGG,MAAMsW,QAAQvP,EAGjB,IAAIia,GAAYxb,EAAiBrB,EAAK,GAAGuE,OAAOzG,OAAU,GAG1D0F,GAAEnB,OAAOqT,GAAWxZ,GAAGkf,OAAOpb,EAAK,GAAGuE,OAAO4L,IAAIwF,GAAMvY,OAAOke,KAE1DwB,EACAtZ,EAAErB,
 MAAMyT,IAA4B,GAAjBvU,EAAsBrB,EAAK,GAAGuE,OAAOzG,OAAQuD,GAAkBrB,EAAK,GAAGuE,OAAOzG,OAAS,IAAOkC,EAAK,GAAGuE,OAAOzG,SAEhI0F,EAAErB,MAAMyT,IAAW,EAAIiH,EAAW,EAAGxb,EAAiBwb,EAAW,EAAI,IAEzE5R,EAAE5I,OAAOyT,IACD5Z,GAAGgJ,IAAIlF,EAAK,GAAGuE,OAAO4L,IAAI4M,GAAQ3f,OAAO4f,IACzC9gB,GAAG6I,IAAI/E,EAAK,GAAGuE,OAAO4L,IAAI8M,GAAS7f,OAAO4f,MAEhD7a,MAAM0U,IAAWvV,EAAiB,IAGhCkC,EAAEnB,SAAS,KAAOmB,EAAEnB,SAAS,KAC7BmB,EAAEnB,SAAS,GACPmB,EAAEnB,QAAQmB,EAAEnB,SAAS,GAAqB,IAAhBmB,EAAEnB,SAAS,GAAWmB,EAAEnB,SAAS,GAAqB,IAAhBmB,EAAEnB,SAAS,KACzEmB,EAAEnB,QAAM,GAAK,KAEnB4I,EAAE5I,SAAS,KAAO4I,EAAE5I,SAAS,KAC7B4I,EAAE5I,SAAS,GACP4I,EAAE5I,QAAQ4I,EAAE5I,SAAS,GAAqB,IAAhB4I,EAAE5I,SAAS,GAAW4I,EAAE5I,SAAS,GAAqB,IAAhB4I,EAAE5I,SAAS,KACzE4I,EAAE5I,QAAM,GAAK,IAGvB,IAAIW,GAAO9G,GAAG2G,OAAOpG,MAAMwG,UAAS,+BAAgCjD,MAAMA,EAAK,GAAGuE,SAC9ErB,EAAYF,EAAKG,QAAQC,OAAM,KAAMC,KAAI,QAAU,kCACnD8V,EAAYjW,EAAUE,OAAM,QAC5B8V,EAAShW,EAAUE,OAAM,KACzByQ,EAAI7Q,EAAKH,OAAM,IAEnBqW,GAAO9V,OAAM,KAAMC,KAAI,QAAU,YAEjCL,EAAKK,KAAI,YAAc,aAAepC,EAAOC,
 KAAO,IAAMD,EAAOE,IAAM,KAEvEyB,EACKvF,GAAE,QAAU,SAASwG,EAAEnF,GACpBtC,EAAS8gB,YACLld,KAAM6D,EACNiB,MAAOpG,EACPiK,IAAKzM,GAAGuE,MACRwF,GAAIA,MAIhBkT,EAAU/V,OAAM,YACXC,KAAI,KAAO,sBAAwB4C,GACnC7C,OAAM,QAEXJ,EAAKH,OAAM,uBAA0BoD,EAAK,SACrC5C,KAAI,QAAUhC,GACdgC,KAAI,SAAW/B,GAEpBuS,EAAKxQ,KAAI,YAAc8Z,EAAW,2BAA6BlX,EAAK,IAAM,GAE1E,IAAI6N,GAAQ9Q,EAAKH,OAAM,aAAcI,UAAS,YACzCjD,KAAK,SAAS6D,GAAK,MAAOA,IAC/BiQ,GAAMhQ,OAAOC,QAEb,IAAIqZ,GAAatJ,EAAM3Q,QAAQC,OAAM,IAGrC0Q,GACKzQ,KAAI,QAAU,SAASQ,EAAGnF,EAAGwY,GAAK,OAAQmG,EAAQxZ,EAAGnF,GAAK4e,EAASzZ,EAAGnF,GAAK,mBAAqB,oBAAsB,YAAcwY,EAAI,IAAMxY,GAEvI0e,GAAWha,OAAM,QACxBC,KAAI,QAAU,wBACdA,KAAI,YAAc,SAASQ,EAAGnF,GAAK,MAAO,aAAe8E,EAAEmS,EAAK9R,EAAGnF,IAAM,QACzE2E,KAAI,KAAO,GACXA,KAAI,KAAO,SAASQ,EAAGnF,GAAK,MAAOuM,GAAEgS,EAAQpZ,EAAGnF,MAChD2E,KAAI,KAAO,GACXA,KAAI,KAAO,SAASQ,EAAGnF,GAAK,MAAOuM,GAAE8R,EAAOlZ,EAAGnF,MAExC0e,EAAWha,OAAM,QACxBC,KAAI,QAAU,gCACdA,KAAI,YAAc,SAASQ,EAAGnF,GAC3B,MAAO,cAAgB8E,EAAEmS,EAAK9R,EAAGnF,IAAMme,EAAS,GAAK,KAClD5R,EAAEsS,EAAK1Z,EA
 AGnF,KAAO2e,EAAQxZ,EAAGnF,GAAK4e,EAASzZ,EAAGnF,GAAMuM,EAAEqS,EAASzZ,EAAGnF,IAAMuM,EAAEoS,EAAQxZ,EAAGnF,IAAO,IAC5F,MAEL2E,KAAI,IAAM,GACVA,KAAI,IAAM,GACVA,KAAI,QAAUwZ,GACdxZ,KAAI,SAAW,SAASQ,EAAGnF,GACxB,GAAI8e,GAAOH,EAAQxZ,EAAGnF,GAClB+e,EAAQH,EAASzZ,EAAGnF,EACxB,OAAO8e,GAAOC,EAAQxS,EAAEwS,GAASxS,EAAEuS,GAAQvS,EAAEuS,GAAQvS,EAAEwS,IAG/D3J,GAAMjR,OAAM,yBAA0BgH,aACjCxG,KAAI,YAAc,SAASQ,EAAGnF,GAAK,MAAO,aAAe8E,EAAEmS,EAAK9R,EAAGnF,IAAM,QACzE2E,KAAI,KAAO,GACXA,KAAI,KAAO,SAASQ,EAAGnF,GAAK,MAAOuM,GAAEgS,EAAQpZ,EAAGnF,MAChD2E,KAAI,KAAO,GACXA,KAAI,KAAO,SAASQ,EAAGnF,GAAK,MAAOuM,GAAE8R,EAAOlZ,EAAGnF,MAEpDoV,EAAMjR,OAAM,yBAA0BgH,aACjCxG,KAAI,YAAc,SAASQ,EAAGnF,GAC3B,MAAO,cAAgB8E,EAAEmS,EAAK9R,EAAGnF,IAAMme,EAAS,GAAK,KAClD5R,EAAEsS,EAAK1Z,EAAGnF,KAAO2e,EAAQxZ,EAAGnF,GAAK4e,EAASzZ,EAAGnF,GAAMuM,EAAEqS,EAASzZ,EAAGnF,IAAMuM,EAAEoS,EAAQxZ,EAAGnF,IAAO,IAC5F,MAEL2E,KAAI,IAAM,GACVA,KAAI,IAAM,GACVA,KAAI,QAAUwZ,GACdxZ,KAAI,SAAW,SAASQ,EAAGnF,GACxB,GAAI8e,GAAOH,EAAQxZ,EAAGnF,GAClB+e,EAAQH,EAASzZ,EAAGnF,EACxB,
 OAAO8e,GAAOC,EAAQxS,EAAEwS,GAASxS,EAAEuS,GAAQvS,EAAEuS,GAAQvS,EAAEwS,OAI5Djf,EA1JX,GAIMoE,GAeA8S,EACAI,EACAF,EACAiB,EAtBF5V,GAAUE,IAAK,EAAGqR,MAAO,EAAGD,OAAQ,EAAGrR,KAAM,GAC3C4B,EAAQ,KACRC,EAAS,KACTkD,EAAKb,KAAKwB,MAAsB,IAAhBxB,KAAKyB,UAErBrD,EAAItH,GAAG8H,MAAMC,SACbgH,EAAI/O,GAAG8H,MAAMC,SACb0R,EAAO,SAAS9R,GAAK,MAAOA,GAAEL,GAC9B+Z,EAAO,SAAS1Z,GAAK,MAAOA,GAAEoH,GAC9BoS,EAAU,SAASxZ,GAAK,MAAOA,GAAE2Z,MACjCF,EAAW,SAASzZ,GAAK,MAAOA,GAAE4Z,OAClCR,EAAU,SAASpZ,GAAK,MAAOA,GAAE6Z,MACjCX,EAAS,SAASlZ,GAAK,MAAOA,GAAE8Z,KAChCrC,KACA0B,KACAF,GAAc,EACdK,GAAW,EACXnV,EAAQtM,EAAGG,MAAMuQ,eACjBwR,GAAc,EAKdxhB,EAAWF,GAAGE,SAAQ,cAAgB,cAAe,YAAa,aAAc,eAAgB,kBAAmB,mBAAoB,kBAAmB,mBAiMhK,OAzDAoC,GAAMqf,eAAiB,SAASC,EAAYC,GACxCvf,EAAMwf,kBACNpb,EAAUC,OAAM,iCAAoCib,GAC/C1X,QAAO,QAAU2X,IAI1Bvf,EAAMwf,gBAAkB,WACpBpb,EAAUC,OAAM,qCACXuD,QAAO,SAAU,IAQ1B5H,EAAMpC,SAAWA,EACjBoC,EAAM8L,QAAU5O,EAAGG,MAAM0O,YAAYhO,KAAKiC,GAE1CA,EAAMgM,SAAWC,OAAOC,WAEpB5H,OAAW6H,IAAK,WAAW,MAAO7H,IAAS8H,IAAK,SAASvG,GAAGvB,EAAMuB,IAClEtB,QA
 AW4H,IAAK,WAAW,MAAO5H,IAAU6H,IAAK,SAASvG,GAAGtB,EAAOsB,IACpEvC,QAAW6I,IAAK,WAAW,MAAOnH,IAAKoH,IAAK,SAASvG,GAAGb,EAAEa,IAC1DuS,QAAWjM,IAAK,WAAW,MAAOM,IAAKL,IAAK,SAASvG,GAAG4G,EAAE5G,IAC1DqR,SAAW/K,IAAK,WAAW,MAAO+K,IAAW9K,IAAK,SAASvG,GAAGqR,EAAQrR,IACtEyR,SAAWnL,IAAK,WAAW,MAAOmL,IAAWlL,IAAK,SAASvG,GAAGyR,EAAQzR,IACtEuR,QAAWjL,IAAK,WAAW,MAAOiL,IAAUhL,IAAK,SAASvG,GAAGuR,EAAOvR,IACpEwS,QAAWlM,IAAK,WAAW,MAAOkM,IAAUjM,IAAK,SAASvG,GAAGwS,EAAOxS,IACpEiX,QAAW3Q,IAAK,WAAW,MAAO2Q,IAAU1Q,IAAK,SAASvG,GAAGiX,EAAOjX,IACpE2Y,QAAWrS,IAAK,WAAW,MAAOqS,IAAUpS,IAAK,SAASvG,GAAG2Y,EAAO3Y,IACpEyY,SAAWnS,IAAK,WAAW,MAAOmS,IAAWlS,IAAK,SAASvG,GAAGyY,EAAQzY,IACtE8Y,UAAWxS,IAAK,WAAW,MAAOwS,IAAYvS,IAAK,SAASvG,GAAG8Y,EAAS9Y,IACxE4B,IAAW0E,IAAK,WAAW,MAAO1E,IAAM2E,IAAK,SAASvG,GAAG4B,EAAG5B,IAC5DuZ,aAAcjT,IAAK,WAAW,MAAOiT,IAAehT,IAAK,SAASvG,GAAGuZ,EAAYvZ,IAEjFb,GAAQmH,IAAK,WAAW,MAAOgL,IAAQ/K,IAAK,SAASvG,GAAGsR,EAAKtR,IAC7D4G,GAAQN,IAAK,WAAW,MAAO4S,IAAQ3S,IAAK,SAASvG,GAAGkZ,EAAKlZ,IAC7DmZ,MAAQ7S,IAAK,WAAW,MAAO0S,MAAazS
 ,IAAK,SAASvG,GAAGgZ,EAAQhZ,IACrEoZ,OAAQ9S,IAAK,WAAW,MAAO2S,MAAc1S,IAAK,SAASvG,GAAGiZ,EAASjZ,IACvEqZ,MAAQ/S,IAAK,WAAW,MAAOsS,IAAWrS,IAAK,SAASvG,GAAG4Y,EAAQ5Y,IACnEsZ,KAAQhT,IAAK,WAAW,MAAOoS,IAAUnS,IAAK,SAASvG,GAAG0Y,EAAO1Y,IAGjEpD,QAAS0J,IAAK,WAAW,MAAO1J,IAAU2J,IAAK,SAASvG,GACpDpD,EAAOE,IAAqB5B,QAAZ8E,EAAElD,IAAsBkD,EAAElD,IAASF,EAAOE,IAC1DF,EAAOuR,MAAqBjT,QAAZ8E,EAAEmO,MAAsBnO,EAAEmO,MAASvR,EAAOuR,MAC1DvR,EAAOsR,OAAqBhT,QAAZ8E,EAAEkO,OAAsBlO,EAAEkO,OAAStR,EAAOsR,OAC1DtR,EAAOC,KAAqB3B,QAAZ8E,EAAEnD,KAAsBmD,EAAEnD,KAASD,EAAOC,OAE9D8G,OAAS2C,IAAK,WAAW,MAAO3C,IAAS4C,IAAK,SAASvG,GACnD2D,EAAQtM,EAAGG,MAAMsQ,SAAS9H,OAIlC3I,EAAGG,MAAMqP,YAAY1M,GACdA,GC/NX9C,EAAGI,OAAOmiB,oBAAsB,WAC5B,YAsFA,SAASzf,GAAMsB,GAsbX,MArbAqO,GAAYW,QACZX,EAAYrS,OAAOoiB,GACftF,GAAWzK,EAAYrS,OAAO+c,GAC9BC,GAAW3K,EAAYrS,OAAOid,GAClCjZ,EAAUC,KAAK,SAASC,GA0CpB,QAASme,GAAUta,EAAEnF,GACjBxC,GAAG2G,OAAOrE,EAAMoE,WACXuD,MAAK,SAAW,aAGzB,QAASiY,GAASva,EAAEnF,GAChBoG,EAAMtB,EAAItH,GAAGuE,MAAM+C,EACnBsB,EAAMpG,EAAI0G,KAAK6E,MAAM
 oU,EAAG/b,OAAOwC,EAAMtB,IACrC8a,IAGJ,QAASC,GAAQ1a,EAAEnF,GACfxC,GAAG2G,OAAOrE,EAAMoE,WACXuD,MAAK,SAAW,QAGrBsH,EAAM3I,MAAQA,EAAMpG,EACpBtC,EAASoiB,YAAY/Q,GAgPzB,QAAS6Q,KACLG,GACKze,MAAM8E,GAIX,IAAI4Z,GAAclgB,EAAM2F,UACxB3F,GAAM2F,SAAS,GACf3F,EAAMqR,SACNrR,EAAM2F,SAASua,GAnTnB,GAAI9b,GAAY1G,GAAG2G,OAAOpG,KAC1Bf,GAAGG,MAAMsW,QAAQvP,GACjBA,EAAUwD,QAAO,YAAeH,GAAI,EACpC,IAEI5E,GAAiB3F,EAAGG,MAAMwF,eAAeyB,EAAOF,EAAW3B,GAC3DK,EAAkB5F,EAAGG,MAAMyF,gBAAgByB,EAAQH,EAAW3B,EAkBlE,IAhBAzC,EAAMqR,OAAS,WACM,IAAb1L,EACAvB,EAAU9F,KAAK0B,GAEfoE,EAAUiH,aAAa1F,SAASA,GAAUrH,KAAK0B,IAEvDA,EAAMoE,UAAYnG,KAElBgR,EACKmC,OAAO+O,EAAY3e,GAAOxB,EAAMqR,QAChCH,OAAOkP,EAAY5e,IACnB6P,SAGLpC,EAAM8L,SAAWvZ,EAAKmQ,IAAI,SAAStM,GAAK,QAASA,EAAE0V,YAE9CsF,EAAc,CACf,GAAI3W,EACJ2W,KACA,KAAK3W,IAAOuF,GACJA,EAAMvF,YAAgBtL,OACtBiiB,EAAa3W,GAAOuF,EAAMvF,GAAKrL,MAAM,GAErCgiB,EAAa3W,GAAOuF,EAAMvF,GAItC,GAAI4W,GAAY5iB,GAAG6iB,SAASC,OACvB3hB,GAAE,YAAc8gB,GAChB9gB,GAAE,OAAS+gB,GACX/gB,GAAE,UAAYkhB,EAwBnB,MAAKve,GAASA,EAAKlC,QAAWkC,EA
 AKmI,OAAO,SAAStE,GAAK,MAAOA,GAAEU,OAAOzG,SAAUA,QAE9E,MADApC,GAAGG,MAAM4W,OAAOjU,EAAOoE,GAChBpE,CASX,IAPIoE,EAAUK,UAAS,cAAec,SAItCP,EAAI0a,EAAMpc,SACVmJ,EAAIiT,EAAMtH,SAELqI,EAsBDf,EAAMpI,QAAQ,UAtBH,CACX,GAAIoJ,GAAgBlf,EACfmI,OAAO,SAASL,GAAU,OAAQA,EAAOyR,WACzCpJ,IAAI,SAASrI,EAAOpJ,GACjB,GAAIygB,GAAgBjjB,GAAGkf,OAAOtT,EAAOvD,OAAQ2Z,EAAMjT,IAKnD,OAFIkU,GAAc,IAAK,MAAMA,EAAc,IAAK,OAGvCA,EAAc,GAAKA,EAAc,KAAO,EAAIA,EAAc,KAC1DA,EAAc,GAAKA,EAAc,KAAO,EAAIA,EAAc,OAIvEC,GACAljB,GAAGgJ,IAAIga,EAAe,SAASrb,GAAK,MAAOA,GAAE,KAC7C3H,GAAG6I,IAAIma,EAAe,SAASrb,GAAK,MAAOA,GAAE,KAGjDqa,GAAMpI,QAAQsJ,GAKlBf,EAAGhc,QAAQ,EAAGrC,EAAK,GAAGuE,OAAOzG,OAAS,IACjCqE,OAAO,EAAGd,IACV4X,OAAM,EAEX,IAAIjZ,GAAOqf,EAASva,EAAMpG,EAAGsB,GAGzBsf,EAA2B,EAA4B,OAAS,MAChEtc,EAAOJ,EAAUK,UAAS,+BAAgCjD,MAAMA,IAChEkZ,EAASlW,EAAKG,QAAQC,OAAM,KAAMC,KAAI,QAAU,kCAAkCD,OAAM,KACxFyQ,EAAI7Q,EAAKH,OAAM,IA+BnB,IA7BAqW,EAAO9V,OAAM,KAAMC,KAAI,QAAU,kBACjC6V,EAAO9V,OAAM,KAAMC,KAAI,QAAU,gBAAgB8C,MAAK,iBAAgB,QACtE+S,EAAO9V,OAAM,KAAMC,KAAI,QAAU,gB
 ACjC6V,EAAO9V,OAAM,KAAMC,KAAI,QAAU,iBACjC6V,EAAO9V,OAAM,KAAMC,KAAI,QAAU,gBAAgB8C,MAAK,iBAAkBmZ,GACxEpG,EAAO9V,OAAM,KAAMC,KAAI,QAAU,mBAAmB8C,MAAK,iBAAgB,QACzE+S,EAAO9V,OAAM,KAAMC,KAAI,QAAU,iBACjC6V,EAAO9V,OAAM,KAAMC,KAAI,QAAU,mBAG5Bkc,GAGDC,EAAO1c,MAAMzB,GAEbwS,EAAEhR,OAAM,kBACHyW,MAAMtZ,GACNlD,KAAK0iB,GAELC,GAAaD,EAAOzc,WAAa9B,EAAOE,MACzCF,EAAOE,IAAMqe,EAAOzc,SACpBzB,EAAkB5F,EAAGG,MAAMyF,gBAAgByB,EAAQH,EAAW3B,IAGlE4S,EAAEhR,OAAM,kBACHQ,KAAI,YAAc,gBAAmBpC,EAAOE,IAAK,MAdtD0S,EAAEhR,OAAM,kBAAmBI,UAAS,KAAMc,SAkBzC2b,EAEE,CACH,GAAIC,KACEzX,IAAK,kBAAmBqR,UAAW0F,GAGzCW,GACK9c,MAAM,KACNkF,OAAK,OAAU,OAAQ,SACvB6X,YAAW,GACX5e,QAAQE,IAAK,EAAGqR,MAAO,EAAGD,OAAQ,EAAGrR,KAAM,KAGhD2S,EAAEhR,OAAM,oBACHyW,MAAMqG,GACNtc,KAAI,YAAc,gBAAmBpC,EAAOE,IAAK,KACjDrE,KAAK8iB,OAhBT/L,GAAEhR,OAAM,oBAAqBI,UAAS,KAAMc,QAmBjDf,GAAKK,KAAI,YAAc,aAAepC,EAAOC,KAAO,IAAMD,EAAOE,IAAM,KAEnEiY,GACAvF,EAAEhR,OAAM,iBACHQ,KAAI,YAAc,aAAehC,EAAiB,MAI3D,IAAIye,GAAe9f,EAAKmI,OAAO,SAAStE,GAAK,MAAOA,GAAEic,cAEtD9c,GAAKH,OAAM,iBAAkBkB,
 SACzB+b,EAAahiB,QACbkF,EAAKI,OAAM,QAASC,KAAI,QAAU,gBAC7BA,KAAI,IAAMhC,EAAiB,GAC3BgC,KAAI,IAAM,UACV8C,MAAK,cAAgB,OACrByH,KAAKkS,EAAa3P,IAAI,SAAStM,GAAK,MAAOA,GAAEqE,MAAOoL,KAAI,MAAS,sDAItEyM,IACAC,EACKld,MAAMzB,GACN0B,OAAOzB,GACPL,QAAQC,KAAKD,EAAOC,KAAKC,IAAIF,EAAOE,MACpCmC,aAAaV,GACbd,OAAO0B,GACZR,EAAKH,OAAM,mBAAoB/F,KAAKkjB,IAGxC9G,EAAOrW,OAAM,kBACRO,OAAM,QAEXyQ,EAAEhR,OAAM,uBACHQ,KAAI,QAAUhC,GACdgC,KAAI,SAAW/B,GAEpB4c,EAEKjT,EAAE,SAASpH,GAAK,MAAOA,GAAEoc,QAAQhV,IACjCnI,MAAMzB,GACN0B,OAAOzB,GACP0G,MAAMhI,EAAKmQ,IAAI,SAAStM,EAAEnF,GACvB,MAAOmF,GAAEmE,OAASA,EAAMnE,EAAGnF,KAC5ByJ,OAAO,SAAStE,EAAEnF,GAAK,OAAQsB,EAAKtB,GAAG6a,WAAavZ,EAAKtB,GAAGohB,eAEnE,IAAII,GAAYrM,EAAEhR,OAAM,iBACnByW,MAAMtZ,EAAKmI,OAAO,SAAStE,GAAK,OAASA,EAAE0V,WAAa1V,EAAEic,eAE/DI,GAAUpjB,KAAKohB,GAGfle,EAAK2F,QAAQ,SAAS9B,EAAEnF,GACpBmF,EAAEsc,YAAczhB,GAGpB,IAAI0hB,GAAcpgB,EAAKmI,OAAO,SAAStE,GACnC,OAAQA,EAAE0V,YAAc8G,EAAQxc,KAGhCyc,EAAWzM,EAAEhR,OAAM,oBAAqBI,UAAS,QAChDjD,KAAKogB,EAAa,SAASvc,GAAK,MAAOA,GAAEqE,MAE1CqY,
 GAAc,SAAS1c,GAEvB,GAAI2c,GAAOvV,EAAEoV,EAAQxc,GACrB,OAAW,GAAP2c,EAAiB,EACjBA,EAAOlf,EAAwBA,EAC5Bkf,EAGXF,GAASnd,QACJC,OAAM,QACN+C,MAAK,eAAgB,GACrBA,MAAK,mBAAkB,SACvBA,MAAK,SAAU,SAAUtC,EAAEnF,GACxB,MAAOwf,GAAMlW,QAAQnE,EAAEA,EAAEsc,eAE5B9c,KAAI,KAAM,GACVA,KAAI,KAAMhC,GACVgC,KAAI,KAAOkd,IACXld,KAAI,KAAOkd,IAEhBD,EACKna,MAAK,iBAAkB,SAAStC,GAE7B,GAAI2c,GAAOvV,EAAEoV,EAAQxc,GACrB,OAAW,GAAP2c,GAAYA,EAAOlf,EAAwB,EACxC,IAEV+B,KAAI,KAAM,GACVA,KAAI,KAAMhC,GACVgC,KAAI,KAAOkd,IACXld,KAAI,KAAOkd,IAEhBD,EAASxc,OAAOC,QAGhB,IAAI0a,IAAYyB,EAAUjd,UAAS,iBAC9BjD,MAAM8E,GACX2Z,IAAUtb,QAAQC,OAAM,QAASC,KAAI,QAAU,gBAC1CA,KAAI,QAAU,GACdA,KAAI,IAAM,IACVA,KAAI,OAAS,OACbA,KAAI,eAAiB,IACrB8C,MAAK,iBAAgB,OACrBrJ,KAAKgiB,GAEVL,GACKpb,KAAI,YAAc,SAASQ,GAAK,MAAO,aAAewa,EAAGxa,EAAEnF,GAAK,QAChE2E,KAAI,SAAW/B,GAGhBsX,IACAC,EACK7U,MAAMR,GACNid,OAAQ/kB,EAAGG,MAAMuU,WAAW/O,EAAe,GAAIrB,IAC/CwZ,UAAUlY,EAAiB,GAEhCuS,EAAEhR,OAAM,iBACHQ,KAAI,YAAc,eAAiB4H,EAAE9I,QAAQ,GAAK,KACvD0R,EAAEhR,OAAM,iBACH/F,KAAK+b,IAGVC,IACAC,EACK/U,M
 AAMiH,GACNwV,OAAQ/kB,EAAGG,MAAM2U,WAAWlP,EAAgB,GAAItB,IAChDwZ,UAAWnY,EAAgB,GAEhCwS,EAAEhR,OAAM,iBACH/F,KAAKic,IAmBdlF,EAAEhR,OAAM,uBACHxF,GAAE,QAAU,WACTyH,EAAMtB,EAAItH,GAAGiE,MAAM1D,MAAM,GACzBqI,EAAMpG,EAAI0G,KAAK6E,MAAMoU,EAAG/b,OAAOwC,EAAMtB,IAGrCiK,EAAM3I,MAAQA,EAAMpG,EACpBtC,EAASoiB,YAAY/Q,GAErB6Q,MAGRJ,EAAM9hB,SAASiB,GAAE,eAAiB,SAASC,GACvCwH,EAAMpG,EAAIpB,EAAEwgB,WACZhZ,EAAMtB,EAAI6a,EAAGvZ,EAAMpG,GAGnB+O,EAAM3I,MAAQA,EAAMpG,EACpBtC,EAASoiB,YAAY/Q,GAErB6Q,MAGJsB,EAASxjB,SAASiB,GAAE,cAAgB,SAASwG,EAAEnF,GAC3CmF,EAAE0V,UAAY1V,EAAE0V,SAChB0F,GAAYpb,EAAE0V,SAEd9L,EAAMwR,SAAWA,EACjB7iB,EAASoiB,YAAY/Q,GACrBjP,EAAMqR,WAGV2P,EAAOpjB,SAASiB,GAAE,cAAgB,SAASqjB,GACvC,IAAK,GAAIxY,KAAOwY,GACZjT,EAAMvF,GAAOwY,EAASxY,EAC1B9L,GAASoiB,YAAY/Q,GACrBjP,EAAMqR,WAGVmQ,EAAiB5jB,SAASiB,GAAE,mBAAqB,SAASC,GACtD4gB,EAAMF,iBACN,IAAI2C,GAAa7C,EAAY8C,EAAgBC,IAsB7C,IApBA7gB,EACKmI,OAAO,SAASL,EAAQpJ,GAErB,MADAoJ,GAAOqY,YAAczhB,GACboJ,EAAOyR,WAElB5T,QAAQ,SAASmC,EAAOpJ,GACrBof,EAAapiB,EAAG4I,kBAAkBwD,EAAOvD,OAA
 QjH,EAAE0E,YAAaxD,EAAMgF,KACtE0a,EAAML,eAAenf,EAAGof,GAAY,EACpC,IAAI9S,GAAQlD,EAAOvD,OAAOuZ,EACL,oBAAV9S,KACgB,mBAAhB2V,KAA6BA,EAAc3V,GACxB,mBAAnB4V,KAAgCA,EAAiBpiB,EAAMsD,SAAStD,EAAMgF,IAAIwH,EAAM8S,KAC3F+C,EAAQ1hB,MACJ+I,IAAKJ,EAAOI,IACZR,MAAOlJ,EAAMyM,IAAID,EAAO8S,GACxB9V,MAAOA,EAAMF,EAAOA,EAAOqY,kBAKnCU,EAAQ/iB,OAAS,EAAG,CACpB,GAAIgjB,GAAStiB,EAAMoY,SAAStU,OAAOhF,EAAE+C,QACjC0gB,EAAe3b,KAAKC,IAAI7G,EAAMoY,SAASvU,SAAS,GAAK7D,EAAMoY,SAASvU,SAAS,IAC7EkD,EAAY,IAAOwb,EACnBrb,EAAmBhK,EAAG4J,kBAAkBub,EAAQ1Q,IAAI,SAAStM,GAAG,MAAOA,GAAE6D,QAAQoZ,EAAOvb,EACnE,QAArBG,IACAmb,EAAQnb,GAAkBqC,WAAY,GAG9C,GAAIiZ,GAASnI,EAAM3E,aAAa1V,EAAMgF,IAAImd,EAAY7C,GAAaA,EACnEkC,GAAiBpkB,QACZuL,eAAe,SAAStD,EAAEnF,GACvB,MAAOqa,GAAM7E,aAAarQ,KAE7B7D,MAEG0H,MAAOsZ,EACPlZ,OAAQ+Y,MAIhBb,EAAiBre,gBAAgBif,KAGrCZ,EAAiB5jB,SAASiB,GAAE,kBAAmB,SAASC,GACpD4gB,EAAMF,oBAIV5hB,EAASiB,GAAE,cAAgB,SAASC,GACN,mBAAfA,GAAEic,WACTvZ,EAAK2F,QAAQ,SAASmC,EAAOpJ,GACzBoJ,EAAOyR,SAAWjc,EAAEic,SAAS7a,KAGjC+O,EAAM8L,SAAWjc,EAAEic,UAGA,m
 BAAZjc,GAAEwH,QACTA,EAAMpG,EAAIpB,EAAEwH,MACZA,EAAMtB,EAAI6a,EAAGvZ,EAAMpG,GAEnB+O,EAAM3I,MAAQxH,EAAEwH,MAEhB2Z,GACKze,MAAM8E,KAGW,mBAAfxH,GAAE2hB,WACTA,EAAW3hB,EAAE2hB,UAGjBzgB,EAAMqR,aAKd1B,EAAYS,UAAS,iCAEdpQ,EA2BX,QAAS6gB,GAAS4B,EAAKjhB,GAEnB,MADKkhB,KAAiBA,EAAkBhD,EAAMjT,KACvCjL,EAAKmQ,IAAI,SAASzM,EAAMhF,GAC3B,IAAKgF,EAAKa,OACN,MAAOb,EAEX,IAAIyd,GAAazd,EAAKa,OAAO0c,EAC7B,IAAkB,MAAdE,EACA,MAAOzd,EAEX,IAAIkB,GAAIsc,EAAgBC,EAAYF,EAGpC,QAAQ,IAAJrc,IAAawc,GAGb1d,EAAKoc,cAAe,EACbpc,IAGXA,EAAKoc,cAAe,EAEpBpc,EAAKa,OAASb,EAAKa,OAAO4L,IAAI,SAASnF,EAAO8S,GAE1C,MADA9S,GAAMiV,SAAUhV,GAAOiW,EAAgBlW,EAAO8S,GAAclZ,IAAM,EAAIA,IAC/DoG,IAGJtH,KA5jBf,GAqBMF,GACAyH,EAtBFiT,EAAQxiB,EAAGI,OAAO4H,OAChBmV,EAAQnd,EAAGI,OAAO8X,OAClBmF,EAAQrd,EAAGI,OAAO8X,OAClB4L,EAAS9jB,EAAGI,OAAO0jB,SACnBI,EAAWlkB,EAAGI,OAAO0jB,SACrBQ,EAAmBtkB,EAAGkE,uBACtBhE,EAAUF,EAAGI,OAAOF,UAGtBqF,GAAUE,IAAK,GAAIqR,MAAO,GAAID,OAAQ,GAAIrR,KAAM,IAC9Cue,EAAY,KACZzX,EAAQtM,EAAGG,MAAMuQ,eACjBtJ,EAAQ,KACRC,EAAS,KACTwc,GAAa,EACb3G,GAAY,EAC
 ZE,GAAY,EACZM,GAAkB,EAClBsG,GAAe,EACfK,GAA0B,EAC1Bd,GAAW,EAGXhZ,EAAKiY,EAAMjY,KACXwH,EAAQ/R,EAAGG,MAAM4R,QACjBoR,EAAe,KACfpM,EAAS,KACT4N,EAAU,SAASxc,GAAK,MAAOA,GAAEwc,SACjCjkB,EAAWF,GAAGE,SAAQ,cAAgB,cAAe,aAErD+H,EAAW,IACXid,GAAe,CAGrB3T,GAAM3I,MAAQ,EACd2I,EAAMwR,SAAWA,EAEjBpG,EAAM9E,OAAM,UAAWW,YAAY,GACnCqE,EAAMhF,OAAM,EAAqB,QAAU,QAE3CnY,EAAQuL,eAAe,SAAStD,EAAGnF,GAC/B,MAAOqa,GAAM7E,aAAarQ,EAAGnF,KAC9B0I,gBAAgB,SAASvD,EAAGnF,GAC3B,MAAOma,GAAM3E,aAAarQ,EAAGnF,KAGjCkhB,EAASyB,aAAY,EAMrB,IAAIhD,GAAKniB,GAAG8H,MAAMC,SACZa,GAASpG,EAAG,EAAG8E,EAAG,GAClB2K,EAAczS,EAAGG,MAAMsS,YAAY/R,EAAU+H,GAG/Cya,EAAc,SAAS5e,GACvB,MAAO,YACH,OACI3B,OAAQ2B,EAAKmQ,IAAI,SAAStM,GAAK,OAAQA,EAAE0V,WACzCzU,MAAOA,EAAMpG,EACbugB,SAAUA,KAKlBN,EAAc,SAAS3e,GACvB,MAAO,UAASyN,GACQlO,SAAhBkO,EAAM3I,QACNA,EAAMpG,EAAI+O,EAAM3I,OACGvF,SAAnBkO,EAAMwR,WACNA,EAAWxR,EAAMwR,UACA1f,SAAjBkO,EAAMpP,QACN2B,EAAK2F,QAAQ,SAASmC,EAAOpJ,GACzBoJ,EAAOyR,UAAY9L,EAAMpP,OAAOK,MAkchDwf,GAAM9hB,SAASiB,GAAE,2BAA6B,SAASoc,GACnD,GAAIzO,IACAxH,EAAGh
 F,EAAMgF,IAAIiW,EAAIzO,OACjBC,EAAGzM,EAAMyM,IAAIwO,EAAIzO,OACjBhD,MAAOyR,EAAIzO,MAAMhD,MAErByR,GAAIzO,MAAQA,EACZpP,EAAQoE,KAAKyZ,GAAK7X,QAAO,KAG7Bsc,EAAM9hB,SAASiB,GAAE,0BAA4B,SAASoc,GAClD7d,EAAQgG,QAAO,IAOnB,IAAIsf,GAAkB,IAqGtB,OA/DA1iB,GAAMpC,SAAWA,EACjBoC,EAAM0f,MAAQA,EACd1f,EAAMghB,OAASA,EACfhhB,EAAMohB,SAAWA,EACjBphB,EAAMqa,MAAQA,EACdra,EAAMua,MAAQA,EACdva,EAAMwhB,iBAAmBA,EACzBxhB,EAAMiP,MAAQA,EACdjP,EAAM5C,QAAUA,EAEhB4C,EAAM8L,QAAU5O,EAAGG,MAAM0O,YAAYhO,KAAKiC,GAE1CA,EAAMgM,SAAWC,OAAOC,WAEpB5H,OAAa6H,IAAK,WAAW,MAAO7H,IAAS8H,IAAK,SAASvG,GAAGvB,EAAMuB,IACpEtB,QAAa4H,IAAK,WAAW,MAAO5H,IAAU6H,IAAK,SAASvG,GAAGtB,EAAOsB,IACtE4a,UAAetU,IAAK,WAAW,MAAOsU,IAAYrU,IAAK,SAASvG,GAAG4a,EAAS5a,IAC5Eqb,cAAmB/U,IAAK,WAAW,MAAO+U,IAAgB9U,IAAK,SAASvG,GAAGqb,EAAarb,IACxFkb,YAAa5U,IAAK,WAAW,MAAO4U,IAAc3U,IAAK,SAASvG,GAAGkb,EAAWlb,IAC9Egc,SAAU1V,IAAK,WAAW,MAAO0V,IAAWzV,IAAK,SAASvG,GAAGgc,EAAQhc,IACrEwa,cAAkBlU,IAAK,WAAW,MAAOkU,IAAgBjU,IAAK,SAASvG,GAAGwa,EAAaxa,IACvFoO,QAAY9H,IAAK,WAAW,MAAO8H,IAAU7H,IA
 AK,SAASvG,GAAGoO,EAAOpO,IACrEuU,WAAejO,IAAK,WAAW,MAAOiO,IAAahO,IAAK,SAASvG,GAAGuU,EAAUvU,IAC9EyU,WAAenO,IAAK,WAAW,MAAOmO,IAAalO,IAAK,SAASvG,GAAGyU,EAAUzU,IAC9E+c,cAAkBzW,IAAK,WAAW,MAAOyW,IAAgBxW,IAAK,SAASvG,GAAG+c,EAAa/c,IAGvFpD,QAAS0J,IAAK,WAAW,MAAO1J,IAAU2J,IAAK,SAASvG,GACtC9E,SAAV8E,EAAElD,MACFF,EAAOE,IAAMkD,EAAElD,IACfse,EAAYpb,EAAElD,KAElBF,EAAOuR,MAAsBjT,SAAb8E,EAAEmO,MAAuBnO,EAAEmO,MAASvR,EAAOuR,MAC3DvR,EAAOsR,OAAsBhT,SAAb8E,EAAEkO,OAAuBlO,EAAEkO,OAAStR,EAAOsR,OAC3DtR,EAAOC,KAAsB3B,SAAb8E,EAAEnD,KAAuBmD,EAAEnD,KAASD,EAAOC,OAE/D8G,OAAS2C,IAAK,WAAW,MAAO3C,IAAS4C,IAAK,SAASvG,GACnD2D,EAAQtM,EAAGG,MAAMsQ,SAAS9H,GAC1Bmb,EAAOxX,MAAMA,KAEjB+X,yBAA0BpV,IAAK,WAAW,MAAOoV,IAA2BnV,IAAK,SAASvG,GACtF0b,EAA0B1b,EACtBA,KAAM,IACN7F,EAAMof,aAAY,GAClBpf,EAAM8iB,YAAW,MAGzBlI,iBAAkBzO,IAAK,WAAW,MAAOyO,IAAmBxO,IAAK,SAASvG,GACtE+U,EAAkB/U,EAClB0U,EAAMhF,OAAQ,EAAM,QAAU,UAElC5P,UAAcwG,IAAK,WAAW,MAAOxG,IAAYyG,IAAK,SAASvG,GAC3DF,EAAWE,EACX6Z,EAAM/Z,SAASA,GACf0U,EAAM1U,SAASA,GACf4U,EAAM5U,SAASA,GACfgK,EA
 AYW,MAAM3K,OAI1BzI,EAAGG,MAAMkW,eAAevT,EAAO0f,GAC/BxiB,EAAGG,MAAMqP,YAAY1M,GAEdA,GC3oBX9C,EAAGI,OAAOylB,YAAc,WACpB,YAmCA,SAAS/iB,GAAMsB,GAyKX,MAxKAqO,GAAYW,QACZhP,EAAUC,KAAK,SAASC,GACpB,GAAIqB,GAAiByB,EAAQ7B,EAAOC,KAAOD,EAAOuR,MAC9ClR,EAAkByB,EAAS9B,EAAOE,IAAMF,EAAOsR,MAEnD3P,GAAY1G,GAAG2G,OAAOpG,MACtBf,EAAGG,MAAMsW,QAAQvP,GAGjB5C,EAAK2F,QAAQ,SAASmC,EAAQpJ,GAC1BoJ,EAAOvD,OAAOoB,QAAQ,SAASqF,GAC3BA,EAAMlD,OAASpJ,KAMvB,IAAI8iB,GAAc9L,GAAWI,KACzB9V,EAAKmQ,IAAI,SAAStM,GACd,MAAOA,GAAEU,OAAO4L,IAAI,SAAStM,EAAEnF,GAC3B,OAAS8E,EAAGmS,EAAK9R,EAAEnF,GAAIuM,EAAGsS,EAAK1Z,EAAEnF,GAAI+iB,GAAI5d,EAAE4d,OAIvDje,GAAKnB,OAAOqT,GAAWxZ,GAAGmf,MAAMmG,GAAYrR,IAAI,SAAStM,GAAK,MAAOA,GAAEL,KAClEzB,WAAW6T,IAAW,EAAGvU,GAAiB,IAC/C4J,EAAK5I,OAAOyT,GAAW5Z,GAAGkf,OAAOlf,GAAGmf,MAAMmG,GAAYrR,IAAI,SAAStM,GAAK,MAAOA,GAAEoH,IAAK7N,OAAO4f,KAGzF0E,EAAYzW,EAAE9I,MAAM0U,IAAWvV,GAAmB2J,EAAE5I,SAAS,GAAK,EAAI,GAAK,GAAI4I,EAAE5I,SAAS,GAAK,EAAI,GAAK,IACvG4I,EAAE9I,MAAM0U,IAAWvV,EAAiB,IAGzC6a,EAAKA,GAAM3Y,EACXie,EAAKA,GAAMxW,E
 AAEuK,OAAOrT,OAAO8I,EAAE,GAAGA,EAAE,IAGlC,IAAIjI,GAAOJ,EAAUK,UAAS,4BAA6BjD,MAAMA,IAC7DkD,EAAYF,EAAKG,QAAQC,OAAM,KAAMC,KAAI,QAAU,+BACnD6V,EAAShW,EAAUE,OAAM,IACrBJ,GAAKH,OAAM,IAEnBqW,GAAO9V,OAAM,KAAMC,KAAI,QAAU,aACjCL,EAAKK,KAAI,YAAc,aAAepC,EAAOC,KAAO,IAAMD,EAAOE,IAAM,IAGvE,IAAIwgB,GAAS3e,EAAKH,OAAM,cAAeI,UAAS,aAC3CjD,KAAK,SAAS6D,GAAK,MAAOA,IAAK,SAASA,GAAK,MAAOA,GAAEqE,KAC3DyZ,GAAOxe,QAAQC,OAAM,KAChB+C,MAAK,iBAAmB,MACxBA,MAAK,eAAiB,MAC3Bwb,EAAO7d,OACFoK,gBAAgBC,EAAa,4BAC7BhI,MAAK,iBAAmB,MACxBA,MAAK,eAAiB,MACtBpC,SACL4d,EACKte,KAAI,QAAU,SAASQ,EAAEnF,GAAK,MAAO,sBAAwBA,IAC7D0H,QAAO,QAAU,SAASvC,GAAK,MAAOA,GAAEsT,QAC7CwK,EACKzT,gBAAgBC,EAAa,uBAC7BhI,MAAK,iBAAmB,GACxBA,MAAK,eAAiB,IAE3B,IAAIyb,GAAOD,EAAO1e,UAAS,YACtBjD,KAAK,SAAS6D,GAAK,MAAOA,GAAEU,QACjCqd,GAAK9d,OAAOC,QAEZ,IAAI8d,GAAYD,EAAKze,QAAQC,OAAM,KAC9BC,KAAI,YAAc,SAASQ,EAAEnF,EAAEwY,GAC5B,MAAO,cAAgB1T,EAAEmS,EAAK9R,EAAEnF,IAAsB,IAAhB8E,EAAEpB,aAAsB,KAAO6I,EAAE,GAAK,MAE/E5N,GAAE,YAAc,SAASwG,EAAEnF,GACxBxC,GAAG2G,OAAOpG,MAAM2J,QAAO,SAAU
 ,GACjChK,EAASsb,kBACL1X,KAAM6D,EACNiB,MAAOpG,EACPsJ,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAGnC9I,GAAE,WAAa,SAASwG,EAAEnF,GACvBxC,GAAG2G,OAAOpG,MAAM2J,QAAO,SAAU,GACjChK,EAASsF,iBACL1B,KAAM6D,EACNiB,MAAOpG,EACPsJ,MAAO9L

<TRUNCATED>

---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[38/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
deleted file mode 100644
index bf301b8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.scm.protocolPB;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolTranslator;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.scm.ScmInfo;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.protocol.StorageContainerLocationProtocol;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ContainerResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.DeleteContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ListContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ListContainerResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.NodeQueryRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.NodeQueryResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.PipelineRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.PipelineResponseProto;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-
-/**
- * This class is the client-side translator to translate the requests made on
- * the {@link StorageContainerLocationProtocol} interface to the RPC server
- * implementing {@link StorageContainerLocationProtocolPB}.
- */
-@InterfaceAudience.Private
-public final class StorageContainerLocationProtocolClientSideTranslatorPB
-    implements StorageContainerLocationProtocol, ProtocolTranslator, Closeable {
-
-  /**
-   * RpcController is not used and hence is set to null.
-   */
-  private static final RpcController NULL_RPC_CONTROLLER = null;
-
-  private final StorageContainerLocationProtocolPB rpcProxy;
-
-  /**
-   * Creates a new StorageContainerLocationProtocolClientSideTranslatorPB.
-   *
-   * @param rpcProxy {@link StorageContainerLocationProtocolPB} RPC proxy
-   */
-  public StorageContainerLocationProtocolClientSideTranslatorPB(
-      StorageContainerLocationProtocolPB rpcProxy) {
-    this.rpcProxy = rpcProxy;
-  }
-
-  /**
-   * Asks SCM where a container should be allocated. SCM responds with the set
-   * of datanodes that should be used creating this container. Ozone/SCM only
-   * supports replication factor of either 1 or 3.
-   * @param type - Replication Type
-   * @param factor - Replication Count
-   * @param containerName - Name
-   * @return
-   * @throws IOException
-   */
-  @Override
-  public Pipeline allocateContainer(OzoneProtos.ReplicationType type,
-      OzoneProtos.ReplicationFactor factor, String
-      containerName, String owner) throws IOException {
-
-    Preconditions.checkNotNull(containerName, "Container Name cannot be Null");
-    Preconditions.checkState(!containerName.isEmpty(), "Container name cannot" +
-        " be empty");
-    ContainerRequestProto request = ContainerRequestProto.newBuilder()
-        .setContainerName(containerName)
-        .setReplicationFactor(factor)
-        .setReplicationType(type)
-        .setOwner(owner)
-        .build();
-
-    final ContainerResponseProto response;
-    try {
-      response = rpcProxy.allocateContainer(NULL_RPC_CONTROLLER, request);
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    if (response.getErrorCode() != ContainerResponseProto.Error.success) {
-      throw new IOException(response.hasErrorMessage() ?
-          response.getErrorMessage() : "Allocate container failed.");
-    }
-    return Pipeline.getFromProtoBuf(response.getPipeline());
-  }
-
-  public Pipeline getContainer(String containerName) throws IOException {
-    Preconditions.checkNotNull(containerName,
-        "Container Name cannot be Null");
-    Preconditions.checkState(!containerName.isEmpty(),
-        "Container name cannot be empty");
-    GetContainerRequestProto request = GetContainerRequestProto
-        .newBuilder()
-        .setContainerName(containerName)
-        .build();
-    try {
-      GetContainerResponseProto response =
-          rpcProxy.getContainer(NULL_RPC_CONTROLLER, request);
-      return Pipeline.getFromProtoBuf(response.getPipeline());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public List<ContainerInfo> listContainer(String startName, String prefixName,
-      int count) throws IOException {
-    ListContainerRequestProto.Builder builder = ListContainerRequestProto
-        .newBuilder();
-    if (prefixName != null) {
-      builder.setPrefixName(prefixName);
-    }
-    if (startName != null) {
-      builder.setStartName(startName);
-    }
-    builder.setCount(count);
-    ListContainerRequestProto request = builder.build();
-
-    try {
-      ListContainerResponseProto response =
-          rpcProxy.listContainer(NULL_RPC_CONTROLLER, request);
-      List<ContainerInfo> containerList = new ArrayList<>();
-      for (OzoneProtos.SCMContainerInfo containerInfoProto : response
-          .getContainersList()) {
-        containerList.add(ContainerInfo.fromProtobuf(containerInfoProto));
-      }
-      return containerList;
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-  }
-
-  /**
-   * Ask SCM to delete a container by name. SCM will remove
-   * the container mapping in its database.
-   *
-   * @param containerName
-   * @throws IOException
-   */
-  @Override
-  public void deleteContainer(String containerName)
-      throws IOException {
-    Preconditions.checkState(!Strings.isNullOrEmpty(containerName),
-        "Container name cannot be null or empty");
-    DeleteContainerRequestProto request = DeleteContainerRequestProto
-        .newBuilder()
-        .setContainerName(containerName)
-        .build();
-    try {
-      rpcProxy.deleteContainer(NULL_RPC_CONTROLLER, request);
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-  }
-
-  /**
-   * Queries a list of Node Statuses.
-   *
-   * @param nodeStatuses
-   * @return List of Datanodes.
-   */
-  @Override
-  public OzoneProtos.NodePool queryNode(EnumSet<OzoneProtos.NodeState>
-      nodeStatuses, OzoneProtos.QueryScope queryScope, String poolName)
-      throws IOException {
-    // TODO : We support only cluster wide query right now. So ignoring checking
-    // queryScope and poolName
-    Preconditions.checkNotNull(nodeStatuses);
-    Preconditions.checkState(nodeStatuses.size() > 0);
-    NodeQueryRequestProto request = NodeQueryRequestProto.newBuilder()
-        .addAllQuery(nodeStatuses)
-        .setScope(queryScope).setPoolName(poolName).build();
-    try {
-      NodeQueryResponseProto response =
-          rpcProxy.queryNode(NULL_RPC_CONTROLLER, request);
-      return response.getDatanodes();
-    } catch (ServiceException e) {
-      throw  ProtobufHelper.getRemoteException(e);
-    }
-
-  }
-
-  /**
-   * Notify from client that creates object on datanodes.
-   * @param type object type
-   * @param name object name
-   * @param op operation type (e.g., create, close, delete)
-   * @param stage object creation stage : begin/complete
-   */
-  @Override
-  public void notifyObjectStageChange(
-      ObjectStageChangeRequestProto.Type type, String name,
-      ObjectStageChangeRequestProto.Op op,
-      ObjectStageChangeRequestProto.Stage stage) throws IOException {
-    Preconditions.checkState(!Strings.isNullOrEmpty(name),
-        "Object name cannot be null or empty");
-    ObjectStageChangeRequestProto request =
-        ObjectStageChangeRequestProto.newBuilder()
-            .setType(type)
-            .setName(name)
-            .setOp(op)
-            .setStage(stage)
-            .build();
-    try {
-      rpcProxy.notifyObjectStageChange(NULL_RPC_CONTROLLER, request);
-    } catch(ServiceException e){
-      throw ProtobufHelper.getRemoteException(e);
-    }
-  }
-
-  /**
-   * Creates a replication pipeline of a specified type.
-   *
-   * @param replicationType - replication type
-   * @param factor - factor 1 or 3
-   * @param nodePool - optional machine list to build a pipeline.
-   * @throws IOException
-   */
-  @Override
-  public Pipeline createReplicationPipeline(OzoneProtos.ReplicationType
-      replicationType, OzoneProtos.ReplicationFactor factor, OzoneProtos
-      .NodePool nodePool) throws IOException {
-    PipelineRequestProto request = PipelineRequestProto.newBuilder()
-        .setNodePool(nodePool)
-        .setReplicationFactor(factor)
-        .setReplicationType(replicationType)
-        .build();
-    try {
-      PipelineResponseProto response =
-          rpcProxy.allocatePipeline(NULL_RPC_CONTROLLER, request);
-      if (response.getErrorCode() ==
-          PipelineResponseProto.Error.success) {
-        Preconditions.checkState(response.hasPipeline(), "With success, " +
-            "must come a pipeline");
-        return Pipeline.getFromProtoBuf(response.getPipeline());
-      } else {
-        String errorMessage = String.format("create replication pipeline " +
-                "failed. code : %s Message: %s", response.getErrorCode(),
-            response.hasErrorMessage() ? response.getErrorMessage() : "");
-        throw new IOException(errorMessage);
-      }
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-  }
-
-  @Override
-  public ScmInfo getScmInfo() throws IOException {
-    OzoneProtos.GetScmInfoRequestProto request =
-        OzoneProtos.GetScmInfoRequestProto.getDefaultInstance();
-    try {
-      OzoneProtos.GetScmInfoRespsonseProto resp = rpcProxy.getScmInfo(
-          NULL_RPC_CONTROLLER, request);
-      ScmInfo.Builder builder = new ScmInfo.Builder()
-          .setClusterId(resp.getClusterId())
-          .setScmId(resp.getScmId());
-      return builder.build();
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-
-  }
-
-  @Override
-  public Object getUnderlyingProxyObject() {
-    return rpcProxy;
-  }
-
-  @Override
-  public void close() {
-    RPC.stopProxy(rpcProxy);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolPB.java
deleted file mode 100644
index 9ee1307..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolPB.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.protocolPB;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.StorageContainerLocationProtocolService;
-
-/**
- * Protocol used from an HDFS node to StorageContainerManager.  This extends the
- * Protocol Buffers service interface to add Hadoop-specific annotations.
- */
-@ProtocolInfo(protocolName =
-    "org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol",
-    protocolVersion = 1)
-@InterfaceAudience.Private
-public interface StorageContainerLocationProtocolPB
-    extends StorageContainerLocationProtocolService.BlockingInterface {
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/package-info.java
deleted file mode 100644
index f9a2c09..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.protocolPB;
-
-/**
- * This package contains classes for the client of the storage container
- * protocol.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkInputStream.java
deleted file mode 100644
index 943be2a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkInputStream.java
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.scm.storage;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.List;
-
-import com.google.protobuf.ByteString;
-
-import org.apache.hadoop.fs.Seekable;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ReadChunkResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ChunkInfo;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.XceiverClientManager;
-
-/**
- * An {@link InputStream} used by the REST service in combination with the
- * SCMClient to read the value of a key from a sequence
- * of container chunks.  All bytes of the key value are stored in container
- * chunks.  Each chunk may contain multiple underlying {@link ByteBuffer}
- * instances.  This class encapsulates all state management for iterating
- * through the sequence of chunks and the sequence of buffers within each chunk.
- */
-public class ChunkInputStream extends InputStream implements Seekable {
-
-  private static final int EOF = -1;
-
-  private final String key;
-  private final String traceID;
-  private XceiverClientManager xceiverClientManager;
-  private XceiverClientSpi xceiverClient;
-  private List<ChunkInfo> chunks;
-  private int chunkIndex;
-  private long[] chunkOffset;
-  private List<ByteBuffer> buffers;
-  private int bufferIndex;
-
-  /**
-   * Creates a new ChunkInputStream.
-   *
-   * @param key chunk key
-   * @param xceiverClientManager client manager that controls client
-   * @param xceiverClient client to perform container calls
-   * @param chunks list of chunks to read
-   * @param traceID container protocol call traceID
-   */
-  public ChunkInputStream(String key, XceiverClientManager xceiverClientManager,
-      XceiverClientSpi xceiverClient, List<ChunkInfo> chunks, String traceID) {
-    this.key = key;
-    this.traceID = traceID;
-    this.xceiverClientManager = xceiverClientManager;
-    this.xceiverClient = xceiverClient;
-    this.chunks = chunks;
-    this.chunkIndex = -1;
-    // chunkOffset[i] stores offset at which chunk i stores data in
-    // ChunkInputStream
-    this.chunkOffset = new long[this.chunks.size()];
-    initializeChunkOffset();
-    this.buffers = null;
-    this.bufferIndex = 0;
-  }
-
-  private void initializeChunkOffset() {
-    int tempOffset = 0;
-    for (int i = 0; i < chunks.size(); i++) {
-      chunkOffset[i] = tempOffset;
-      tempOffset += chunks.get(i).getLen();
-    }
-  }
-
-  @Override
-  public synchronized int read()
-      throws IOException {
-    checkOpen();
-    int available = prepareRead(1);
-    return available == EOF ? EOF :
-        Byte.toUnsignedInt(buffers.get(bufferIndex).get());
-  }
-
-  @Override
-  public synchronized int read(byte[] b, int off, int len) throws IOException {
-    // According to the JavaDocs for InputStream, it is recommended that
-    // subclasses provide an override of bulk read if possible for performance
-    // reasons.  In addition to performance, we need to do it for correctness
-    // reasons.  The Ozone REST service uses PipedInputStream and
-    // PipedOutputStream to relay HTTP response data between a Jersey thread and
-    // a Netty thread.  It turns out that PipedInputStream/PipedOutputStream
-    // have a subtle dependency (bug?) on the wrapped stream providing separate
-    // implementations of single-byte read and bulk read.  Without this, get key
-    // responses might close the connection before writing all of the bytes
-    // advertised in the Content-Length.
-    if (b == null) {
-      throw new NullPointerException();
-    }
-    if (off < 0 || len < 0 || len > b.length - off) {
-      throw new IndexOutOfBoundsException();
-    }
-    if (len == 0) {
-      return 0;
-    }
-    checkOpen();
-    int available = prepareRead(len);
-    if (available == EOF) {
-      return EOF;
-    }
-    buffers.get(bufferIndex).get(b, off, available);
-    return available;
-  }
-
-  @Override
-  public synchronized void close() {
-    if (xceiverClientManager != null && xceiverClient != null) {
-      xceiverClientManager.releaseClient(xceiverClient);
-      xceiverClientManager = null;
-      xceiverClient = null;
-    }
-  }
-
-  /**
-   * Checks if the stream is open.  If not, throws an exception.
-   *
-   * @throws IOException if stream is closed
-   */
-  private synchronized void checkOpen() throws IOException {
-    if (xceiverClient == null) {
-      throw new IOException("ChunkInputStream has been closed.");
-    }
-  }
-
-  /**
-   * Prepares to read by advancing through chunks and buffers as needed until it
-   * finds data to return or encounters EOF.
-   *
-   * @param len desired length of data to read
-   * @return length of data available to read, possibly less than desired length
-   */
-  private synchronized int prepareRead(int len) throws IOException {
-    for (;;) {
-      if (chunks == null || chunks.isEmpty()) {
-        // This must be an empty key.
-        return EOF;
-      } else if (buffers == null) {
-        // The first read triggers fetching the first chunk.
-        readChunkFromContainer();
-      } else if (!buffers.isEmpty() &&
-          buffers.get(bufferIndex).hasRemaining()) {
-        // Data is available from the current buffer.
-        ByteBuffer bb = buffers.get(bufferIndex);
-        return len > bb.remaining() ? bb.remaining() : len;
-      } else if (!buffers.isEmpty() &&
-          !buffers.get(bufferIndex).hasRemaining() &&
-          bufferIndex < buffers.size() - 1) {
-        // There are additional buffers available.
-        ++bufferIndex;
-      } else if (chunkIndex < chunks.size() - 1) {
-        // There are additional chunks available.
-        readChunkFromContainer();
-      } else {
-        // All available input has been consumed.
-        return EOF;
-      }
-    }
-  }
-
-  /**
-   * Attempts to read the chunk at the specified offset in the chunk list.  If
-   * successful, then the data of the read chunk is saved so that its bytes can
-   * be returned from subsequent read calls.
-   *
-   * @throws IOException if there is an I/O error while performing the call
-   */
-  private synchronized void readChunkFromContainer() throws IOException {
-    // On every chunk read chunkIndex should be increased so as to read the
-    // next chunk
-    chunkIndex += 1;
-    final ReadChunkResponseProto readChunkResponse;
-    try {
-      readChunkResponse = ContainerProtocolCalls.readChunk(xceiverClient,
-          chunks.get(chunkIndex), key, traceID);
-    } catch (IOException e) {
-      throw new IOException("Unexpected OzoneException: " + e.toString(), e);
-    }
-    ByteString byteString = readChunkResponse.getData();
-    buffers = byteString.asReadOnlyByteBufferList();
-    bufferIndex = 0;
-  }
-
-  @Override
-  public synchronized void seek(long pos) throws IOException {
-    if (pos < 0 || (chunks.size() == 0 && pos > 0)
-        || pos >= chunkOffset[chunks.size() - 1] + chunks.get(chunks.size() - 1)
-        .getLen()) {
-      throw new EOFException(
-          "EOF encountered pos: " + pos + " container key: " + key);
-    }
-    if (chunkIndex == -1) {
-      chunkIndex = Arrays.binarySearch(chunkOffset, pos);
-    } else if (pos < chunkOffset[chunkIndex]) {
-      chunkIndex = Arrays.binarySearch(chunkOffset, 0, chunkIndex, pos);
-    } else if (pos >= chunkOffset[chunkIndex] + chunks.get(chunkIndex)
-        .getLen()) {
-      chunkIndex =
-          Arrays.binarySearch(chunkOffset, chunkIndex + 1, chunks.size(), pos);
-    }
-    if (chunkIndex < 0) {
-      // Binary search returns -insertionPoint - 1  if element is not present
-      // in the array. insertionPoint is the point at which element would be
-      // inserted in the sorted array. We need to adjust the chunkIndex
-      // accordingly so that chunkIndex = insertionPoint - 1
-      chunkIndex = -chunkIndex -2;
-    }
-    // adjust chunkIndex so that readChunkFromContainer reads the correct chunk
-    chunkIndex -= 1;
-    readChunkFromContainer();
-    adjustBufferIndex(pos);
-  }
-
-  private void adjustBufferIndex(long pos) {
-    long tempOffest = chunkOffset[chunkIndex];
-    for (int i = 0; i < buffers.size(); i++) {
-      if (pos - tempOffest >= buffers.get(i).capacity()) {
-        tempOffest += buffers.get(i).capacity();
-      } else {
-        bufferIndex = i;
-        break;
-      }
-    }
-    buffers.get(bufferIndex).position((int) (pos - tempOffest));
-  }
-
-  @Override
-  public synchronized long getPos() throws IOException {
-    return chunkIndex == -1 ? 0 :
-        chunkOffset[chunkIndex] + buffers.get(bufferIndex).position();
-  }
-
-  @Override
-  public boolean seekToNewSource(long targetPos) throws IOException {
-    return false;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
deleted file mode 100644
index 64c10da..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.scm.storage;
-
-import static org.apache.hadoop.scm.storage.ContainerProtocolCalls.putKey;
-import static org.apache.hadoop.scm.storage.ContainerProtocolCalls.writeChunk;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.util.UUID;
-
-import com.google.protobuf.ByteString;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ChunkInfo;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyData;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.XceiverClientSpi;
-
-/**
- * An {@link OutputStream} used by the REST service in combination with the
- * SCMClient to write the value of a key to a sequence
- * of container chunks.  Writes are buffered locally and periodically written to
- * the container as a new chunk.  In order to preserve the semantics that
- * replacement of a pre-existing key is atomic, each instance of the stream has
- * an internal unique identifier.  This unique identifier and a monotonically
- * increasing chunk index form a composite key that is used as the chunk name.
- * After all data is written, a putKey call creates or updates the corresponding
- * container key, and this call includes the full list of chunks that make up
- * the key data.  The list of chunks is updated all at once.  Therefore, a
- * concurrent reader never can see an intermediate state in which different
- * chunks of data from different versions of the key data are interleaved.
- * This class encapsulates all state management for buffering and writing
- * through to the container.
- */
-public class ChunkOutputStream extends OutputStream {
-
-  private final String containerKey;
-  private final String key;
-  private final String traceID;
-  private final KeyData.Builder containerKeyData;
-  private XceiverClientManager xceiverClientManager;
-  private XceiverClientSpi xceiverClient;
-  private ByteBuffer buffer;
-  private final String streamId;
-  private int chunkIndex;
-  private int chunkSize;
-
-  /**
-   * Creates a new ChunkOutputStream.
-   *
-   * @param containerKey container key
-   * @param key chunk key
-   * @param xceiverClientManager client manager that controls client
-   * @param xceiverClient client to perform container calls
-   * @param traceID container protocol call args
-   * @param chunkSize chunk size
-   */
-  public ChunkOutputStream(String containerKey, String key,
-      XceiverClientManager xceiverClientManager, XceiverClientSpi xceiverClient,
-      String traceID, int chunkSize) {
-    this.containerKey = containerKey;
-    this.key = key;
-    this.traceID = traceID;
-    this.chunkSize = chunkSize;
-    KeyValue keyValue = KeyValue.newBuilder()
-        .setKey("TYPE").setValue("KEY").build();
-    this.containerKeyData = KeyData.newBuilder()
-        .setContainerName(xceiverClient.getPipeline().getContainerName())
-        .setName(containerKey)
-        .addMetadata(keyValue);
-    this.xceiverClientManager = xceiverClientManager;
-    this.xceiverClient = xceiverClient;
-    this.buffer = ByteBuffer.allocate(chunkSize);
-    this.streamId = UUID.randomUUID().toString();
-    this.chunkIndex = 0;
-  }
-
-  @Override
-  public synchronized void write(int b) throws IOException {
-    checkOpen();
-    int rollbackPosition = buffer.position();
-    int rollbackLimit = buffer.limit();
-    buffer.put((byte)b);
-    if (buffer.position() == chunkSize) {
-      flushBufferToChunk(rollbackPosition, rollbackLimit);
-    }
-  }
-
-  @Override
-  public void write(byte[] b, int off, int len) throws IOException {
-    if (b == null) {
-      throw new NullPointerException();
-    }
-    if ((off < 0) || (off > b.length) || (len < 0) ||
-        ((off + len) > b.length) || ((off + len) < 0)) {
-      throw new IndexOutOfBoundsException();
-    }
-    if (len == 0) {
-      return;
-    }
-    checkOpen();
-    while (len > 0) {
-      int writeLen = Math.min(chunkSize - buffer.position(), len);
-      int rollbackPosition = buffer.position();
-      int rollbackLimit = buffer.limit();
-      buffer.put(b, off, writeLen);
-      if (buffer.position() == chunkSize) {
-        flushBufferToChunk(rollbackPosition, rollbackLimit);
-      }
-      off += writeLen;
-      len -= writeLen;
-    }
-  }
-
-  @Override
-  public synchronized void flush() throws IOException {
-    checkOpen();
-    if (buffer.position() > 0) {
-      int rollbackPosition = buffer.position();
-      int rollbackLimit = buffer.limit();
-      flushBufferToChunk(rollbackPosition, rollbackLimit);
-    }
-  }
-
-  @Override
-  public synchronized void close() throws IOException {
-    if (xceiverClientManager != null && xceiverClient != null &&
-        buffer != null) {
-      try {
-        if (buffer.position() > 0) {
-          writeChunkToContainer();
-        }
-        putKey(xceiverClient, containerKeyData.build(), traceID);
-      } catch (IOException e) {
-        throw new IOException(
-            "Unexpected Storage Container Exception: " + e.toString(), e);
-      } finally {
-        xceiverClientManager.releaseClient(xceiverClient);
-        xceiverClientManager = null;
-        xceiverClient = null;
-        buffer = null;
-      }
-    }
-
-  }
-
-  /**
-   * Checks if the stream is open.  If not, throws an exception.
-   *
-   * @throws IOException if stream is closed
-   */
-  private synchronized void checkOpen() throws IOException {
-    if (xceiverClient == null) {
-      throw new IOException("ChunkOutputStream has been closed.");
-    }
-  }
-
-  /**
-   * Attempts to flush buffered writes by writing a new chunk to the container.
-   * If successful, then clears the buffer to prepare to receive writes for a
-   * new chunk.
-   *
-   * @param rollbackPosition position to restore in buffer if write fails
-   * @param rollbackLimit limit to restore in buffer if write fails
-   * @throws IOException if there is an I/O error while performing the call
-   */
-  private synchronized void flushBufferToChunk(int rollbackPosition,
-      int rollbackLimit) throws IOException {
-    boolean success = false;
-    try {
-      writeChunkToContainer();
-      success = true;
-    } finally {
-      if (success) {
-        buffer.clear();
-      } else {
-        buffer.position(rollbackPosition);
-        buffer.limit(rollbackLimit);
-      }
-    }
-  }
-
-  /**
-   * Writes buffered data as a new chunk to the container and saves chunk
-   * information to be used later in putKey call.
-   *
-   * @throws IOException if there is an I/O error while performing the call
-   */
-  private synchronized void writeChunkToContainer() throws IOException {
-    buffer.flip();
-    ByteString data = ByteString.copyFrom(buffer);
-    ChunkInfo chunk = ChunkInfo
-        .newBuilder()
-        .setChunkName(
-            DigestUtils.md5Hex(key) + "_stream_"
-                + streamId + "_chunk_" + ++chunkIndex)
-        .setOffset(0)
-        .setLen(data.size())
-        .build();
-    try {
-      writeChunk(xceiverClient, chunk, key, data, traceID);
-    } catch (IOException e) {
-      throw new IOException(
-          "Unexpected Storage Container Exception: " + e.toString(), e);
-    }
-    containerKeyData.addChunks(chunk);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ContainerProtocolCalls.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ContainerProtocolCalls.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ContainerProtocolCalls.java
deleted file mode 100644
index 1cde67c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ContainerProtocolCalls.java
+++ /dev/null
@@ -1,397 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.scm.storage;
-
-import com.google.protobuf.ByteString;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ChunkInfo;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .ContainerCommandRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .ContainerCommandResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .GetKeyRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .GetKeyResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .GetSmallFileRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .GetSmallFileResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyData;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .PutKeyRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .PutSmallFileRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .ReadChunkRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .ReadChunkResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Type;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .WriteChunkRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .ReadContainerResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .ReadContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-
-import java.io.IOException;
-import org.apache.hadoop.scm.XceiverClientSpi;
-
-/**
- * Implementation of all container protocol calls performed by Container
- * clients.
- */
-public final class ContainerProtocolCalls  {
-
-  /**
-   * There is no need to instantiate this class.
-   */
-  private ContainerProtocolCalls() {
-  }
-
-  /**
-   * Calls the container protocol to get a container key.
-   *
-   * @param xceiverClient client to perform call
-   * @param containerKeyData key data to identify container
-   * @param traceID container protocol call args
-   * @return container protocol get key response
-   * @throws IOException if there is an I/O error while performing the call
-   */
-  public static GetKeyResponseProto getKey(XceiverClientSpi xceiverClient,
-      KeyData containerKeyData, String traceID) throws IOException {
-    GetKeyRequestProto.Builder readKeyRequest = GetKeyRequestProto
-        .newBuilder()
-        .setPipeline(xceiverClient.getPipeline().getProtobufMessage())
-        .setKeyData(containerKeyData);
-    String id = xceiverClient.getPipeline().getLeader().getDatanodeUuid();
-    ContainerCommandRequestProto request = ContainerCommandRequestProto
-        .newBuilder()
-        .setCmdType(Type.GetKey)
-        .setTraceID(traceID)
-        .setDatanodeID(id)
-        .setGetKey(readKeyRequest)
-        .build();
-    ContainerCommandResponseProto response = xceiverClient.sendCommand(request);
-    validateContainerResponse(response);
-    return response.getGetKey();
-  }
-
-  /**
-   * Calls the container protocol to put a container key.
-   *
-   * @param xceiverClient client to perform call
-   * @param containerKeyData key data to identify container
-   * @param traceID container protocol call args
-   * @throws IOException if there is an I/O error while performing the call
-   */
-  public static void putKey(XceiverClientSpi xceiverClient,
-      KeyData containerKeyData, String traceID) throws IOException {
-    PutKeyRequestProto.Builder createKeyRequest = PutKeyRequestProto
-        .newBuilder()
-        .setPipeline(xceiverClient.getPipeline().getProtobufMessage())
-        .setKeyData(containerKeyData);
-    String id = xceiverClient.getPipeline().getLeader().getDatanodeUuid();
-    ContainerCommandRequestProto request = ContainerCommandRequestProto
-        .newBuilder()
-        .setCmdType(Type.PutKey)
-        .setTraceID(traceID)
-        .setDatanodeID(id)
-        .setPutKey(createKeyRequest)
-        .build();
-    ContainerCommandResponseProto response = xceiverClient.sendCommand(request);
-    validateContainerResponse(response);
-  }
-
-  /**
-   * Calls the container protocol to read a chunk.
-   *
-   * @param xceiverClient client to perform call
-   * @param chunk information about chunk to read
-   * @param key the key name
-   * @param traceID container protocol call args
-   * @return container protocol read chunk response
-   * @throws IOException if there is an I/O error while performing the call
-   */
-  public static ReadChunkResponseProto readChunk(XceiverClientSpi xceiverClient,
-      ChunkInfo chunk, String key, String traceID)
-      throws IOException {
-    ReadChunkRequestProto.Builder readChunkRequest = ReadChunkRequestProto
-        .newBuilder()
-        .setPipeline(xceiverClient.getPipeline().getProtobufMessage())
-        .setKeyName(key)
-        .setChunkData(chunk);
-    String id = xceiverClient.getPipeline().getLeader().getDatanodeUuid();
-    ContainerCommandRequestProto request = ContainerCommandRequestProto
-        .newBuilder()
-        .setCmdType(Type.ReadChunk)
-        .setTraceID(traceID)
-        .setDatanodeID(id)
-        .setReadChunk(readChunkRequest)
-        .build();
-    ContainerCommandResponseProto response = xceiverClient.sendCommand(request);
-    validateContainerResponse(response);
-    return response.getReadChunk();
-  }
-
-  /**
-   * Calls the container protocol to write a chunk.
-   *
-   * @param xceiverClient client to perform call
-   * @param chunk information about chunk to write
-   * @param key the key name
-   * @param data the data of the chunk to write
-   * @param traceID container protocol call args
-   * @throws IOException if there is an I/O error while performing the call
-   */
-  public static void writeChunk(XceiverClientSpi xceiverClient, ChunkInfo chunk,
-      String key, ByteString data, String traceID)
-      throws IOException {
-    WriteChunkRequestProto.Builder writeChunkRequest = WriteChunkRequestProto
-        .newBuilder()
-        .setPipeline(xceiverClient.getPipeline().getProtobufMessage())
-        .setKeyName(key)
-        .setChunkData(chunk)
-        .setData(data);
-    String id = xceiverClient.getPipeline().getLeader().getDatanodeUuid();
-    ContainerCommandRequestProto request = ContainerCommandRequestProto
-        .newBuilder()
-        .setCmdType(Type.WriteChunk)
-        .setTraceID(traceID)
-        .setDatanodeID(id)
-        .setWriteChunk(writeChunkRequest)
-        .build();
-    ContainerCommandResponseProto response = xceiverClient.sendCommand(request);
-    validateContainerResponse(response);
-  }
-
-  /**
-   * Allows writing a small file using single RPC. This takes the container
-   * name, key name and data to write sends all that data to the container using
-   * a single RPC. This API is designed to be used for files which are smaller
-   * than 1 MB.
-   *
-   * @param client - client that communicates with the container.
-   * @param containerName - Name of the container
-   * @param key - Name of the Key
-   * @param data - Data to be written into the container.
-   * @param traceID - Trace ID for logging purpose.
-   * @throws IOException
-   */
-  public static void writeSmallFile(XceiverClientSpi client,
-      String containerName, String key, byte[] data, String traceID)
-      throws IOException {
-
-    KeyData containerKeyData =
-        KeyData.newBuilder().setContainerName(containerName).setName(key)
-            .build();
-    PutKeyRequestProto.Builder createKeyRequest =
-        PutKeyRequestProto.newBuilder()
-            .setPipeline(client.getPipeline().getProtobufMessage())
-            .setKeyData(containerKeyData);
-
-    KeyValue keyValue =
-        KeyValue.newBuilder().setKey("OverWriteRequested").setValue("true")
-            .build();
-    ChunkInfo chunk =
-        ChunkInfo.newBuilder().setChunkName(key + "_chunk").setOffset(0)
-            .setLen(data.length).addMetadata(keyValue).build();
-
-    PutSmallFileRequestProto putSmallFileRequest =
-        PutSmallFileRequestProto.newBuilder().setChunkInfo(chunk)
-            .setKey(createKeyRequest).setData(ByteString.copyFrom(data))
-            .build();
-
-    String id = client.getPipeline().getLeader().getDatanodeUuid();
-    ContainerCommandRequestProto request =
-        ContainerCommandRequestProto.newBuilder()
-            .setCmdType(Type.PutSmallFile)
-            .setTraceID(traceID)
-            .setDatanodeID(id)
-            .setPutSmallFile(putSmallFileRequest)
-            .build();
-    ContainerCommandResponseProto response = client.sendCommand(request);
-    validateContainerResponse(response);
-  }
-
-  /**
-   * createContainer call that creates a container on the datanode.
-   * @param client  - client
-   * @param traceID - traceID
-   * @throws IOException
-   */
-  public static void createContainer(XceiverClientSpi client, String traceID)
-      throws IOException {
-    ContainerProtos.CreateContainerRequestProto.Builder createRequest =
-        ContainerProtos.CreateContainerRequestProto
-            .newBuilder();
-    ContainerProtos.ContainerData.Builder containerData = ContainerProtos
-        .ContainerData.newBuilder();
-    containerData.setName(client.getPipeline().getContainerName());
-    createRequest.setPipeline(client.getPipeline().getProtobufMessage());
-    createRequest.setContainerData(containerData.build());
-
-    String id = client.getPipeline().getLeader().getDatanodeUuid();
-    ContainerCommandRequestProto.Builder request =
-        ContainerCommandRequestProto.newBuilder();
-    request.setCmdType(ContainerProtos.Type.CreateContainer);
-    request.setCreateContainer(createRequest);
-    request.setDatanodeID(id);
-    request.setTraceID(traceID);
-    ContainerCommandResponseProto response = client.sendCommand(
-        request.build());
-    validateContainerResponse(response);
-  }
-
-  /**
-   * Deletes a container from a pipeline.
-   *
-   * @param client
-   * @param force whether or not to forcibly delete the container.
-   * @param traceID
-   * @throws IOException
-   */
-  public static void deleteContainer(XceiverClientSpi client,
-      boolean force, String traceID) throws IOException {
-    ContainerProtos.DeleteContainerRequestProto.Builder deleteRequest =
-        ContainerProtos.DeleteContainerRequestProto.newBuilder();
-    deleteRequest.setName(client.getPipeline().getContainerName());
-    deleteRequest.setPipeline(client.getPipeline().getProtobufMessage());
-    deleteRequest.setForceDelete(force);
-    String id = client.getPipeline().getLeader().getDatanodeUuid();
-    ContainerCommandRequestProto.Builder request =
-        ContainerCommandRequestProto.newBuilder();
-    request.setCmdType(ContainerProtos.Type.DeleteContainer);
-    request.setDeleteContainer(deleteRequest);
-    request.setTraceID(traceID);
-    request.setDatanodeID(id);
-    ContainerCommandResponseProto response =
-        client.sendCommand(request.build());
-    validateContainerResponse(response);
-  }
-
-  /**
-   * Close a container.
-   *
-   * @param client
-   * @param traceID
-   * @throws IOException
-   */
-  public static void closeContainer(XceiverClientSpi client, String traceID)
-      throws IOException {
-    ContainerProtos.CloseContainerRequestProto.Builder closeRequest =
-        ContainerProtos.CloseContainerRequestProto.newBuilder();
-    closeRequest.setPipeline(client.getPipeline().getProtobufMessage());
-
-    String id = client.getPipeline().getLeader().getDatanodeUuid();
-    ContainerCommandRequestProto.Builder request =
-        ContainerCommandRequestProto.newBuilder();
-    request.setCmdType(Type.CloseContainer);
-    request.setCloseContainer(closeRequest);
-    request.setTraceID(traceID);
-    request.setDatanodeID(id);
-    ContainerCommandResponseProto response =
-        client.sendCommand(request.build());
-    validateContainerResponse(response);
-  }
-
-  /**
-   * readContainer call that gets meta data from an existing container.
-   *
-   * @param client - client
-   * @param traceID - trace ID
-   * @throws IOException
-   */
-  public static ReadContainerResponseProto readContainer(
-      XceiverClientSpi client, String containerName,
-      String traceID) throws IOException {
-    ReadContainerRequestProto.Builder readRequest =
-        ReadContainerRequestProto.newBuilder();
-    readRequest.setName(containerName);
-    readRequest.setPipeline(client.getPipeline().getProtobufMessage());
-    String id = client.getPipeline().getLeader().getDatanodeUuid();
-    ContainerCommandRequestProto.Builder request =
-        ContainerCommandRequestProto.newBuilder();
-    request.setCmdType(Type.ReadContainer);
-    request.setReadContainer(readRequest);
-    request.setDatanodeID(id);
-    request.setTraceID(traceID);
-    ContainerCommandResponseProto response =
-        client.sendCommand(request.build());
-    validateContainerResponse(response);
-    return response.getReadContainer();
-  }
-
-  /**
-   * Reads the data given the container name and key.
-   *
-   * @param client
-   * @param containerName - name of the container
-   * @param key - key
-   * @param traceID - trace ID
-   * @return GetSmallFileResponseProto
-   * @throws IOException
-   */
-  public static GetSmallFileResponseProto readSmallFile(XceiverClientSpi client,
-      String containerName, String key, String traceID) throws IOException {
-    KeyData containerKeyData = KeyData
-        .newBuilder()
-        .setContainerName(containerName)
-        .setName(key).build();
-
-    GetKeyRequestProto.Builder getKey = GetKeyRequestProto
-        .newBuilder()
-        .setPipeline(client.getPipeline().getProtobufMessage())
-        .setKeyData(containerKeyData);
-    ContainerProtos.GetSmallFileRequestProto getSmallFileRequest =
-        GetSmallFileRequestProto
-            .newBuilder().setKey(getKey)
-            .build();
-    String id = client.getPipeline().getLeader().getDatanodeUuid();
-    ContainerCommandRequestProto request = ContainerCommandRequestProto
-        .newBuilder()
-        .setCmdType(Type.GetSmallFile)
-        .setTraceID(traceID)
-        .setDatanodeID(id)
-        .setGetSmallFile(getSmallFileRequest)
-        .build();
-    ContainerCommandResponseProto response = client.sendCommand(request);
-    validateContainerResponse(response);
-    return response.getGetSmallFile();
-  }
-
-  /**
-   * Validates a response from a container protocol call.  Any non-successful
-   * return code is mapped to a corresponding exception and thrown.
-   *
-   * @param response container protocol call response
-   * @throws IOException if the container protocol call failed
-   */
-  private static void validateContainerResponse(
-      ContainerCommandResponseProto response
-  ) throws StorageContainerException {
-    if (response.getResult() == ContainerProtos.Result.SUCCESS) {
-      return;
-    }
-    throw new StorageContainerException(
-        response.getMessage(), response.getResult());
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/package-info.java
deleted file mode 100644
index aa89af0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.storage;
-
-/**
- * This package contains StorageContainerManager classes.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/RatisHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/RatisHelper.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/RatisHelper.java
deleted file mode 100644
index de159ab..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/RatisHelper.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.ratis;
-
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.ratis.client.RaftClient;
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.grpc.GrpcConfigKeys;
-import org.apache.ratis.protocol.RaftGroup;
-import org.apache.ratis.protocol.RaftGroupId;
-import org.apache.ratis.protocol.RaftPeer;
-import org.apache.ratis.protocol.RaftPeerId;
-import org.apache.ratis.rpc.RpcType;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
-import org.apache.ratis.util.SizeInBytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Collections;
-import java.util.Collection;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.stream.Collectors;
-
-/**
- * Ratis helper methods.
- */
-public interface RatisHelper {
-  Logger LOG = LoggerFactory.getLogger(RatisHelper.class);
-
-  static String toRaftPeerIdString(DatanodeID id) {
-    return id.getIpAddr() + "_" + id.getRatisPort();
-  }
-
-  static String toRaftPeerAddressString(DatanodeID id) {
-    return id.getIpAddr() + ":" + id.getRatisPort();
-  }
-
-  static RaftPeerId toRaftPeerId(DatanodeID id) {
-    return RaftPeerId.valueOf(toRaftPeerIdString(id));
-  }
-
-  static RaftPeer toRaftPeer(DatanodeID id) {
-    return new RaftPeer(toRaftPeerId(id), toRaftPeerAddressString(id));
-  }
-
-  static List<RaftPeer> toRaftPeers(Pipeline pipeline) {
-    return toRaftPeers(pipeline.getMachines());
-  }
-
-  static <E extends DatanodeID> List<RaftPeer> toRaftPeers(List<E> datanodes) {
-    return datanodes.stream().map(RatisHelper::toRaftPeer)
-        .collect(Collectors.toList());
-  }
-
-  /* TODO: use a dummy id for all groups for the moment.
-   *       It should be changed to a unique id for each group.
-   */
-  RaftGroupId DUMMY_GROUP_ID =
-      RaftGroupId.valueOf(ByteString.copyFromUtf8("AOzoneRatisGroup"));
-
-  RaftGroup EMPTY_GROUP = new RaftGroup(DUMMY_GROUP_ID,
-      Collections.emptyList());
-
-  static RaftGroup emptyRaftGroup() {
-    return EMPTY_GROUP;
-  }
-
-  static RaftGroup newRaftGroup(List<DatanodeID> datanodes) {
-    final List<RaftPeer> newPeers = datanodes.stream()
-        .map(RatisHelper::toRaftPeer)
-        .collect(Collectors.toList());
-    return RatisHelper.newRaftGroup(newPeers);
-  }
-
-  static RaftGroup newRaftGroup(Collection<RaftPeer> peers) {
-    return peers.isEmpty()? emptyRaftGroup()
-        : new RaftGroup(DUMMY_GROUP_ID, peers);
-  }
-
-  static RaftGroup newRaftGroup(Pipeline pipeline) {
-    return newRaftGroup(toRaftPeers(pipeline));
-  }
-
-  static RaftClient newRaftClient(RpcType rpcType, Pipeline pipeline) {
-    return newRaftClient(rpcType, toRaftPeerId(pipeline.getLeader()),
-        newRaftGroup(pipeline));
-  }
-
-  static RaftClient newRaftClient(RpcType rpcType, RaftPeer leader) {
-    return newRaftClient(rpcType, leader.getId(),
-        newRaftGroup(new ArrayList<>(Arrays.asList(leader))));
-  }
-
-  static RaftClient newRaftClient(
-      RpcType rpcType, RaftPeerId leader, RaftGroup group) {
-    LOG.trace("newRaftClient: {}, leader={}, group={}", rpcType, leader, group);
-    final RaftProperties properties = new RaftProperties();
-    RaftConfigKeys.Rpc.setType(properties, rpcType);
-    GrpcConfigKeys.setMessageSizeMax(properties,
-        SizeInBytes.valueOf(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE));
-
-    return RaftClient.newBuilder()
-        .setRaftGroup(group)
-        .setLeaderId(leader)
-        .setProperties(properties)
-        .build();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/package-info.java
deleted file mode 100644
index c13c20c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis;
-
-/**
- * This package contains classes related to Apache Ratis.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/shaded/com/google/protobuf/ShadedProtoUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/shaded/com/google/protobuf/ShadedProtoUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/shaded/com/google/protobuf/ShadedProtoUtil.java
deleted file mode 100644
index 29242ad..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/shaded/com/google/protobuf/ShadedProtoUtil.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.ratis.shaded.com.google.protobuf;
-
-/** Utilities for the shaded protobuf in Ratis. */
-public interface ShadedProtoUtil {
-  /**
-   * @param bytes
-   * @return the wrapped shaded {@link ByteString} (no coping).
-   */
-  static ByteString asShadedByteString(byte[] bytes) {
-    return ByteString.wrap(bytes);
-  }
-
-  /**
-   * @param shaded
-   * @return a {@link com.google.protobuf.ByteString} (require coping).
-   */
-  static com.google.protobuf.ByteString asByteString(ByteString shaded) {
-    return com.google.protobuf.ByteString.copyFrom(
-        shaded.asReadOnlyByteBuffer());
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/shaded/com/google/protobuf/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/shaded/com/google/protobuf/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/shaded/com/google/protobuf/package-info.java
deleted file mode 100644
index 032dd96..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/shaded/com/google/protobuf/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.shaded.com.google.protobuf;
-
-/**
- * This package contains classes related to the shaded protobuf in Apache Ratis.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
deleted file mode 100644
index 4e79827..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
+++ /dev/null
@@ -1,415 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are private and Unstable.
- * Please see http://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/InterfaceClassification.html
- * for what changes are allowed for a *Unstable* .proto interface.
- */
-
-// This file contains protocol buffers that are used to transfer data
-// to and from the datanode.
-option java_package = "org.apache.hadoop.hdfs.ozone.protocol.proto";
-option java_outer_classname = "ContainerProtos";
-option java_generate_equals_and_hash = true;
-package hadoop.hdfs.ozone;
-import "hdfs.proto";
-import "Ozone.proto";
-
-/**
- * Commands that are used to manipulate the state of containers on a datanode.
- *
- * These commands allow us to work against the datanode - from
- * StorageContainer Manager as well as clients.
- *
- *  1. CreateContainer - This call is usually made by Storage Container
- *     manager, when we need to create a new container on a given datanode.
- *
- *  2. ReadContainer - Allows end user to stat a container. For example
- *     this allows us to return the metadata of a container.
- *
- *  3. UpdateContainer - Updates a container metadata.
-
- *  4. DeleteContainer - This call is made to delete a container.
- *
- *  5. ListContainer - Returns the list of containers on this
- *     datanode. This will be used by tests and tools.
- *
- *  6. PutKey - Given a valid container, creates a key.
- *
- *  7. GetKey - Allows user to read the metadata of a Key.
- *
- *  8. DeleteKey - Deletes a given key.
- *
- *  9. ListKey - Returns a list of keys that are present inside
- *      a given container.
- *
- *  10. ReadChunk - Allows us to read a chunk.
- *
- *  11. DeleteChunk - Delete an unused chunk.
- *
- *  12. WriteChunk - Allows us to write a chunk
- *
- *  13. ListChunk - Given a Container/Key returns the list of Chunks.
- *
- *  14. CompactChunk - Re-writes a chunk based on Offsets.
- *
- *  15. PutSmallFile - A single RPC that combines both putKey and WriteChunk.
- *
- *  16. GetSmallFile - A single RPC that combines both getKey and ReadChunk.
- *
- *  17. CloseContainer - Closes an open container and makes it immutable.
- *
- *  18. CopyContainer - Copies a container from a remote machine.
- */
-
-enum Type {
-  CreateContainer = 1;
-  ReadContainer = 2;
-  UpdateContainer = 3;
-  DeleteContainer = 4;
-  ListContainer = 5;
-
-  PutKey = 6;
-  GetKey = 7;
-  DeleteKey = 8;
-  ListKey = 9;
-
-  ReadChunk = 10;
-  DeleteChunk = 11;
-  WriteChunk = 12;
-  ListChunk = 13;
-  CompactChunk = 14;
-
-  /** Combines Key and Chunk Operation into Single RPC. */
-  PutSmallFile = 15;
-  GetSmallFile = 16;
-  CloseContainer = 17;
-
-}
-
-
-enum Result {
-  SUCCESS = 1;
-  UNSUPPORTED_REQUEST = 2;
-  MALFORMED_REQUEST = 3;
-  CONTAINER_INTERNAL_ERROR = 4;
-  INVALID_CONFIG = 5;
-  INVALID_FILE_HASH_FOUND = 6;
-  CONTAINER_EXISTS = 7;
-  NO_SUCH_ALGORITHM = 8;
-  CONTAINER_NOT_FOUND = 9;
-  IO_EXCEPTION = 10;
-  UNABLE_TO_READ_METADATA_DB = 11;
-  NO_SUCH_KEY = 12;
-  OVERWRITE_FLAG_REQUIRED = 13;
-  UNABLE_TO_FIND_DATA_DIR = 14;
-  INVALID_WRITE_SIZE = 15;
-  CHECKSUM_MISMATCH = 16;
-  UNABLE_TO_FIND_CHUNK = 17;
-  PROTOC_DECODING_ERROR = 18;
-  INVALID_ARGUMENT = 19;
-  PUT_SMALL_FILE_ERROR = 20;
-  GET_SMALL_FILE_ERROR = 21;
-  CLOSED_CONTAINER_IO = 22;
-  ERROR_CONTAINER_NOT_EMPTY = 23;
-  ERROR_IN_COMPACT_DB = 24;
-  UNCLOSED_CONTAINER_IO = 25;
-  DELETE_ON_OPEN_CONTAINER = 26;
-  CLOSED_CONTAINER_RETRY = 27;
-}
-
-message ContainerCommandRequestProto {
-  required Type cmdType = 1; // Type of the command
-
-  // A string that identifies this command, we generate  Trace ID in Ozone
-  // frontend and this allows us to trace that command all over ozone.
-  optional string traceID = 2;
-
-  // One of the following command is available when the corresponding
-  // cmdType is set. At the protocol level we allow only
-  // one command in each packet.
-  // TODO : Upgrade to Protobuf 2.6 or later.
-  optional   CreateContainerRequestProto createContainer = 3;
-  optional   ReadContainerRequestProto readContainer = 4;
-  optional   UpdateContainerRequestProto updateContainer = 5;
-  optional   DeleteContainerRequestProto deleteContainer = 6;
-  optional   ListContainerRequestProto listContainer = 7;
-
-  optional   PutKeyRequestProto putKey = 8;
-  optional   GetKeyRequestProto getKey = 9;
-  optional   DeleteKeyRequestProto deleteKey = 10;
-  optional   ListKeyRequestProto listKey = 11;
-
-  optional   ReadChunkRequestProto readChunk = 12;
-  optional   WriteChunkRequestProto writeChunk = 13;
-  optional   DeleteChunkRequestProto deleteChunk = 14;
-  optional   ListChunkRequestProto listChunk = 15;
-
-  optional   PutSmallFileRequestProto putSmallFile = 16;
-  optional   GetSmallFileRequestProto getSmallFile = 17;
-  optional   CloseContainerRequestProto closeContainer = 18;
-  required   string datanodeID = 19;
-}
-
-message ContainerCommandResponseProto {
-  required Type cmdType = 1;
-  optional string traceID = 2;
-
-  optional   CreateContainerResponseProto createContainer = 3;
-  optional   ReadContainerResponseProto readContainer = 4;
-  optional   UpdateContainerResponseProto updateContainer = 5;
-  optional   DeleteContainerResponseProto deleteContainer = 6;
-  optional   ListContainerResponseProto listContainer = 7;
-
-  optional   PutKeyResponseProto putKey = 8;
-  optional   GetKeyResponseProto getKey = 9;
-  optional   DeleteKeyResponseProto deleteKey = 10;
-  optional   ListKeyResponseProto listKey = 11;
-
-  optional  WriteChunkResponseProto writeChunk = 12;
-  optional  ReadChunkResponseProto readChunk = 13;
-  optional  DeleteChunkResponseProto deleteChunk = 14;
-  optional  ListChunkResponseProto listChunk = 15;
-
-  required Result result = 17;
-  optional string message = 18;
-
-  optional PutSmallFileResponseProto putSmallFile = 19;
-  optional GetSmallFileResponseProto getSmallFile = 20;
-  optional CloseContainerResponseProto closeContainer = 21;
-
-}
-
-message ContainerData {
-  required string name = 1;
-  repeated KeyValue metadata = 2;
-  optional string dbPath = 3;
-  optional string containerPath = 4;
-  optional string hash = 6;
-  optional int64 bytesUsed = 7;
-  optional int64 size = 8;
-  optional int64 keyCount = 9;
-  //TODO: change required after we switch container ID from string to long
-  optional int64 containerID = 10;
-  optional LifeCycleState state = 11 [default = OPEN];
-}
-
-message ContainerMeta {
-  required string fileName = 1;
-  required string hash = 2;
-}
-
-// Container Messages.
-message  CreateContainerRequestProto {
-  required Pipeline pipeline = 1;
-  required ContainerData containerData = 2;
-}
-
-message  CreateContainerResponseProto {
-}
-
-message  ReadContainerRequestProto {
-  required Pipeline pipeline = 1;
-  required string name = 2;
-}
-
-message  ReadContainerResponseProto {
-  optional ContainerData containerData = 2;
-}
-
-message  UpdateContainerRequestProto {
-  required Pipeline pipeline = 1;
-  required ContainerData containerData = 2;
-  optional bool forceUpdate = 3 [default = false];
-}
-
-message  UpdateContainerResponseProto {
-}
-
-message  DeleteContainerRequestProto {
-  required Pipeline pipeline = 1;
-  required string name = 2;
-  optional bool forceDelete = 3 [default = false];
-}
-
-message  DeleteContainerResponseProto {
-}
-
-message  ListContainerRequestProto {
-  required Pipeline pipeline = 1;
-  optional string prefix = 2;
-  required uint32 count = 3; // Max Results to return
-  optional string prevKey = 4;  // if this is not set query from start.
-}
-
-message  ListContainerResponseProto {
-  repeated ContainerData containerData = 1;
-}
-
-message CloseContainerRequestProto {
-  required Pipeline pipeline = 1;
-}
-
-message CloseContainerResponseProto {
-  optional Pipeline pipeline = 1;
-  optional string hash = 2;
-}
-
-message KeyData {
-  required string containerName = 1;
-  required string name = 2;
-  optional int64 flags = 3; // for future use.
-  repeated KeyValue metadata = 4;
-  repeated ChunkInfo chunks = 5;
-}
-
-// Key Messages.
-message  PutKeyRequestProto {
-  required Pipeline pipeline = 1;
-  required KeyData keyData = 2;
-}
-
-message  PutKeyResponseProto {
-}
-
-message  GetKeyRequestProto  {
-  required Pipeline pipeline = 1;
-  required KeyData keyData = 2;
-}
-
-message  GetKeyResponseProto  {
-  required KeyData keyData = 1;
-}
-
-
-message  DeleteKeyRequestProto {
-  required Pipeline pipeline = 1;
-  required string name = 2;
-}
-
-message   DeleteKeyResponseProto {
-}
-
-message  ListKeyRequestProto {
-  required Pipeline pipeline = 1;
-  optional string prefix = 2; // if specified returns keys that match prefix.
-  required string prevKey = 3;
-  required uint32 count = 4;
-
-}
-
-message  ListKeyResponseProto {
-  repeated KeyData keyData = 1;
-}
-
-// Chunk Operations
-
-message ChunkInfo {
-  required string chunkName = 1;
-  required uint64 offset = 2;
-  required uint64 len = 3;
-  optional string checksum = 4;
-  repeated KeyValue metadata = 5;
-}
-
-enum Stage {
-    WRITE_DATA = 1;
-    COMMIT_DATA = 2;
-    COMBINED = 3;
-}
-
-message  WriteChunkRequestProto  {
-  required Pipeline pipeline = 1;
-  required string keyName = 2;
-  required ChunkInfo chunkData = 3;
-  optional bytes data = 4;
-  optional Stage stage = 5 [default = COMBINED];
-}
-
-message  WriteChunkResponseProto {
-}
-
-message  ReadChunkRequestProto  {
-  required Pipeline pipeline = 1;
-  required string keyName = 2;
-  required ChunkInfo chunkData = 3;
-}
-
-message  ReadChunkResponseProto {
-  required Pipeline pipeline = 1;
-  required ChunkInfo chunkData = 2;
-  required bytes data = 3;
-}
-
-message  DeleteChunkRequestProto {
-  required Pipeline pipeline = 1;
-  required string keyName = 2;
-  required ChunkInfo chunkData = 3;
-}
-
-message  DeleteChunkResponseProto {
-}
-
-message  ListChunkRequestProto {
-  required Pipeline pipeline = 1;
-  required string keyName = 2;
-  required string prevChunkName = 3;
-  required uint32 count = 4;
-}
-
-message  ListChunkResponseProto {
-  repeated ChunkInfo chunkData = 1;
-}
-
-/** For small file access combines write chunk and putKey into a single
-RPC */
-
-message PutSmallFileRequestProto {
-  required PutKeyRequestProto key = 1;
-  required ChunkInfo chunkInfo = 2;
-  required bytes data = 3;
-}
-
-
-message PutSmallFileResponseProto {
-
-}
-
-message GetSmallFileRequestProto {
-  required GetKeyRequestProto key = 1;
-}
-
-message GetSmallFileResponseProto {
-  required ReadChunkResponseProto data = 1;
-}
-
-message CopyContainerRequestProto {
-  required string containerName = 1;
-  required uint64 readOffset = 2;
-  optional uint64 len = 3;
-}
-
-message CopyContainerResponseProto {
-  required string archiveName = 1;
-  required uint64 readOffset = 2;
-  required uint64 len = 3;
-  required bool eof = 4;
-  repeated bytes data = 5;
-  optional int64 checksum = 6;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
deleted file mode 100644
index cdc1df5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
+++ /dev/null
@@ -1,457 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are private and unstable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
- * for what changes are allowed for a *unstable* .proto interface.
- */
-
-option java_package = "org.apache.hadoop.ozone.protocol.proto";
-option java_outer_classname = "KeySpaceManagerProtocolProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.hdfs.ozone;
-
-/**
-This is file contains the protocol to communicate with
-Ozone key space manager. Ozone KSM manages the namespace for ozone.
-This is similar to Namenode for Ozone.
-*/
-
-import "hdfs.proto";
-import "Ozone.proto";
-
-enum Status {
-    OK = 1;
-    VOLUME_NOT_UNIQUE = 2;
-    VOLUME_NOT_FOUND = 3;
-    VOLUME_NOT_EMPTY = 4;
-    VOLUME_ALREADY_EXISTS = 5;
-    USER_NOT_FOUND = 6;
-    USER_TOO_MANY_VOLUMES = 7;
-    BUCKET_NOT_FOUND = 8;
-    BUCKET_NOT_EMPTY = 9;
-    BUCKET_ALREADY_EXISTS = 10;
-    KEY_ALREADY_EXISTS = 11;
-    KEY_NOT_FOUND = 12;
-    ACCESS_DENIED = 13;
-    INTERNAL_ERROR = 14;
-}
-
-
-message VolumeInfo {
-    required string adminName = 1;
-    required string ownerName = 2;
-    required string volume = 3;
-    optional uint64 quotaInBytes = 4;
-    repeated KeyValue metadata = 5;
-    repeated OzoneAclInfo volumeAcls = 6;
-    required uint64 creationTime = 7;
-}
-
-/**
-    Creates a volume
-*/
-message CreateVolumeRequest {
-    required VolumeInfo volumeInfo = 1;
-}
-
-message CreateVolumeResponse {
-
-    required Status status = 1;
-}
-
-message VolumeList {
-    repeated string volumeNames = 1;
-}
-
-/**
-    Changes the Volume Properties -- like ownership and quota for a volume.
-*/
-message SetVolumePropertyRequest {
-    required string volumeName = 1;
-    optional string ownerName = 2;
-    optional uint64 quotaInBytes = 3;
-}
-
-message SetVolumePropertyResponse {
-    required Status status = 1;
-}
-
-/**
- * Checks if the user has specified permissions for the volume
- */
-message CheckVolumeAccessRequest {
-    required string volumeName = 1;
-    required OzoneAclInfo userAcl = 2;
-}
-
-message CheckVolumeAccessResponse {
-
-    required Status status = 1;
-}
-
-
-/**
-    Returns information about a volume.
-*/
-
-message InfoVolumeRequest {
-    required string volumeName = 1;
-}
-
-message InfoVolumeResponse {
-    required Status status = 1;
-    optional VolumeInfo volumeInfo = 2;
-
-}
-
-/**
-    Deletes an existing volume.
-*/
-message DeleteVolumeRequest {
-    required string volumeName = 1;
-}
-
-message DeleteVolumeResponse {
-    required Status status = 1;
-}
-
-
-/**
-    List Volumes -- List all volumes in the cluster or by user.
-*/
-
-message ListVolumeRequest {
-    enum Scope {
-        USER_VOLUMES = 1;   // User volumes -- called by user
-        VOLUMES_BY_USER = 2; // User volumes - called by Admin
-        VOLUMES_BY_CLUSTER = 3; // All volumes in the cluster
-    }
-    required Scope scope = 1;
-    optional string userName = 2;
-    optional string prefix = 3;
-    optional string prevKey = 4;
-    optional uint32 maxKeys = 5;
-}
-
-message ListVolumeResponse {
-    required Status status = 1;
-    repeated VolumeInfo volumeInfo = 2;
-}
-
-message BucketInfo {
-    required string volumeName = 1;
-    required string bucketName = 2;
-    repeated OzoneAclInfo acls = 3;
-    required bool isVersionEnabled = 4 [default = false];
-    required StorageTypeProto storageType = 5 [default = DISK];
-    required uint64 creationTime = 6;
-}
-
-message BucketArgs {
-    required string volumeName = 1;
-    required string bucketName = 2;
-    repeated OzoneAclInfo addAcls = 3;
-    repeated OzoneAclInfo removeAcls = 4;
-    optional bool isVersionEnabled = 5;
-    optional StorageTypeProto storageType = 6;
-}
-
-message OzoneAclInfo {
-    enum OzoneAclType {
-        USER = 1;
-        GROUP = 2;
-        WORLD = 3;
-    }
-    enum OzoneAclRights {
-        READ = 1;
-        WRITE = 2;
-        READ_WRITE = 3;
-    }
-    required OzoneAclType type = 1;
-    required string name = 2;
-    required OzoneAclRights rights = 3;
-}
-
-message CreateBucketRequest {
-    required BucketInfo bucketInfo = 1;
-}
-
-message CreateBucketResponse {
-    required Status status = 1;
-}
-
-message InfoBucketRequest {
-    required string volumeName = 1;
-    required string bucketName = 2;
-}
-
-message InfoBucketResponse {
-    required Status status = 1;
-    optional BucketInfo bucketInfo = 2;
-}
-
-message ListBucketsRequest {
-    required string volumeName = 1;
-    optional string startKey = 2;
-    optional string prefix = 3;
-    optional int32 count = 4;
-}
-
-message ListBucketsResponse {
-    required Status status = 1;
-    repeated BucketInfo bucketInfo = 2;
-}
-
-message KeyArgs {
-    required string volumeName = 1;
-    required string bucketName = 2;
-    required string keyName = 3;
-    optional uint64 dataSize = 4;
-    optional hadoop.hdfs.ozone.ReplicationType type = 5;
-    optional hadoop.hdfs.ozone.ReplicationFactor factor = 6;
-}
-
-message KeyLocation {
-    required string blockID = 1;
-    required string containerName = 2;
-    required bool shouldCreateContainer = 3;
-    required uint64 offset = 4;
-    required uint64 length = 5;
-    // indicated at which version this block gets created.
-    optional uint64 createVersion = 6;
-}
-
-message KeyLocationList {
-    optional uint64 version = 1;
-    repeated KeyLocation keyLocations = 2;
-}
-
-message KeyInfo {
-    required string volumeName = 1;
-    required string bucketName = 2;
-    required string keyName = 3;
-    required uint64 dataSize = 4;
-    repeated KeyLocationList keyLocationList = 5;
-    required uint64 creationTime = 6;
-    required uint64 modificationTime = 7;
-    optional uint64 latestVersion = 8;
-}
-
-message LocateKeyRequest {
-    required KeyArgs keyArgs = 1;
-}
-
-message LocateKeyResponse {
-    required Status status = 1;
-    optional KeyInfo keyInfo = 2;
-    // clients' followup request may carry this ID for stateful operations (similar
-    // to a cookie).
-    optional uint32 ID = 3;
-    // TODO : allow specifiying a particular version to read.
-    optional uint64 openVersion = 4;
-}
-
-message SetBucketPropertyRequest {
-    required BucketArgs bucketArgs = 1;
-}
-
-message SetBucketPropertyResponse {
-    required Status status = 1;
-}
-
-message DeleteBucketRequest {
-    required string volumeName = 1;
-    required string bucketName = 2;
-}
-
-message DeleteBucketResponse {
-    required Status status = 1;
-}
-
-message ListKeysRequest {
-    required string volumeName = 1;
-    required string bucketName = 2;
-    optional string startKey = 3;
-    optional string prefix = 4;
-    optional int32 count = 5;
-}
-
-message ListKeysResponse {
-    required Status status = 1;
-    repeated KeyInfo keyInfo = 2;
-}
-
-message AllocateBlockRequest {
-    required KeyArgs keyArgs = 1;
-    required uint32 clientID = 2;
-}
-
-message AllocateBlockResponse {
-    required Status status = 1;
-    required KeyLocation keyLocation = 2;
-}
-
-message CommitKeyRequest {
-    required KeyArgs keyArgs = 1;
-    required uint32 clientID = 2;
-}
-
-message CommitKeyResponse {
-    required Status status = 1;
-}
-
-message ServiceListRequest {
-}
-
-message ServiceListResponse {
-    required Status status = 1;
-    repeated ServiceInfo serviceInfo = 2;
-}
-
-message ServicePort {
-    enum Type {
-        RPC = 1;
-        HTTP = 2;
-        HTTPS = 3;
-        RATIS = 4;
-    };
-    required Type type = 1;
-    required uint32 value = 2;
-}
-
-message ServiceInfo {
-    required NodeType nodeType = 1;
-    required string hostname = 2;
-    repeated ServicePort servicePorts = 3;
-}
-
-/**
- The KSM service that takes care of Ozone namespace.
-*/
-service KeySpaceManagerService {
-
-    /**
-        Creates a Volume.
-    */
-    rpc createVolume(CreateVolumeRequest)
-        returns(CreateVolumeResponse);
-
-    /**
-        Allows modificiation of volume properties.
-    */
-    rpc setVolumeProperty(SetVolumePropertyRequest)
-        returns (SetVolumePropertyResponse);
-
-    /**
-        Checks if the specified volume is accesible by the specified user.
-    */
-    rpc checkVolumeAccess(CheckVolumeAccessRequest)
-        returns (CheckVolumeAccessResponse);
-
-    /**
-        Gets Volume information.
-    */
-    rpc infoVolume(InfoVolumeRequest)
-        returns(InfoVolumeResponse);
-    /**
-        Deletes a volume if it is empty.
-    */
-    rpc deleteVolume(DeleteVolumeRequest)
-        returns (DeleteVolumeResponse);
-
-    /**
-        Lists Volumes
-    */
-    rpc listVolumes(ListVolumeRequest)
-        returns (ListVolumeResponse);
-
-    /**
-        Creates a Bucket.
-    */
-    rpc createBucket(CreateBucketRequest)
-        returns(CreateBucketResponse);
-
-    /**
-        Get Bucket information.
-    */
-    rpc infoBucket(InfoBucketRequest)
-        returns(InfoBucketResponse);
-
-    /**
-        Sets bucket properties.
-    */
-    rpc setBucketProperty(SetBucketPropertyRequest)
-        returns(SetBucketPropertyResponse);
-
-    /**
-        Get key.
-    */
-    rpc createKey(LocateKeyRequest)
-        returns(LocateKeyResponse);
-
-    /**
-       Look up for an existing key.
-    */
-    rpc lookupKey(LocateKeyRequest)
-        returns(LocateKeyResponse);
-
-    /**
-       Delete an existing key.
-    */
-    rpc deleteKey(LocateKeyRequest)
-        returns(LocateKeyResponse);
-
-    /**
-       Deletes a bucket from volume if it is empty.
-    */
-    rpc deleteBucket(DeleteBucketRequest)
-        returns (DeleteBucketResponse);
-
-    /**
-       List Buckets.
-    */
-    rpc listBuckets(ListBucketsRequest)
-    returns(ListBucketsResponse);
-
-    /**
-       List Keys.
-    */
-    rpc listKeys(ListKeysRequest)
-    returns(ListKeysResponse);
-
-    /**
-      Commit a key.
-    */
-    rpc commitKey(CommitKeyRequest)
-    returns(CommitKeyResponse);
-
-    /**
-      Allocate a new block for a key.
-    */
-    rpc allocateBlock(AllocateBlockRequest)
-    returns(AllocateBlockResponse);
-
-    /**
-      Returns list of Ozone services with its configuration details.
-    */
-    rpc getServiceList(ServiceListRequest)
-    returns(ServiceListResponse);
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[26/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
deleted file mode 100644
index 3acbcd6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
+++ /dev/null
@@ -1,512 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException.ResultCodes;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfoGroup;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.KeyInfo;
-import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.BackgroundService;
-import org.apache.hadoop.utils.BatchOperation;
-import org.iq80.leveldb.DBException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.ozone
-    .OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT;
-import static org.apache.hadoop.ozone
-    .OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY;
-import static org.apache.hadoop.ozone
-    .OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
-import static org.apache.hadoop.ozone
-    .OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT;
-import static org.apache.hadoop.ozone
-    .OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
-import static org.apache.hadoop.ozone
-    .OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
-import static org.apache.hadoop.ozone
-    .OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE;
-import static org.apache.hadoop.ozone
-    .OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_SECONDS;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_SECONDS_DEFAULT;
-import static org.apache.hadoop.ozone
-    .OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
-import static org.apache.hadoop.ozone
-    .OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB;
-import org.apache.hadoop.ozone.protocol
-    .proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol
-    .proto.OzoneProtos.ReplicationFactor;
-
-
-/**
- * Implementation of keyManager.
- */
-public class KeyManagerImpl implements KeyManager {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(KeyManagerImpl.class);
-
-  /**
-   * A SCM block client, used to talk to SCM to allocate block during putKey.
-   */
-  private final ScmBlockLocationProtocol scmBlockClient;
-  private final KSMMetadataManager metadataManager;
-  private final long scmBlockSize;
-  private final boolean useRatis;
-  private final BackgroundService keyDeletingService;
-  private final BackgroundService openKeyCleanupService;
-
-  private final long preallocateMax;
-  private final Random random;
-  private final String ksmId;
-
-  public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient,
-      KSMMetadataManager metadataManager, OzoneConfiguration conf,
-      String ksmId) {
-    this.scmBlockClient = scmBlockClient;
-    this.metadataManager = metadataManager;
-    this.scmBlockSize = conf.getLong(OZONE_SCM_BLOCK_SIZE_IN_MB,
-        OZONE_SCM_BLOCK_SIZE_DEFAULT) * OzoneConsts.MB;
-    this.useRatis = conf.getBoolean(DFS_CONTAINER_RATIS_ENABLED_KEY,
-        DFS_CONTAINER_RATIS_ENABLED_DEFAULT);
-    long  blockDeleteInterval = conf.getTimeDuration(
-        OZONE_BLOCK_DELETING_SERVICE_INTERVAL,
-        OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT,
-        TimeUnit.MILLISECONDS);
-    long serviceTimeout = conf.getTimeDuration(
-        OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
-        OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT,
-        TimeUnit.MILLISECONDS);
-    this.preallocateMax = conf.getLong(
-        OZONE_KEY_PREALLOCATION_MAXSIZE,
-        OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT);
-    keyDeletingService = new KeyDeletingService(
-        scmBlockClient, this, blockDeleteInterval, serviceTimeout, conf);
-    int openkeyCheckInterval = conf.getInt(
-        OZONE_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_SECONDS,
-        OZONE_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_SECONDS_DEFAULT);
-    openKeyCleanupService = new OpenKeyCleanupService(
-        scmBlockClient, this, openkeyCheckInterval, serviceTimeout);
-    random = new Random();
-    this.ksmId = ksmId;
-  }
-
-  @VisibleForTesting
-  public BackgroundService getOpenKeyCleanupService() {
-    return openKeyCleanupService;
-  }
-
-  @Override
-  public void start() {
-    keyDeletingService.start();
-    openKeyCleanupService.start();
-  }
-
-  @Override
-  public void stop() throws IOException {
-    keyDeletingService.shutdown();
-    openKeyCleanupService.shutdown();
-  }
-
-  private void validateBucket(String volumeName, String bucketName)
-      throws IOException {
-    byte[] volumeKey = metadataManager.getVolumeKey(volumeName);
-    byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
-
-    //Check if the volume exists
-    if(metadataManager.get(volumeKey) == null) {
-      LOG.error("volume not found: {}", volumeName);
-      throw new KSMException("Volume not found",
-          KSMException.ResultCodes.FAILED_VOLUME_NOT_FOUND);
-    }
-    //Check if bucket already exists
-    if(metadataManager.get(bucketKey) == null) {
-      LOG.error("bucket not found: {}/{} ", volumeName, bucketName);
-      throw new KSMException("Bucket not found",
-          KSMException.ResultCodes.FAILED_BUCKET_NOT_FOUND);
-    }
-  }
-
-  @Override
-  public KsmKeyLocationInfo allocateBlock(KsmKeyArgs args, int clientID)
-      throws IOException {
-    Preconditions.checkNotNull(args);
-    metadataManager.writeLock().lock();
-    String volumeName = args.getVolumeName();
-    String bucketName = args.getBucketName();
-    String keyName = args.getKeyName();
-    ReplicationFactor factor = args.getFactor();
-    ReplicationType type = args.getType();
-
-    // If user does not specify a replication strategy or
-    // replication factor, KSM will use defaults.
-    if(factor == null) {
-      factor = useRatis ? ReplicationFactor.THREE: ReplicationFactor.ONE;
-    }
-
-    if(type == null) {
-      type = useRatis ? ReplicationType.RATIS : ReplicationType.STAND_ALONE;
-    }
-
-    try {
-      validateBucket(volumeName, bucketName);
-      String objectKey = metadataManager.getKeyWithDBPrefix(
-          volumeName, bucketName, keyName);
-      byte[] openKey = metadataManager.getOpenKeyNameBytes(objectKey, clientID);
-      byte[] keyData = metadataManager.get(openKey);
-      if (keyData == null) {
-        LOG.error("Allocate block for a key not in open status in meta store " +
-            objectKey + " with ID " + clientID);
-        throw new KSMException("Open Key not found",
-            KSMException.ResultCodes.FAILED_KEY_NOT_FOUND);
-      }
-      AllocatedBlock allocatedBlock =
-          scmBlockClient.allocateBlock(scmBlockSize, type, factor, ksmId);
-      KsmKeyInfo keyInfo =
-          KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(keyData));
-      KsmKeyLocationInfo info = new KsmKeyLocationInfo.Builder()
-          .setContainerName(allocatedBlock.getPipeline().getContainerName())
-          .setBlockID(allocatedBlock.getKey())
-          .setShouldCreateContainer(allocatedBlock.getCreateContainer())
-          .setLength(scmBlockSize)
-          .setOffset(0)
-          .build();
-      // current version not committed, so new blocks coming now are added to
-      // the same version
-      keyInfo.appendNewBlocks(Collections.singletonList(info));
-      keyInfo.updateModifcationTime();
-      metadataManager.put(openKey, keyInfo.getProtobuf().toByteArray());
-      return info;
-    } finally {
-      metadataManager.writeLock().unlock();
-    }
-  }
-
-  @Override
-  public OpenKeySession openKey(KsmKeyArgs args) throws IOException {
-    Preconditions.checkNotNull(args);
-    metadataManager.writeLock().lock();
-    String volumeName = args.getVolumeName();
-    String bucketName = args.getBucketName();
-    String keyName = args.getKeyName();
-    ReplicationFactor factor = args.getFactor();
-    ReplicationType type = args.getType();
-
-    // If user does not specify a replication strategy or
-    // replication factor, KSM will use defaults.
-    if(factor == null) {
-      factor = useRatis ? ReplicationFactor.THREE: ReplicationFactor.ONE;
-    }
-
-    if(type == null) {
-      type = useRatis ? ReplicationType.RATIS : ReplicationType.STAND_ALONE;
-    }
-
-    try {
-      validateBucket(volumeName, bucketName);
-      long requestedSize = Math.min(preallocateMax, args.getDataSize());
-      List<KsmKeyLocationInfo> locations = new ArrayList<>();
-      String objectKey = metadataManager.getKeyWithDBPrefix(
-          volumeName, bucketName, keyName);
-      // requested size is not required but more like a optimization:
-      // SCM looks at the requested, if it 0, no block will be allocated at
-      // the point, if client needs more blocks, client can always call
-      // allocateBlock. But if requested size is not 0, KSM will preallocate
-      // some blocks and piggyback to client, to save RPC calls.
-      while (requestedSize > 0) {
-        long allocateSize = Math.min(scmBlockSize, requestedSize);
-        AllocatedBlock allocatedBlock =
-            scmBlockClient.allocateBlock(allocateSize, type, factor, ksmId);
-        KsmKeyLocationInfo subKeyInfo = new KsmKeyLocationInfo.Builder()
-            .setContainerName(allocatedBlock.getPipeline().getContainerName())
-            .setBlockID(allocatedBlock.getKey())
-            .setShouldCreateContainer(allocatedBlock.getCreateContainer())
-            .setLength(allocateSize)
-            .setOffset(0)
-            .build();
-        locations.add(subKeyInfo);
-        requestedSize -= allocateSize;
-      }
-      // NOTE size of a key is not a hard limit on anything, it is a value that
-      // client should expect, in terms of current size of key. If client sets a
-      // value, then this value is used, otherwise, we allocate a single block
-      // which is the current size, if read by the client.
-      long size = args.getDataSize() >= 0 ? args.getDataSize() : scmBlockSize;
-      byte[] keyKey = metadataManager.getDBKeyBytes(
-          volumeName, bucketName, keyName);
-      byte[] value = metadataManager.get(keyKey);
-      KsmKeyInfo keyInfo;
-      long openVersion;
-      if (value != null) {
-        // the key already exist, the new blocks will be added as new version
-        keyInfo = KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(value));
-        // when locations.size = 0, the new version will have identical blocks
-        // as its previous version
-        openVersion = keyInfo.addNewVersion(locations);
-        keyInfo.setDataSize(size + keyInfo.getDataSize());
-      } else {
-        // the key does not exist, create a new object, the new blocks are the
-        // version 0
-        long currentTime = Time.now();
-        keyInfo = new KsmKeyInfo.Builder()
-            .setVolumeName(args.getVolumeName())
-            .setBucketName(args.getBucketName())
-            .setKeyName(args.getKeyName())
-            .setKsmKeyLocationInfos(Collections.singletonList(
-                new KsmKeyLocationInfoGroup(0, locations)))
-            .setCreationTime(currentTime)
-            .setModificationTime(currentTime)
-            .setDataSize(size)
-            .build();
-        openVersion = 0;
-      }
-      // Generate a random ID which is not already in meta db.
-      int id = -1;
-      // in general this should finish in a couple times at most. putting some
-      // arbitrary large number here to avoid dead loop.
-      for (int j = 0; j < 10000; j++) {
-        id = random.nextInt();
-        byte[] openKey = metadataManager.getOpenKeyNameBytes(objectKey, id);
-        if (metadataManager.get(openKey) == null) {
-          metadataManager.put(openKey, keyInfo.getProtobuf().toByteArray());
-          break;
-        }
-      }
-      if (id == -1) {
-        throw new IOException("Failed to find a usable id for " + objectKey);
-      }
-      LOG.debug("Key {} allocated in volume {} bucket {}",
-          keyName, volumeName, bucketName);
-      return new OpenKeySession(id, keyInfo, openVersion);
-    } catch (KSMException e) {
-      throw e;
-    } catch (IOException ex) {
-      if (!(ex instanceof KSMException)) {
-        LOG.error("Key open failed for volume:{} bucket:{} key:{}",
-            volumeName, bucketName, keyName, ex);
-      }
-      throw new KSMException(ex.getMessage(),
-          KSMException.ResultCodes.FAILED_KEY_ALLOCATION);
-    } finally {
-      metadataManager.writeLock().unlock();
-    }
-  }
-
-  @Override
-  public void commitKey(KsmKeyArgs args, int clientID) throws IOException {
-    Preconditions.checkNotNull(args);
-    metadataManager.writeLock().lock();
-    String volumeName = args.getVolumeName();
-    String bucketName = args.getBucketName();
-    String keyName = args.getKeyName();
-    try {
-      validateBucket(volumeName, bucketName);
-      String objectKey = metadataManager.getKeyWithDBPrefix(
-          volumeName, bucketName, keyName);
-      byte[] objectKeyBytes = metadataManager.getDBKeyBytes(volumeName,
-          bucketName, keyName);
-      byte[] openKey = metadataManager.getOpenKeyNameBytes(objectKey, clientID);
-      byte[] openKeyData = metadataManager.get(openKey);
-      if (openKeyData == null) {
-        throw new KSMException("Commit a key without corresponding entry " +
-            DFSUtil.bytes2String(openKey), ResultCodes.FAILED_KEY_NOT_FOUND);
-      }
-      KsmKeyInfo keyInfo =
-          KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(openKeyData));
-      keyInfo.setDataSize(args.getDataSize());
-      keyInfo.setModificationTime(Time.now());
-      BatchOperation batch = new BatchOperation();
-      batch.delete(openKey);
-      batch.put(objectKeyBytes, keyInfo.getProtobuf().toByteArray());
-      metadataManager.writeBatch(batch);
-    } catch (KSMException e) {
-      throw e;
-    } catch (IOException ex) {
-      LOG.error("Key commit failed for volume:{} bucket:{} key:{}",
-          volumeName, bucketName, keyName, ex);
-      throw new KSMException(ex.getMessage(),
-          KSMException.ResultCodes.FAILED_KEY_ALLOCATION);
-    } finally {
-      metadataManager.writeLock().unlock();
-    }
-  }
-
-  @Override
-  public KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException {
-    Preconditions.checkNotNull(args);
-    metadataManager.writeLock().lock();
-    String volumeName = args.getVolumeName();
-    String bucketName = args.getBucketName();
-    String keyName = args.getKeyName();
-    try {
-      byte[] keyKey = metadataManager.getDBKeyBytes(
-          volumeName, bucketName, keyName);
-      byte[] value = metadataManager.get(keyKey);
-      if (value == null) {
-        LOG.debug("volume:{} bucket:{} Key:{} not found",
-            volumeName, bucketName, keyName);
-        throw new KSMException("Key not found",
-            KSMException.ResultCodes.FAILED_KEY_NOT_FOUND);
-      }
-      return KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(value));
-    } catch (DBException ex) {
-      LOG.error("Get key failed for volume:{} bucket:{} key:{}",
-          volumeName, bucketName, keyName, ex);
-      throw new KSMException(ex.getMessage(),
-          KSMException.ResultCodes.FAILED_KEY_NOT_FOUND);
-    } finally {
-      metadataManager.writeLock().unlock();
-    }
-  }
-
-  @Override
-  public void deleteKey(KsmKeyArgs args) throws IOException {
-    Preconditions.checkNotNull(args);
-    metadataManager.writeLock().lock();
-    String volumeName = args.getVolumeName();
-    String bucketName = args.getBucketName();
-    String keyName = args.getKeyName();
-    try {
-      byte[] objectKey = metadataManager.getDBKeyBytes(
-          volumeName, bucketName, keyName);
-      byte[] objectValue = metadataManager.get(objectKey);
-      if (objectValue == null) {
-        throw new KSMException("Key not found",
-            KSMException.ResultCodes.FAILED_KEY_NOT_FOUND);
-      }
-      byte[] deletingKey = metadataManager.getDeletedKeyName(objectKey);
-      BatchOperation batch = new BatchOperation();
-      batch.put(deletingKey, objectValue);
-      batch.delete(objectKey);
-      metadataManager.writeBatch(batch);
-    } catch (DBException ex) {
-      LOG.error(String.format("Delete key failed for volume:%s "
-          + "bucket:%s key:%s", volumeName, bucketName, keyName), ex);
-      throw new KSMException(ex.getMessage(), ex,
-          ResultCodes.FAILED_KEY_DELETION);
-    } finally {
-      metadataManager.writeLock().unlock();
-    }
-  }
-
-  @Override
-  public List<KsmKeyInfo> listKeys(String volumeName, String bucketName,
-      String startKey, String keyPrefix, int maxKeys) throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(bucketName);
-
-    metadataManager.readLock().lock();
-    try {
-      return metadataManager.listKeys(volumeName, bucketName,
-          startKey, keyPrefix, maxKeys);
-    } finally {
-      metadataManager.readLock().unlock();
-    }
-  }
-
-  @Override
-  public List<BlockGroup> getPendingDeletionKeys(final int count)
-      throws IOException {
-    metadataManager.readLock().lock();
-    try {
-      return metadataManager.getPendingDeletionKeys(count);
-    } finally {
-      metadataManager.readLock().unlock();
-    }
-  }
-
-  @Override
-  public void deletePendingDeletionKey(String objectKeyName)
-      throws IOException{
-    Preconditions.checkNotNull(objectKeyName);
-    if (!objectKeyName.startsWith(OzoneConsts.DELETING_KEY_PREFIX)) {
-      throw new IllegalArgumentException("Invalid key name,"
-          + " the name should be the key name with deleting prefix");
-    }
-
-    // Simply removes the entry from KSM DB.
-    metadataManager.writeLock().lock();
-    try {
-      byte[] pendingDelKey = DFSUtil.string2Bytes(objectKeyName);
-      byte[] delKeyValue = metadataManager.get(pendingDelKey);
-      if (delKeyValue == null) {
-        throw new IOException("Failed to delete key " + objectKeyName
-            + " because it is not found in DB");
-      }
-      metadataManager.delete(pendingDelKey);
-    } finally {
-      metadataManager.writeLock().unlock();
-    }
-  }
-
-  @Override
-  public List<BlockGroup> getExpiredOpenKeys() throws IOException {
-    metadataManager.readLock().lock();
-    try {
-      return metadataManager.getExpiredOpenKeys();
-    } finally {
-      metadataManager.readLock().unlock();
-    }
-  }
-
-  @Override
-  public void deleteExpiredOpenKey(String objectKeyName) throws IOException {
-    Preconditions.checkNotNull(objectKeyName);
-    if (!objectKeyName.startsWith(OzoneConsts.OPEN_KEY_PREFIX)) {
-      throw new IllegalArgumentException("Invalid key name,"
-          + " the name should be the key name with open key prefix");
-    }
-
-    // Simply removes the entry from KSM DB.
-    metadataManager.writeLock().lock();
-    try {
-      byte[] openKey = DFSUtil.string2Bytes(objectKeyName);
-      byte[] delKeyValue = metadataManager.get(openKey);
-      if (delKeyValue == null) {
-        throw new IOException("Failed to delete key " + objectKeyName
-            + " because it is not found in DB");
-      }
-      metadataManager.delete(openKey);
-    } finally {
-      metadataManager.writeLock().unlock();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
deleted file mode 100644
index 6139292..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
+++ /dev/null
@@ -1,902 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.protobuf.BlockingService;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.jmx.ServiceRuntimeInfoImpl;
-import org.apache.hadoop.ozone.common.Storage.StorageState;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
-import org.apache.hadoop.ozone.ksm.protocol.KeySpaceManagerProtocol;
-import org.apache.hadoop.ozone.ksm.protocolPB.KeySpaceManagerProtocolPB;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException.ResultCodes;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos
-    .ServicePort;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.OzoneAclInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocolPB
-    .KeySpaceManagerProtocolServerSideTranslatorPB;
-import org.apache.hadoop.scm.ScmInfo;
-import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.scm.protocol.StorageContainerLocationProtocol;
-import org.apache.hadoop.scm.protocolPB
-    .ScmBlockLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.protocolPB.ScmBlockLocationProtocolPB;
-import org.apache.hadoop.scm.protocolPB
-    .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.ObjectName;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
-    .OZONE_KSM_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
-    .OZONE_KSM_HANDLER_COUNT_DEFAULT;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
-    .OZONE_KSM_HANDLER_COUNT_KEY;
-import static org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.KeySpaceManagerService
-    .newReflectiveBlockingService;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos
-    .NodeState.HEALTHY;
-import static org.apache.hadoop.util.ExitUtil.terminate;
-
-/**
- * Ozone Keyspace manager is the metadata manager of ozone.
- */
-@InterfaceAudience.LimitedPrivate({"HDFS", "CBLOCK", "OZONE", "HBASE"})
-public class KeySpaceManager extends ServiceRuntimeInfoImpl
-    implements KeySpaceManagerProtocol, KSMMXBean {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(KeySpaceManager.class);
-
-  private static final String USAGE =
-      "Usage: \n hdfs ksm [genericOptions] " + "[ "
-          + StartupOption.CREATEOBJECTSTORE.getName() + " ]\n " + "hdfs ksm [ "
-          + StartupOption.HELP.getName() + " ]\n";
-
-  /** Startup options. */
-  public enum StartupOption {
-    CREATEOBJECTSTORE("-createObjectStore"),
-    HELP("-help"),
-    REGULAR("-regular");
-
-    private final String name;
-
-    StartupOption(String arg) {
-      this.name = arg;
-    }
-
-    public String getName() {
-      return name;
-    }
-
-    public static StartupOption parse(String value) {
-      for (StartupOption option : StartupOption.values()) {
-        if (option.name.equalsIgnoreCase(value)) {
-          return option;
-        }
-      }
-      return null;
-    }
-  }
-
-  private final OzoneConfiguration configuration;
-  private final RPC.Server ksmRpcServer;
-  private final InetSocketAddress ksmRpcAddress;
-  private final KSMMetadataManager metadataManager;
-  private final VolumeManager volumeManager;
-  private final BucketManager bucketManager;
-  private final KeyManager keyManager;
-  private final KSMMetrics metrics;
-  private final KeySpaceManagerHttpServer httpServer;
-  private final KSMStorage ksmStorage;
-  private final ScmBlockLocationProtocol scmBlockClient;
-  private final StorageContainerLocationProtocol scmContainerClient;
-  private ObjectName ksmInfoBeanName;
-
-  private KeySpaceManager(OzoneConfiguration conf) throws IOException {
-    Preconditions.checkNotNull(conf);
-    configuration = conf;
-    ksmStorage = new KSMStorage(conf);
-    scmBlockClient = getScmBlockClient(configuration);
-    scmContainerClient = getScmContainerClient(configuration);
-    if (ksmStorage.getState() != StorageState.INITIALIZED) {
-      throw new KSMException("KSM not initialized.",
-          ResultCodes.KSM_NOT_INITIALIZED);
-    }
-
-    // verifies that the SCM info in the KSM Version file is correct.
-    ScmInfo scmInfo = scmBlockClient.getScmInfo();
-    if (!(scmInfo.getClusterId().equals(ksmStorage.getClusterID()) && scmInfo
-        .getScmId().equals(ksmStorage.getScmId()))) {
-      throw new KSMException("SCM version info mismatch.",
-          ResultCodes.SCM_VERSION_MISMATCH_ERROR);
-    }
-    final int handlerCount = conf.getInt(OZONE_KSM_HANDLER_COUNT_KEY,
-        OZONE_KSM_HANDLER_COUNT_DEFAULT);
-
-    RPC.setProtocolEngine(configuration, KeySpaceManagerProtocolPB.class,
-        ProtobufRpcEngine.class);
-
-    BlockingService ksmService = newReflectiveBlockingService(
-        new KeySpaceManagerProtocolServerSideTranslatorPB(this));
-    final InetSocketAddress ksmNodeRpcAddr = OzoneClientUtils.
-        getKsmAddress(configuration);
-    ksmRpcServer = startRpcServer(configuration, ksmNodeRpcAddr,
-        KeySpaceManagerProtocolPB.class, ksmService,
-        handlerCount);
-    ksmRpcAddress = OzoneClientUtils.updateRPCListenAddress(configuration,
-        OZONE_KSM_ADDRESS_KEY, ksmNodeRpcAddr, ksmRpcServer);
-    metadataManager = new KSMMetadataManagerImpl(configuration);
-    volumeManager = new VolumeManagerImpl(metadataManager, configuration);
-    bucketManager = new BucketManagerImpl(metadataManager);
-    metrics = KSMMetrics.create();
-    keyManager =
-        new KeyManagerImpl(scmBlockClient, metadataManager, configuration,
-            ksmStorage.getKsmId());
-    httpServer = new KeySpaceManagerHttpServer(configuration, this);
-  }
-
-  /**
-   * Create a scm block client, used by putKey() and getKey().
-   *
-   * @return {@link ScmBlockLocationProtocol}
-   * @throws IOException
-   */
-  private static ScmBlockLocationProtocol getScmBlockClient(
-      OzoneConfiguration conf) throws IOException {
-    RPC.setProtocolEngine(conf, ScmBlockLocationProtocolPB.class,
-        ProtobufRpcEngine.class);
-    long scmVersion =
-        RPC.getProtocolVersion(ScmBlockLocationProtocolPB.class);
-    InetSocketAddress scmBlockAddress =
-        OzoneClientUtils.getScmAddressForBlockClients(conf);
-    ScmBlockLocationProtocolClientSideTranslatorPB scmBlockLocationClient =
-        new ScmBlockLocationProtocolClientSideTranslatorPB(
-            RPC.getProxy(ScmBlockLocationProtocolPB.class, scmVersion,
-                scmBlockAddress, UserGroupInformation.getCurrentUser(), conf,
-                NetUtils.getDefaultSocketFactory(conf),
-                Client.getRpcTimeout(conf)));
-    return scmBlockLocationClient;
-  }
-
-  /**
-   * Returns a scm container client.
-   *
-   * @return {@link StorageContainerLocationProtocol}
-   * @throws IOException
-   */
-  private static StorageContainerLocationProtocol getScmContainerClient(
-      OzoneConfiguration conf) throws IOException {
-    RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class,
-        ProtobufRpcEngine.class);
-    long scmVersion =
-        RPC.getProtocolVersion(StorageContainerLocationProtocolPB.class);
-    InetSocketAddress scmAddr = OzoneClientUtils.getScmAddressForClients(
-        conf);
-    StorageContainerLocationProtocolClientSideTranslatorPB scmContainerClient =
-        new StorageContainerLocationProtocolClientSideTranslatorPB(
-            RPC.getProxy(StorageContainerLocationProtocolPB.class, scmVersion,
-                scmAddr, UserGroupInformation.getCurrentUser(), conf,
-                NetUtils.getDefaultSocketFactory(conf),
-                Client.getRpcTimeout(conf)));
-    return scmContainerClient;
-  }
-
-  @VisibleForTesting
-  public KeyManager getKeyManager() {
-    return keyManager;
-  }
-
-  @VisibleForTesting
-  public ScmInfo getScmInfo() throws IOException {
-    return scmBlockClient.getScmInfo();
-  }
-
-  @VisibleForTesting
-  public KSMStorage getKsmStorage() {
-    return ksmStorage;
-  }
-  /**
-   * Starts an RPC server, if configured.
-   *
-   * @param conf configuration
-   * @param addr configured address of RPC server
-   * @param protocol RPC protocol provided by RPC server
-   * @param instance RPC protocol implementation instance
-   * @param handlerCount RPC server handler count
-   *
-   * @return RPC server
-   * @throws IOException if there is an I/O error while creating RPC server
-   */
-  private static RPC.Server startRpcServer(OzoneConfiguration conf,
-      InetSocketAddress addr, Class<?> protocol, BlockingService instance,
-      int handlerCount) throws IOException {
-    RPC.Server rpcServer = new RPC.Builder(conf)
-        .setProtocol(protocol)
-        .setInstance(instance)
-        .setBindAddress(addr.getHostString())
-        .setPort(addr.getPort())
-        .setNumHandlers(handlerCount)
-        .setVerbose(false)
-        .setSecretManager(null)
-        .build();
-
-    DFSUtil.addPBProtocol(conf, protocol, instance, rpcServer);
-    return rpcServer;
-  }
-
-  /**
-   * Get metadata manager.
-   * @return metadata manager.
-   */
-  public KSMMetadataManager getMetadataManager() {
-    return metadataManager;
-  }
-
-  public KSMMetrics getMetrics() {
-    return metrics;
-  }
-
-  /**
-   * Main entry point for starting KeySpaceManager.
-   *
-   * @param argv arguments
-   * @throws IOException if startup fails due to I/O error
-   */
-  public static void main(String[] argv) throws IOException {
-    if (DFSUtil.parseHelpArgument(argv, USAGE, System.out, true)) {
-      System.exit(0);
-    }
-    try {
-      OzoneConfiguration conf = new OzoneConfiguration();
-      GenericOptionsParser hParser = new GenericOptionsParser(conf, argv);
-      if (!hParser.isParseSuccessful()) {
-        System.err.println("USAGE: " + USAGE + " \n");
-        hParser.printGenericCommandUsage(System.err);
-        System.exit(1);
-      }
-      StringUtils.startupShutdownMessage(KeySpaceManager.class, argv, LOG);
-      KeySpaceManager ksm = createKSM(hParser.getRemainingArgs(), conf);
-      if (ksm != null) {
-        ksm.start();
-        ksm.join();
-      }
-    } catch (Throwable t) {
-      LOG.error("Failed to start the KeyspaceManager.", t);
-      terminate(1, t);
-    }
-  }
-
-  private static void printUsage(PrintStream out) {
-    out.println(USAGE + "\n");
-  }
-
-  /**
-   * Constructs KSM instance based on command line arguments.
-   * @param argv Command line arguments
-   * @param conf OzoneConfiguration
-   * @return KSM instance
-   * @throws IOException in case KSM instance creation fails.
-   */
-
-  public static KeySpaceManager createKSM(String[] argv,
-      OzoneConfiguration conf) throws IOException {
-    if (!DFSUtil.isOzoneEnabled(conf)) {
-      System.err.println("KSM cannot be started in secure mode or when " +
-          OZONE_ENABLED + " is set to false");
-      System.exit(1);
-    }
-    StartupOption startOpt = parseArguments(argv);
-    if (startOpt == null) {
-      printUsage(System.err);
-      terminate(1);
-      return null;
-    }
-    switch (startOpt) {
-    case CREATEOBJECTSTORE:
-      terminate(ksmInit(conf) ? 0 : 1);
-      return null;
-    case HELP:
-      printUsage(System.err);
-      terminate(0);
-      return null;
-    default:
-      return new KeySpaceManager(conf);
-    }
-  }
-
-  /**
-   * Initializes the KSM instance.
-   * @param conf OzoneConfiguration
-   * @return true if KSM initialization succeeds , false otherwise
-   * @throws IOException in case ozone metadata directory path is not accessible
-   */
-
-  private static boolean ksmInit(OzoneConfiguration conf) throws IOException {
-    KSMStorage ksmStorage = new KSMStorage(conf);
-    StorageState state = ksmStorage.getState();
-    if (state != StorageState.INITIALIZED) {
-      try {
-        ScmBlockLocationProtocol scmBlockClient = getScmBlockClient(conf);
-        ScmInfo scmInfo = scmBlockClient.getScmInfo();
-        String clusterId = scmInfo.getClusterId();
-        String scmId = scmInfo.getScmId();
-        if (clusterId == null || clusterId.isEmpty()) {
-          throw new IOException("Invalid Cluster ID");
-        }
-        if (scmId == null || scmId.isEmpty()) {
-          throw new IOException("Invalid SCM ID");
-        }
-        ksmStorage.setClusterId(clusterId);
-        ksmStorage.setScmId(scmId);
-        ksmStorage.initialize();
-        System.out.println(
-            "KSM initialization succeeded.Current cluster id for sd="
-                + ksmStorage.getStorageDir() + ";cid=" + ksmStorage
-                .getClusterID());
-        return true;
-      } catch (IOException ioe) {
-        LOG.error("Could not initialize KSM version file", ioe);
-        return false;
-      }
-    } else {
-      System.out.println(
-          "KSM already initialized.Reusing existing cluster id for sd="
-              + ksmStorage.getStorageDir() + ";cid=" + ksmStorage
-              .getClusterID());
-      return true;
-    }
-  }
-
-  /**
-   * Parses the command line options for KSM initialization.
-   * @param args command line arguments
-   * @return StartupOption if options are valid, null otherwise
-   */
-  private static StartupOption parseArguments(String[] args) {
-    if (args == null || args.length == 0) {
-      return StartupOption.REGULAR;
-    } else if (args.length == 1) {
-      return StartupOption.parse(args[0]);
-    }
-    return null;
-  }
-
-  /**
-   * Builds a message for logging startup information about an RPC server.
-   *
-   * @param description RPC server description
-   * @param addr RPC server listening address
-   * @return server startup message
-   */
-  private static String buildRpcServerStartMessage(String description,
-      InetSocketAddress addr) {
-    return addr != null ? String.format("%s is listening at %s",
-        description, addr.toString()) :
-        String.format("%s not started", description);
-  }
-
-  /**
-   * Start service.
-   */
-  public void start() throws IOException {
-    LOG.info(buildRpcServerStartMessage("KeyspaceManager RPC server",
-        ksmRpcAddress));
-    DefaultMetricsSystem.initialize("KeySpaceManager");
-    metadataManager.start();
-    keyManager.start();
-    ksmRpcServer.start();
-    httpServer.start();
-    registerMXBean();
-    setStartTime();
-  }
-
-  /**
-   * Stop service.
-   */
-  public void stop() {
-    try {
-      metadataManager.stop();
-      ksmRpcServer.stop();
-      keyManager.stop();
-      httpServer.stop();
-      metrics.unRegister();
-      unregisterMXBean();
-    } catch (Exception e) {
-      LOG.error("Key Space Manager stop failed.", e);
-    }
-  }
-
-  /**
-   * Wait until service has completed shutdown.
-   */
-  public void join() {
-    try {
-      ksmRpcServer.join();
-    } catch (InterruptedException e) {
-      Thread.currentThread().interrupt();
-      LOG.info("Interrupted during KeyspaceManager join.", e);
-    }
-  }
-
-  /**
-   * Creates a volume.
-   *
-   * @param args - Arguments to create Volume.
-   * @throws IOException
-   */
-  @Override
-  public void createVolume(KsmVolumeArgs args) throws IOException {
-    try {
-      metrics.incNumVolumeCreates();
-      volumeManager.createVolume(args);
-    } catch (Exception ex) {
-      metrics.incNumVolumeCreateFails();
-      throw ex;
-    }
-  }
-
-  /**
-   * Changes the owner of a volume.
-   *
-   * @param volume - Name of the volume.
-   * @param owner - Name of the owner.
-   * @throws IOException
-   */
-  @Override
-  public void setOwner(String volume, String owner) throws IOException {
-    try {
-      metrics.incNumVolumeUpdates();
-      volumeManager.setOwner(volume, owner);
-    } catch (Exception ex) {
-      metrics.incNumVolumeUpdateFails();
-      throw ex;
-    }
-  }
-
-  /**
-   * Changes the Quota on a volume.
-   *
-   * @param volume - Name of the volume.
-   * @param quota - Quota in bytes.
-   * @throws IOException
-   */
-  @Override
-  public void setQuota(String volume, long quota) throws IOException {
-    try {
-      metrics.incNumVolumeUpdates();
-      volumeManager.setQuota(volume, quota);
-    } catch (Exception ex) {
-      metrics.incNumVolumeUpdateFails();
-      throw ex;
-    }
-  }
-
-  /**
-   * Checks if the specified user can access this volume.
-   *
-   * @param volume - volume
-   * @param userAcl - user acls which needs to be checked for access
-   * @return true if the user has required access for the volume,
-   *         false otherwise
-   * @throws IOException
-   */
-  @Override
-  public boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl)
-      throws IOException {
-    try {
-      metrics.incNumVolumeCheckAccesses();
-      return volumeManager.checkVolumeAccess(volume, userAcl);
-    } catch (Exception ex) {
-      metrics.incNumVolumeCheckAccessFails();
-      throw ex;
-    }
-  }
-
-  /**
-   * Gets the volume information.
-   *
-   * @param volume - Volume name.
-   * @return VolumeArgs or exception is thrown.
-   * @throws IOException
-   */
-  @Override
-  public KsmVolumeArgs getVolumeInfo(String volume) throws IOException {
-    try {
-      metrics.incNumVolumeInfos();
-      return volumeManager.getVolumeInfo(volume);
-    } catch (Exception ex) {
-      metrics.incNumVolumeInfoFails();
-      throw ex;
-    }
-  }
-
-  /**
-   * Deletes an existing empty volume.
-   *
-   * @param volume - Name of the volume.
-   * @throws IOException
-   */
-  @Override
-  public void deleteVolume(String volume) throws IOException {
-    try {
-      metrics.incNumVolumeDeletes();
-      volumeManager.deleteVolume(volume);
-    } catch (Exception ex) {
-      metrics.incNumVolumeDeleteFails();
-      throw ex;
-    }
-  }
-
-  /**
-   * Lists volume owned by a specific user.
-   *
-   * @param userName - user name
-   * @param prefix - Filter prefix -- Return only entries that match this.
-   * @param prevKey - Previous key -- List starts from the next from the
-   * prevkey
-   * @param maxKeys - Max number of keys to return.
-   * @return List of Volumes.
-   * @throws IOException
-   */
-  @Override
-  public List<KsmVolumeArgs> listVolumeByUser(String userName, String prefix,
-      String prevKey, int maxKeys) throws IOException {
-    try {
-      metrics.incNumVolumeLists();
-      return volumeManager.listVolumes(userName, prefix, prevKey, maxKeys);
-    } catch (Exception ex) {
-      metrics.incNumVolumeListFails();
-      throw ex;
-    }
-  }
-
-  /**
-   * Lists volume all volumes in the cluster.
-   *
-   * @param prefix - Filter prefix -- Return only entries that match this.
-   * @param prevKey - Previous key -- List starts from the next from the
-   * prevkey
-   * @param maxKeys - Max number of keys to return.
-   * @return List of Volumes.
-   * @throws IOException
-   */
-  @Override
-  public List<KsmVolumeArgs> listAllVolumes(String prefix, String prevKey, int
-      maxKeys) throws IOException {
-    try {
-      metrics.incNumVolumeLists();
-      return volumeManager.listVolumes(null, prefix, prevKey, maxKeys);
-    } catch (Exception ex) {
-      metrics.incNumVolumeListFails();
-      throw ex;
-    }
-  }
-
-  /**
-   * Creates a bucket.
-   *
-   * @param bucketInfo - BucketInfo to create bucket.
-   * @throws IOException
-   */
-  @Override
-  public void createBucket(KsmBucketInfo bucketInfo) throws IOException {
-    try {
-      metrics.incNumBucketCreates();
-      bucketManager.createBucket(bucketInfo);
-    } catch (Exception ex) {
-      metrics.incNumBucketCreateFails();
-      throw ex;
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public List<KsmBucketInfo> listBuckets(String volumeName,
-      String startKey, String prefix, int maxNumOfBuckets)
-      throws IOException {
-    try {
-      metrics.incNumBucketLists();
-      return bucketManager.listBuckets(volumeName,
-          startKey, prefix, maxNumOfBuckets);
-    } catch (IOException ex) {
-      metrics.incNumBucketListFails();
-      throw ex;
-    }
-  }
-
-  /**
-   * Gets the bucket information.
-   *
-   * @param volume - Volume name.
-   * @param bucket - Bucket name.
-   * @return KsmBucketInfo or exception is thrown.
-   * @throws IOException
-   */
-  @Override
-  public KsmBucketInfo getBucketInfo(String volume, String bucket)
-      throws IOException {
-    try {
-      metrics.incNumBucketInfos();
-      return bucketManager.getBucketInfo(volume, bucket);
-    } catch (Exception ex) {
-      metrics.incNumBucketInfoFails();
-      throw ex;
-    }
-  }
-
-  /**
-   * Allocate a key.
-   *
-   * @param args - attributes of the key.
-   * @return KsmKeyInfo - the info about the allocated key.
-   * @throws IOException
-   */
-  @Override
-  public OpenKeySession openKey(KsmKeyArgs args) throws IOException {
-    try {
-      metrics.incNumKeyAllocates();
-      return keyManager.openKey(args);
-    } catch (Exception ex) {
-      metrics.incNumKeyAllocateFails();
-      throw ex;
-    }
-  }
-
-  @Override
-  public void commitKey(KsmKeyArgs args, int clientID)
-      throws IOException {
-    try {
-      metrics.incNumKeyCommits();
-      keyManager.commitKey(args, clientID);
-    } catch (Exception ex) {
-      metrics.incNumKeyCommitFails();
-      throw ex;
-    }
-  }
-
-  @Override
-  public KsmKeyLocationInfo  allocateBlock(KsmKeyArgs args, int clientID)
-      throws IOException {
-    try {
-      metrics.incNumBlockAllocateCalls();
-      return keyManager.allocateBlock(args, clientID);
-    } catch (Exception ex) {
-      metrics.incNumBlockAllocateCallFails();
-      throw ex;
-    }
-  }
-
-  /**
-   * Lookup a key.
-   *
-   * @param args - attributes of the key.
-   * @return KsmKeyInfo - the info about the requested key.
-   * @throws IOException
-   */
-  @Override
-  public KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException {
-    try {
-      metrics.incNumKeyLookups();
-      return keyManager.lookupKey(args);
-    } catch (Exception ex) {
-      metrics.incNumKeyLookupFails();
-      throw ex;
-    }
-  }
-
-  /**
-   * Deletes an existing key.
-   *
-   * @param args - attributes of the key.
-   * @throws IOException
-   */
-  @Override
-  public void deleteKey(KsmKeyArgs args) throws IOException {
-    try {
-      metrics.incNumKeyDeletes();
-      keyManager.deleteKey(args);
-    } catch (Exception ex) {
-      metrics.incNumKeyDeleteFails();
-      throw ex;
-    }
-  }
-
-  @Override
-  public List<KsmKeyInfo> listKeys(String volumeName, String bucketName,
-      String startKey, String keyPrefix, int maxKeys) throws IOException {
-    try {
-      metrics.incNumKeyLists();
-      return keyManager.listKeys(volumeName, bucketName,
-          startKey, keyPrefix, maxKeys);
-    } catch (IOException ex) {
-      metrics.incNumKeyListFails();
-      throw ex;
-    }
-  }
-
-  /**
-   * Sets bucket property from args.
-   * @param args - BucketArgs.
-   * @throws IOException
-   */
-  @Override
-  public void setBucketProperty(KsmBucketArgs args)
-      throws IOException {
-    try {
-      metrics.incNumBucketUpdates();
-      bucketManager.setBucketProperty(args);
-    } catch (Exception ex) {
-      metrics.incNumBucketUpdateFails();
-      throw ex;
-    }
-  }
-
-
-  /**
-   * Deletes an existing empty bucket from volume.
-   * @param volume - Name of the volume.
-   * @param bucket - Name of the bucket.
-   * @throws IOException
-   */
-  public void deleteBucket(String volume, String bucket) throws IOException {
-    try {
-      metrics.incNumBucketDeletes();
-      bucketManager.deleteBucket(volume, bucket);
-    } catch (Exception ex) {
-      metrics.incNumBucketDeleteFails();
-      throw ex;
-    }
-  }
-
-  private void registerMXBean() {
-    Map<String, String> jmxProperties = new HashMap<String, String>();
-    jmxProperties.put("component", "ServerRuntime");
-    this.ksmInfoBeanName =
-        MBeans.register("KeySpaceManager",
-            "KeySpaceManagerInfo",
-            jmxProperties,
-            this);
-  }
-
-  private void unregisterMXBean() {
-    if (this.ksmInfoBeanName != null) {
-      MBeans.unregister(this.ksmInfoBeanName);
-      this.ksmInfoBeanName = null;
-    }
-  }
-
-  @Override
-  public String getRpcPort() {
-    return "" + ksmRpcAddress.getPort();
-  }
-
-  @VisibleForTesting
-  public KeySpaceManagerHttpServer getHttpServer() {
-    return httpServer;
-  }
-
-  @Override
-  public List<ServiceInfo> getServiceList() throws IOException {
-    // When we implement multi-home this call has to be handled properly.
-    List<ServiceInfo> services = new ArrayList<>();
-    ServiceInfo.Builder ksmServiceInfoBuilder = ServiceInfo.newBuilder()
-        .setNodeType(OzoneProtos.NodeType.KSM)
-        .setHostname(ksmRpcAddress.getHostName())
-        .addServicePort(ServicePort.newBuilder()
-                .setType(ServicePort.Type.RPC)
-                .setValue(ksmRpcAddress.getPort())
-            .build());
-    if (httpServer.getHttpAddress() != null) {
-      ksmServiceInfoBuilder.addServicePort(ServicePort.newBuilder()
-          .setType(ServicePort.Type.HTTP)
-          .setValue(httpServer.getHttpAddress().getPort())
-          .build());
-    }
-    if (httpServer.getHttpsAddress() != null) {
-      ksmServiceInfoBuilder.addServicePort(ServicePort.newBuilder()
-          .setType(ServicePort.Type.HTTPS)
-          .setValue(httpServer.getHttpsAddress().getPort())
-          .build());
-    }
-    services.add(ksmServiceInfoBuilder.build());
-
-    // For client we have to return SCM with container protocol port,
-    // not block protocol.
-    InetSocketAddress scmAddr = OzoneClientUtils.getScmAddressForClients(
-        configuration);
-    ServiceInfo.Builder scmServiceInfoBuilder = ServiceInfo.newBuilder()
-        .setNodeType(OzoneProtos.NodeType.SCM)
-        .setHostname(scmAddr.getHostName())
-        .addServicePort(ServicePort.newBuilder()
-            .setType(ServicePort.Type.RPC)
-            .setValue(scmAddr.getPort()).build());
-    services.add(scmServiceInfoBuilder.build());
-
-    List<OzoneProtos.Node> nodes = scmContainerClient.queryNode(
-        EnumSet.of(HEALTHY), OzoneProtos.QueryScope.CLUSTER, "")
-        .getNodesList();
-
-    for (OzoneProtos.Node node : nodes) {
-      HdfsProtos.DatanodeIDProto datanode = node.getNodeID();
-
-      ServiceInfo.Builder dnServiceInfoBuilder = ServiceInfo.newBuilder()
-          .setNodeType(OzoneProtos.NodeType.DATANODE)
-          .setHostname(datanode.getHostName());
-
-      dnServiceInfoBuilder.addServicePort(ServicePort.newBuilder()
-          .setType(ServicePort.Type.HTTP)
-          .setValue(datanode.getInfoPort())
-          .build());
-
-      if (datanode.hasInfoSecurePort() && datanode.getInfoSecurePort() > 0) {
-        dnServiceInfoBuilder.addServicePort(ServicePort.newBuilder()
-            .setType(ServicePort.Type.HTTPS)
-            .setValue(datanode.getInfoSecurePort())
-            .build());
-      }
-      services.add(dnServiceInfoBuilder.build());
-    }
-
-    metrics.incNumGetServiceLists();
-    // For now there is no exception that can can happen in this call,
-    // so failure metrics is not handled. In future if there is any need to
-    // handle exception in this method, we need to incorporate
-    // metrics.incNumGetServiceListFails()
-    return services;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManagerHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManagerHttpServer.java
deleted file mode 100644
index b229c89..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManagerHttpServer.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.web.OzoneHttpServer;
-
-import java.io.IOException;
-
-/**
- * HttpServer wrapper for the KeySpaceManager.
- */
-public class KeySpaceManagerHttpServer extends OzoneHttpServer {
-
-  public KeySpaceManagerHttpServer(Configuration conf, KeySpaceManager ksm)
-      throws IOException {
-    super(conf, "ksm");
-    addServlet("serviceList", "/serviceList", ServiceListJSONServlet.class);
-    getWebAppContext().setAttribute(OzoneConsts.KSM_CONTEXT_ATTRIBUTE, ksm);
-  }
-
-  @Override protected String getHttpAddressKey() {
-    return KSMConfigKeys.OZONE_KSM_HTTP_ADDRESS_KEY;
-  }
-
-  @Override protected String getHttpBindHostKey() {
-    return KSMConfigKeys.OZONE_KSM_HTTP_BIND_HOST_KEY;
-  }
-
-  @Override protected String getHttpsAddressKey() {
-    return KSMConfigKeys.OZONE_KSM_HTTPS_ADDRESS_KEY;
-  }
-
-  @Override protected String getHttpsBindHostKey() {
-    return KSMConfigKeys.OZONE_KSM_HTTPS_BIND_HOST_KEY;
-  }
-
-  @Override protected String getBindHostDefault() {
-    return KSMConfigKeys.OZONE_KSM_HTTP_BIND_HOST_DEFAULT;
-  }
-
-  @Override protected int getHttpBindPortDefault() {
-    return KSMConfigKeys.OZONE_KSM_HTTP_BIND_PORT_DEFAULT;
-  }
-
-  @Override protected int getHttpsBindPortDefault() {
-    return KSMConfigKeys.OZONE_KSM_HTTPS_BIND_PORT_DEFAULT;
-  }
-
-  @Override protected String getKeytabFile() {
-    return KSMConfigKeys.OZONE_KSM_KEYTAB_FILE;
-  }
-
-  @Override protected String getSpnegoPrincipal() {
-    return OzoneConfigKeys.OZONE_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL;
-  }
-
-  @Override protected String getEnabledKey() {
-    return KSMConfigKeys.OZONE_KSM_HTTP_ENABLED_KEY;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/OpenKeyCleanupService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/OpenKeyCleanupService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/OpenKeyCleanupService.java
deleted file mode 100644
index 7f60bf8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/OpenKeyCleanupService.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
-import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.utils.BackgroundService;
-import org.apache.hadoop.utils.BackgroundTask;
-import org.apache.hadoop.utils.BackgroundTaskQueue;
-import org.apache.hadoop.utils.BackgroundTaskResult;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-/**
- * This is the background service to delete hanging open keys.
- * Scan the metadata of ksm periodically to get
- * the keys with prefix "#open#" and ask scm to
- * delete metadata accordingly, if scm returns
- * success for keys, then clean up those keys.
- */
-public class OpenKeyCleanupService extends BackgroundService {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(OpenKeyCleanupService.class);
-
-  private final static int OPEN_KEY_DELETING_CORE_POOL_SIZE = 2;
-
-  private final KeyManager keyManager;
-  private final ScmBlockLocationProtocol scmClient;
-
-  public OpenKeyCleanupService(ScmBlockLocationProtocol scmClient,
-      KeyManager keyManager, int serviceInterval,
-      long serviceTimeout) {
-    super("OpenKeyCleanupService", serviceInterval, TimeUnit.SECONDS,
-        OPEN_KEY_DELETING_CORE_POOL_SIZE, serviceTimeout);
-    this.keyManager = keyManager;
-    this.scmClient = scmClient;
-  }
-
-  @Override
-  public BackgroundTaskQueue getTasks() {
-    BackgroundTaskQueue queue = new BackgroundTaskQueue();
-    queue.add(new OpenKeyDeletingTask());
-    return queue;
-  }
-
-  private class OpenKeyDeletingTask
-      implements BackgroundTask<BackgroundTaskResult> {
-
-    @Override
-    public int getPriority() {
-      return 0;
-    }
-
-    @Override
-    public BackgroundTaskResult call() throws Exception {
-      try {
-        List<BlockGroup> keyBlocksList = keyManager.getExpiredOpenKeys();
-        if (keyBlocksList.size() > 0) {
-          int toDeleteSize = keyBlocksList.size();
-          LOG.debug("Found {} to-delete open keys in KSM", toDeleteSize);
-          List<DeleteBlockGroupResult> results =
-              scmClient.deleteKeyBlocks(keyBlocksList);
-          int deletedSize = 0;
-          for (DeleteBlockGroupResult result : results) {
-            if (result.isSuccess()) {
-              try {
-                keyManager.deleteExpiredOpenKey(result.getObjectKey());
-                LOG.debug("Key {} deleted from KSM DB", result.getObjectKey());
-                deletedSize += 1;
-              } catch (IOException e) {
-                LOG.warn("Failed to delete hanging-open key {}",
-                    result.getObjectKey(), e);
-              }
-            } else {
-              LOG.warn("Deleting open Key {} failed because some of the blocks"
-                      + " were failed to delete, failed blocks: {}",
-                  result.getObjectKey(),
-                  String.join(",", result.getFailedBlocks()));
-            }
-          }
-          LOG.info("Found {} expired open key entries, successfully " +
-              "cleaned up {} entries", toDeleteSize, deletedSize);
-          return results::size;
-        } else {
-          LOG.debug("No hanging open key fond in KSM");
-        }
-      } catch (IOException e) {
-        LOG.error("Unable to get hanging open keys, retry in"
-            + " next interval", e);
-      }
-      return BackgroundTaskResult.EmptyTaskResult.newResult();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/ServiceListJSONServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/ServiceListJSONServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/ServiceListJSONServlet.java
deleted file mode 100644
index 34a80ce..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/ServiceListJSONServlet.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-import java.io.PrintWriter;
-
-
-/**
- * Provides REST access to Ozone Service List.
- * <p>
- * This servlet generally will be placed under the /serviceList URL of
- * KeySpaceManager HttpServer.
- *
- * The return format is of JSON and in the form
- * <p>
- *  <code><pre>
- *  {
- *    "services" : [
- *      {
- *        "NodeType":"KSM",
- *        "Hostname" "$hostname",
- *        "ports" : {
- *          "$PortType" : "$port",
- *          ...
- *        }
- *      }
- *    ]
- *  }
- *  </pre></code>
- *  <p>
- *
- */
-public class ServiceListJSONServlet  extends HttpServlet  {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ServiceListJSONServlet.class);
-  private static final long serialVersionUID = 1L;
-
-  private KeySpaceManager ksm;
-
-  public void init() throws ServletException {
-    this.ksm = (KeySpaceManager) getServletContext()
-        .getAttribute(OzoneConsts.KSM_CONTEXT_ATTRIBUTE);
-  }
-
-  /**
-   * Process a GET request for the specified resource.
-   *
-   * @param request
-   *          The servlet request we are processing
-   * @param response
-   *          The servlet response we are creating
-   */
-  @Override
-  public void doGet(HttpServletRequest request, HttpServletResponse response) {
-    try {
-      ObjectMapper objectMapper = new ObjectMapper();
-      objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
-      response.setContentType("application/json; charset=utf8");
-      PrintWriter writer = response.getWriter();
-      try {
-        writer.write(objectMapper.writeValueAsString(ksm.getServiceList()));
-      } finally {
-        if (writer != null) {
-          writer.close();
-        }
-      }
-    } catch (IOException e) {
-      LOG.error(
-          "Caught an exception while processing ServiceList request", e);
-      response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManager.java
deleted file mode 100644
index 6ac78d6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManager.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.OzoneAclInfo;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * KSM volume manager interface.
- */
-public interface VolumeManager {
-
-  /**
-   * Create a new volume.
-   * @param args - Volume args to create a volume
-   */
-  void createVolume(KsmVolumeArgs args) throws IOException;
-
-  /**
-   * Changes the owner of a volume.
-   *
-   * @param volume - Name of the volume.
-   * @param owner - Name of the owner.
-   * @throws IOException
-   */
-  void setOwner(String volume, String owner) throws IOException;
-
-  /**
-   * Changes the Quota on a volume.
-   *
-   * @param volume - Name of the volume.
-   * @param quota - Quota in bytes.
-   * @throws IOException
-   */
-  void setQuota(String volume, long quota) throws IOException;
-
-  /**
-   * Gets the volume information.
-   * @param volume - Volume name.
-   * @return VolumeArgs or exception is thrown.
-   * @throws IOException
-   */
-  KsmVolumeArgs getVolumeInfo(String volume) throws IOException;
-
-  /**
-   * Deletes an existing empty volume.
-   *
-   * @param volume - Name of the volume.
-   * @throws IOException
-   */
-  void deleteVolume(String volume) throws IOException;
-
-  /**
-   * Checks if the specified user with a role can access this volume.
-   *
-   * @param volume - volume
-   * @param userAcl - user acl which needs to be checked for access
-   * @return true if the user has access for the volume, false otherwise
-   * @throws IOException
-   */
-  boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl)
-      throws IOException;
-
-  /**
-   * Returns a list of volumes owned by a given user; if user is null,
-   * returns all volumes.
-   *
-   * @param userName
-   *   volume owner
-   * @param prefix
-   *   the volume prefix used to filter the listing result.
-   * @param startKey
-   *   the start volume name determines where to start listing from,
-   *   this key is excluded from the result.
-   * @param maxKeys
-   *   the maximum number of volumes to return.
-   * @return a list of {@link KsmVolumeArgs}
-   * @throws IOException
-   */
-  List<KsmVolumeArgs> listVolumes(String userName, String prefix,
-      String startKey, int maxKeys) throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java
deleted file mode 100644
index e9b094f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java
+++ /dev/null
@@ -1,391 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.OzoneAclInfo;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.VolumeList;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.VolumeInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.BatchOperation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
-    .OZONE_KSM_USER_MAX_VOLUME_DEFAULT;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
-    .OZONE_KSM_USER_MAX_VOLUME;
-import static org.apache.hadoop.ozone.ksm.exceptions
-    .KSMException.ResultCodes;
-
-/**
- * KSM volume management code.
- */
-public class VolumeManagerImpl implements VolumeManager {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(VolumeManagerImpl.class);
-
-  private final KSMMetadataManager metadataManager;
-  private final int maxUserVolumeCount;
-
-  /**
-   * Constructor.
-   * @param conf - Ozone configuration.
-   * @throws IOException
-   */
-  public VolumeManagerImpl(KSMMetadataManager metadataManager,
-      OzoneConfiguration conf) throws IOException {
-    this.metadataManager = metadataManager;
-    this.maxUserVolumeCount = conf.getInt(OZONE_KSM_USER_MAX_VOLUME,
-        OZONE_KSM_USER_MAX_VOLUME_DEFAULT);
-  }
-
-  // Helpers to add and delete volume from user list
-  private void addVolumeToOwnerList(String volume, String owner,
-      BatchOperation batchOperation) throws IOException {
-    // Get the volume list
-    byte[] dbUserKey = metadataManager.getUserKey(owner);
-    byte[] volumeList  = metadataManager.get(dbUserKey);
-    List<String> prevVolList = new LinkedList<>();
-    if (volumeList != null) {
-      VolumeList vlist = VolumeList.parseFrom(volumeList);
-      prevVolList.addAll(vlist.getVolumeNamesList());
-    }
-
-    // Check the volume count
-    if (prevVolList.size() >= maxUserVolumeCount) {
-      LOG.debug("Too many volumes for user:{}", owner);
-      throw new KSMException(ResultCodes.FAILED_TOO_MANY_USER_VOLUMES);
-    }
-
-    // Add the new volume to the list
-    prevVolList.add(volume);
-    VolumeList newVolList = VolumeList.newBuilder()
-        .addAllVolumeNames(prevVolList).build();
-    batchOperation.put(dbUserKey, newVolList.toByteArray());
-  }
-
-  private void delVolumeFromOwnerList(String volume, String owner,
-                                      BatchOperation batchOperation)
-      throws IOException {
-    // Get the volume list
-    byte[] dbUserKey = metadataManager.getUserKey(owner);
-    byte[] volumeList  = metadataManager.get(dbUserKey);
-    List<String> prevVolList = new LinkedList<>();
-    if (volumeList != null) {
-      VolumeList vlist = VolumeList.parseFrom(volumeList);
-      prevVolList.addAll(vlist.getVolumeNamesList());
-    } else {
-      LOG.debug("volume:{} not found for user:{}");
-      throw new KSMException(ResultCodes.FAILED_USER_NOT_FOUND);
-    }
-
-    // Remove the volume from the list
-    prevVolList.remove(volume);
-    if (prevVolList.size() == 0) {
-      batchOperation.delete(dbUserKey);
-    } else {
-      VolumeList newVolList = VolumeList.newBuilder()
-          .addAllVolumeNames(prevVolList).build();
-      batchOperation.put(dbUserKey, newVolList.toByteArray());
-    }
-  }
-
-  /**
-   * Creates a volume.
-   * @param args - KsmVolumeArgs.
-   */
-  @Override
-  public void createVolume(KsmVolumeArgs args) throws IOException {
-    Preconditions.checkNotNull(args);
-    metadataManager.writeLock().lock();
-    try {
-      byte[] dbVolumeKey = metadataManager.getVolumeKey(args.getVolume());
-      byte[] volumeInfo = metadataManager.get(dbVolumeKey);
-
-      // Check of the volume already exists
-      if (volumeInfo != null) {
-        LOG.debug("volume:{} already exists", args.getVolume());
-        throw new KSMException(ResultCodes.FAILED_VOLUME_ALREADY_EXISTS);
-      }
-
-      BatchOperation batch = new BatchOperation();
-      // Write the vol info
-      List<OzoneProtos.KeyValue> metadataList = new LinkedList<>();
-      for (Map.Entry<String, String> entry : args.getKeyValueMap().entrySet()) {
-        metadataList.add(OzoneProtos.KeyValue.newBuilder()
-            .setKey(entry.getKey()).setValue(entry.getValue()).build());
-      }
-      List<OzoneAclInfo> aclList = args.getAclMap().ozoneAclGetProtobuf();
-
-      VolumeInfo newVolumeInfo = VolumeInfo.newBuilder()
-          .setAdminName(args.getAdminName())
-          .setOwnerName(args.getOwnerName())
-          .setVolume(args.getVolume())
-          .setQuotaInBytes(args.getQuotaInBytes())
-          .addAllMetadata(metadataList)
-          .addAllVolumeAcls(aclList)
-          .setCreationTime(Time.now())
-          .build();
-      batch.put(dbVolumeKey, newVolumeInfo.toByteArray());
-
-      // Add volume to user list
-      addVolumeToOwnerList(args.getVolume(), args.getOwnerName(), batch);
-      metadataManager.writeBatch(batch);
-      LOG.debug("created volume:{} user:{}", args.getVolume(),
-          args.getOwnerName());
-    } catch (IOException ex) {
-      if (!(ex instanceof KSMException)) {
-        LOG.error("Volume creation failed for user:{} volume:{}",
-            args.getOwnerName(), args.getVolume(), ex);
-      }
-      throw ex;
-    } finally {
-      metadataManager.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Changes the owner of a volume.
-   *
-   * @param volume - Name of the volume.
-   * @param owner - Name of the owner.
-   * @throws IOException
-   */
-  @Override
-  public void setOwner(String volume, String owner) throws IOException {
-    Preconditions.checkNotNull(volume);
-    Preconditions.checkNotNull(owner);
-    metadataManager.writeLock().lock();
-    try {
-      byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
-      byte[] volInfo = metadataManager.get(dbVolumeKey);
-      if (volInfo == null) {
-        LOG.debug("Changing volume ownership failed for user:{} volume:{}",
-            owner, volume);
-        throw  new KSMException(ResultCodes.FAILED_VOLUME_NOT_FOUND);
-      }
-
-      VolumeInfo volumeInfo = VolumeInfo.parseFrom(volInfo);
-      KsmVolumeArgs volumeArgs = KsmVolumeArgs.getFromProtobuf(volumeInfo);
-      Preconditions.checkState(volume.equals(volumeInfo.getVolume()));
-
-      BatchOperation batch = new BatchOperation();
-      delVolumeFromOwnerList(volume, volumeArgs.getOwnerName(), batch);
-      addVolumeToOwnerList(volume, owner, batch);
-
-      KsmVolumeArgs newVolumeArgs =
-          KsmVolumeArgs.newBuilder().setVolume(volumeArgs.getVolume())
-              .setAdminName(volumeArgs.getAdminName())
-              .setOwnerName(owner)
-              .setQuotaInBytes(volumeArgs.getQuotaInBytes())
-              .setCreationTime(volumeArgs.getCreationTime())
-              .build();
-
-      VolumeInfo newVolumeInfo = newVolumeArgs.getProtobuf();
-      batch.put(dbVolumeKey, newVolumeInfo.toByteArray());
-
-      metadataManager.writeBatch(batch);
-    } catch (IOException ex) {
-      if (!(ex instanceof KSMException)) {
-        LOG.error("Changing volume ownership failed for user:{} volume:{}",
-            owner, volume, ex);
-      }
-      throw ex;
-    } finally {
-      metadataManager.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Changes the Quota on a volume.
-   *
-   * @param volume - Name of the volume.
-   * @param quota - Quota in bytes.
-   * @throws IOException
-   */
-  public void setQuota(String volume, long quota) throws IOException {
-    Preconditions.checkNotNull(volume);
-    metadataManager.writeLock().lock();
-    try {
-      byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
-      byte[] volInfo = metadataManager.get(dbVolumeKey);
-      if (volInfo == null) {
-        LOG.debug("volume:{} does not exist", volume);
-        throw new KSMException(ResultCodes.FAILED_VOLUME_NOT_FOUND);
-      }
-
-      VolumeInfo volumeInfo = VolumeInfo.parseFrom(volInfo);
-      KsmVolumeArgs volumeArgs = KsmVolumeArgs.getFromProtobuf(volumeInfo);
-      Preconditions.checkState(volume.equals(volumeInfo.getVolume()));
-
-      KsmVolumeArgs newVolumeArgs =
-          KsmVolumeArgs.newBuilder()
-              .setVolume(volumeArgs.getVolume())
-              .setAdminName(volumeArgs.getAdminName())
-              .setOwnerName(volumeArgs.getOwnerName())
-              .setQuotaInBytes(quota)
-              .setCreationTime(volumeArgs.getCreationTime()).build();
-
-      VolumeInfo newVolumeInfo = newVolumeArgs.getProtobuf();
-      metadataManager.put(dbVolumeKey, newVolumeInfo.toByteArray());
-    } catch (IOException ex) {
-      if (!(ex instanceof KSMException)) {
-        LOG.error("Changing volume quota failed for volume:{} quota:{}", volume,
-            quota, ex);
-      }
-      throw ex;
-    } finally {
-      metadataManager.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Gets the volume information.
-   * @param volume - Volume name.
-   * @return VolumeArgs or exception is thrown.
-   * @throws IOException
-   */
-  public KsmVolumeArgs getVolumeInfo(String volume) throws IOException {
-    Preconditions.checkNotNull(volume);
-    metadataManager.readLock().lock();
-    try {
-      byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
-      byte[] volInfo = metadataManager.get(dbVolumeKey);
-      if (volInfo == null) {
-        LOG.debug("volume:{} does not exist", volume);
-        throw new KSMException(ResultCodes.FAILED_VOLUME_NOT_FOUND);
-      }
-
-      VolumeInfo volumeInfo = VolumeInfo.parseFrom(volInfo);
-      KsmVolumeArgs volumeArgs = KsmVolumeArgs.getFromProtobuf(volumeInfo);
-      Preconditions.checkState(volume.equals(volumeInfo.getVolume()));
-      return volumeArgs;
-    } catch (IOException ex) {
-      if (!(ex instanceof KSMException)) {
-        LOG.warn("Info volume failed for volume:{}", volume, ex);
-      }
-      throw ex;
-    } finally {
-      metadataManager.readLock().unlock();
-    }
-  }
-
-  /**
-   * Deletes an existing empty volume.
-   *
-   * @param volume - Name of the volume.
-   * @throws IOException
-   */
-  @Override
-  public void deleteVolume(String volume) throws IOException {
-    Preconditions.checkNotNull(volume);
-    metadataManager.writeLock().lock();
-    try {
-      BatchOperation batch = new BatchOperation();
-      byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
-      byte[] volInfo = metadataManager.get(dbVolumeKey);
-      if (volInfo == null) {
-        LOG.debug("volume:{} does not exist", volume);
-        throw new KSMException(ResultCodes.FAILED_VOLUME_NOT_FOUND);
-      }
-
-      if (!metadataManager.isVolumeEmpty(volume)) {
-        LOG.debug("volume:{} is not empty", volume);
-        throw new KSMException(ResultCodes.FAILED_VOLUME_NOT_EMPTY);
-      }
-
-      VolumeInfo volumeInfo = VolumeInfo.parseFrom(volInfo);
-      Preconditions.checkState(volume.equals(volumeInfo.getVolume()));
-      // delete the volume from the owner list
-      // as well as delete the volume entry
-      delVolumeFromOwnerList(volume, volumeInfo.getOwnerName(), batch);
-      batch.delete(dbVolumeKey);
-      metadataManager.writeBatch(batch);
-    } catch (IOException ex) {
-      if (!(ex instanceof KSMException)) {
-        LOG.error("Delete volume failed for volume:{}", volume, ex);
-      }
-      throw ex;
-    } finally {
-      metadataManager.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Checks if the specified user with a role can access this volume.
-   *
-   * @param volume - volume
-   * @param userAcl - user acl which needs to be checked for access
-   * @return true if the user has access for the volume, false otherwise
-   * @throws IOException
-   */
-  public boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl)
-      throws IOException {
-    Preconditions.checkNotNull(volume);
-    Preconditions.checkNotNull(userAcl);
-    metadataManager.readLock().lock();
-    try {
-      byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
-      byte[] volInfo = metadataManager.get(dbVolumeKey);
-      if (volInfo == null) {
-        LOG.debug("volume:{} does not exist", volume);
-        throw  new KSMException(ResultCodes.FAILED_VOLUME_NOT_FOUND);
-      }
-
-      VolumeInfo volumeInfo = VolumeInfo.parseFrom(volInfo);
-      KsmVolumeArgs volumeArgs = KsmVolumeArgs.getFromProtobuf(volumeInfo);
-      Preconditions.checkState(volume.equals(volumeInfo.getVolume()));
-      return volumeArgs.getAclMap().hasAccess(userAcl);
-    } catch (IOException ex) {
-      if (!(ex instanceof KSMException)) {
-        LOG.error("Check volume access failed for volume:{} user:{} rights:{}",
-            volume, userAcl.getName(), userAcl.getRights(), ex);
-      }
-      throw ex;
-    } finally {
-      metadataManager.readLock().unlock();
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public List<KsmVolumeArgs> listVolumes(String userName,
-      String prefix, String startKey, int maxKeys) throws IOException {
-    metadataManager.readLock().lock();
-    try {
-      return metadataManager.listVolumes(
-          userName, prefix, startKey, maxKeys);
-    } finally {
-      metadataManager.readLock().unlock();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java
deleted file mode 100644
index e2f3580..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.exceptions;
-
-import java.io.IOException;
-
-/**
- * Exception thrown by KSM.
- */
-public class KSMException extends IOException {
-  private final KSMException.ResultCodes result;
-
-  /**
-   * Constructs an {@code IOException} with {@code null}
-   * as its error detail message.
-   */
-  public KSMException(KSMException.ResultCodes result) {
-    this.result = result;
-  }
-
-  /**
-   * Constructs an {@code IOException} with the specified detail message.
-   *
-   * @param message The detail message (which is saved for later retrieval by
-   * the
-   * {@link #getMessage()} method)
-   */
-  public KSMException(String message, KSMException.ResultCodes result) {
-    super(message);
-    this.result = result;
-  }
-
-  /**
-   * Constructs an {@code IOException} with the specified detail message
-   * and cause.
-   * <p>
-   * <p> Note that the detail message associated with {@code cause} is
-   * <i>not</i> automatically incorporated into this exception's detail
-   * message.
-   *
-   * @param message The detail message (which is saved for later retrieval by
-   * the
-   * {@link #getMessage()} method)
-   * @param cause The cause (which is saved for later retrieval by the {@link
-   * #getCause()} method).  (A null value is permitted, and indicates that the
-   * cause is nonexistent or unknown.)
-   * @since 1.6
-   */
-  public KSMException(String message, Throwable cause,
-                      KSMException.ResultCodes result) {
-    super(message, cause);
-    this.result = result;
-  }
-
-  /**
-   * Constructs an {@code IOException} with the specified cause and a
-   * detail message of {@code (cause==null ? null : cause.toString())}
-   * (which typically contains the class and detail message of {@code cause}).
-   * This constructor is useful for IO exceptions that are little more
-   * than wrappers for other throwables.
-   *
-   * @param cause The cause (which is saved for later retrieval by the {@link
-   * #getCause()} method).  (A null value is permitted, and indicates that the
-   * cause is nonexistent or unknown.)
-   * @since 1.6
-   */
-  public KSMException(Throwable cause, KSMException.ResultCodes result) {
-    super(cause);
-    this.result = result;
-  }
-
-  /**
-   * Returns resultCode.
-   * @return ResultCode
-   */
-  public KSMException.ResultCodes getResult() {
-    return result;
-  }
-
-  /**
-   * Error codes to make it easy to decode these exceptions.
-   */
-  public enum ResultCodes {
-    FAILED_TOO_MANY_USER_VOLUMES,
-    FAILED_VOLUME_ALREADY_EXISTS,
-    FAILED_VOLUME_NOT_FOUND,
-    FAILED_VOLUME_NOT_EMPTY,
-    FAILED_USER_NOT_FOUND,
-    FAILED_BUCKET_ALREADY_EXISTS,
-    FAILED_BUCKET_NOT_FOUND,
-    FAILED_BUCKET_NOT_EMPTY,
-    FAILED_KEY_ALREADY_EXISTS,
-    FAILED_KEY_NOT_FOUND,
-    FAILED_KEY_ALLOCATION,
-    FAILED_KEY_DELETION,
-    FAILED_METADATA_ERROR,
-    FAILED_INTERNAL_ERROR,
-    KSM_NOT_INITIALIZED,
-    SCM_VERSION_MISMATCH_ERROR
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/package-info.java
deleted file mode 100644
index 09fd87f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.ksm.exceptions;
-// Exception thrown by KSM.


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[23/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java
deleted file mode 100644
index 3eb6dc1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java
+++ /dev/null
@@ -1,1260 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
-import com.google.protobuf.BlockingService;
-import com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.jmx.ServiceRuntimeInfoImpl;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
-import org.apache.hadoop.ozone.common.Storage.StorageState;
-import org.apache.hadoop.ozone.common.StorageInfo;
-import org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol;
-import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
-import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
-import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMNodeAddressList;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMRegisteredCmdResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMReregisterCmdResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SendContainerReportProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.Type;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto.DeleteBlockTransactionResult;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
-import org.apache.hadoop.ozone.protocolPB.ScmBlockLocationProtocolServerSideTranslatorPB;
-import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolPB;
-import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolServerSideTranslatorPB;
-import org.apache.hadoop.ozone.protocolPB.StorageContainerLocationProtocolServerSideTranslatorPB;
-import org.apache.hadoop.ozone.scm.block.BlockManager;
-import org.apache.hadoop.ozone.scm.block.BlockManagerImpl;
-import org.apache.hadoop.ozone.scm.container.ContainerMapping;
-import org.apache.hadoop.ozone.scm.container.Mapping;
-import org.apache.hadoop.ozone.scm.container.placement.metrics.ContainerStat;
-import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMMetrics;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.scm.ScmInfo;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.scm.node.SCMNodeManager;
-import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.DeleteBlockResult;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.scm.protocol.StorageContainerLocationProtocol;
-import org.apache.hadoop.scm.protocolPB.ScmBlockLocationProtocolPB;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.ObjectName;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.ozone.protocol.proto
-    .ScmBlockLocationProtocolProtos.DeleteScmBlockResult.Result;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_DATANODE_ADDRESS_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_HANDLER_COUNT_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
-import static org.apache.hadoop.util.ExitUtil.terminate;
-
-/**
- * StorageContainerManager is the main entry point for the service that provides
- * information about which SCM nodes host containers.
- *
- * DataNodes report to StorageContainerManager using heartbeat
- * messages. SCM allocates containers and returns a pipeline.
- *
- * A client once it gets a pipeline (a list of datanodes) will connect to the
- * datanodes and create a container, which then can be used to store data.
- */
-@InterfaceAudience.LimitedPrivate({"HDFS", "CBLOCK", "OZONE", "HBASE"})
-public class StorageContainerManager extends ServiceRuntimeInfoImpl
-    implements StorageContainerDatanodeProtocol,
-    StorageContainerLocationProtocol, ScmBlockLocationProtocol, SCMMXBean {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(StorageContainerManager.class);
-
-  /**
-   *  Startup options.
-   */
-  public enum StartupOption {
-    INIT("-init"),
-    CLUSTERID("-clusterid"),
-    GENCLUSTERID("-genclusterid"),
-    REGULAR("-regular"),
-    HELP("-help");
-
-    private final String name;
-    private String clusterId = null;
-
-    public void setClusterId(String cid) {
-      if(cid != null && !cid.isEmpty()) {
-        clusterId = cid;
-      }
-    }
-
-    public String getClusterId() {
-      return clusterId;
-    }
-
-    StartupOption(String arg) {
-      this.name = arg;
-    }
-
-    public String getName() {
-      return name;
-    }
-  }
-
-  /**
-   * NodeManager and container Managers for SCM.
-   */
-  private final NodeManager scmNodeManager;
-  private final Mapping scmContainerManager;
-  private final BlockManager scmBlockManager;
-  private final SCMStorage scmStorage;
-
-  /** The RPC server that listens to requests from DataNodes. */
-  private final RPC.Server datanodeRpcServer;
-  private final InetSocketAddress datanodeRpcAddress;
-
-  /** The RPC server that listens to requests from clients. */
-  private final RPC.Server clientRpcServer;
-  private final InetSocketAddress clientRpcAddress;
-
-  /** The RPC server that listens to requests from block service clients. */
-  private final RPC.Server blockRpcServer;
-  private final InetSocketAddress blockRpcAddress;
-
-  private final StorageContainerManagerHttpServer httpServer;
-
-  /** SCM mxbean. */
-  private ObjectName scmInfoBeanName;
-
-  /** SCM super user. */
-  private final String scmUsername;
-  private final Collection<String> scmAdminUsernames;
-
-  /** SCM metrics. */
-  private static SCMMetrics metrics;
-  /** Key = DatanodeUuid, value = ContainerStat. */
-  private Cache<String, ContainerStat> containerReportCache;
-
-
-  private static final String USAGE =
-      "Usage: \n hdfs scm [genericOptions] "
-          + "[ " + StartupOption.INIT.getName() + " [ "
-          + StartupOption.CLUSTERID.getName() + " <cid> ] ]\n "
-          + "hdfs scm [genericOptions] [ "
-          + StartupOption.GENCLUSTERID.getName() + " ]\n " +
-          "hdfs scm [ "
-          + StartupOption.HELP.getName() + " ]\n";
-  /**
-   * Creates a new StorageContainerManager.  Configuration will be updated with
-   * information on the actual listening addresses used for RPC servers.
-   *
-   * @param conf configuration
-   */
-  private StorageContainerManager(OzoneConfiguration conf)
-      throws IOException {
-
-    final int handlerCount = conf.getInt(
-        OZONE_SCM_HANDLER_COUNT_KEY, OZONE_SCM_HANDLER_COUNT_DEFAULT);
-    final int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
-        OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
-
-    StorageContainerManager.initMetrics();
-    initContainerReportCache(conf);
-
-    scmStorage = new SCMStorage(conf);
-    if (scmStorage.getState() != StorageState.INITIALIZED) {
-      throw new SCMException("SCM not initialized.",
-          ResultCodes.SCM_NOT_INITIALIZED);
-    }
-    scmNodeManager = new SCMNodeManager(conf, scmStorage.getClusterID(), this);
-    scmContainerManager = new ContainerMapping(conf, scmNodeManager, cacheSize);
-    scmBlockManager = new BlockManagerImpl(conf, scmNodeManager,
-        scmContainerManager, cacheSize);
-
-    scmAdminUsernames = conf.getTrimmedStringCollection(
-        OzoneConfigKeys.OZONE_ADMINISTRATORS);
-    scmUsername = UserGroupInformation.getCurrentUser().getUserName();
-    if (!scmAdminUsernames.contains(scmUsername)) {
-      scmAdminUsernames.add(scmUsername);
-    }
-
-    RPC.setProtocolEngine(conf, StorageContainerDatanodeProtocolPB.class,
-        ProtobufRpcEngine.class);
-    RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class,
-        ProtobufRpcEngine.class);
-    RPC.setProtocolEngine(conf, ScmBlockLocationProtocolPB.class,
-        ProtobufRpcEngine.class);
-
-    BlockingService dnProtoPbService = StorageContainerDatanodeProtocolProtos.
-        StorageContainerDatanodeProtocolService.newReflectiveBlockingService(
-        new StorageContainerDatanodeProtocolServerSideTranslatorPB(this));
-
-    final InetSocketAddress datanodeRpcAddr =
-        OzoneClientUtils.getScmDataNodeBindAddress(conf);
-    datanodeRpcServer = startRpcServer(conf, datanodeRpcAddr,
-        StorageContainerDatanodeProtocolPB.class, dnProtoPbService,
-        handlerCount);
-    datanodeRpcAddress = OzoneClientUtils.updateRPCListenAddress(conf,
-        OZONE_SCM_DATANODE_ADDRESS_KEY, datanodeRpcAddr, datanodeRpcServer);
-
-    // SCM Container Service RPC
-    BlockingService storageProtoPbService =
-        StorageContainerLocationProtocolProtos
-            .StorageContainerLocationProtocolService
-            .newReflectiveBlockingService(
-            new StorageContainerLocationProtocolServerSideTranslatorPB(this));
-
-    final InetSocketAddress scmAddress =
-        OzoneClientUtils.getScmClientBindAddress(conf);
-    clientRpcServer = startRpcServer(conf, scmAddress,
-        StorageContainerLocationProtocolPB.class, storageProtoPbService,
-        handlerCount);
-    clientRpcAddress = OzoneClientUtils.updateRPCListenAddress(conf,
-        OZONE_SCM_CLIENT_ADDRESS_KEY, scmAddress, clientRpcServer);
-
-    // SCM Block Service RPC
-    BlockingService blockProtoPbService =
-        ScmBlockLocationProtocolProtos
-            .ScmBlockLocationProtocolService
-            .newReflectiveBlockingService(
-            new ScmBlockLocationProtocolServerSideTranslatorPB(this));
-
-    final InetSocketAddress scmBlockAddress =
-        OzoneClientUtils.getScmBlockClientBindAddress(conf);
-    blockRpcServer = startRpcServer(conf, scmBlockAddress,
-        ScmBlockLocationProtocolPB.class, blockProtoPbService,
-        handlerCount);
-    blockRpcAddress = OzoneClientUtils.updateRPCListenAddress(conf,
-        OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY, scmBlockAddress, blockRpcServer);
-
-    httpServer = new StorageContainerManagerHttpServer(conf);
-
-    registerMXBean();
-  }
-
-  /**
-   * Initialize container reports cache that sent from datanodes.
-   *
-   * @param conf
-   */
-  private void initContainerReportCache(OzoneConfiguration conf) {
-    containerReportCache = CacheBuilder.newBuilder()
-        .expireAfterAccess(Long.MAX_VALUE, TimeUnit.MILLISECONDS)
-        .maximumSize(Integer.MAX_VALUE)
-        .removalListener(new RemovalListener<String, ContainerStat>() {
-          @Override
-          public void onRemoval(
-              RemovalNotification<String, ContainerStat> removalNotification) {
-            synchronized (containerReportCache) {
-              ContainerStat stat = removalNotification.getValue();
-              // remove invalid container report
-              metrics.decrContainerStat(stat);
-              LOG.debug(
-                  "Remove expired container stat entry for datanode: {}.",
-                  removalNotification.getKey());
-            }
-          }
-        }).build();
-  }
-
-  /**
-   * Builds a message for logging startup information about an RPC server.
-   *
-   * @param description RPC server description
-   * @param addr RPC server listening address
-   * @return server startup message
-   */
-  private static String buildRpcServerStartMessage(String description,
-      InetSocketAddress addr) {
-    return addr != null ? String.format("%s is listening at %s",
-        description, addr.toString()) :
-        String.format("%s not started", description);
-  }
-
-  /**
-   * Starts an RPC server, if configured.
-   *
-   * @param conf configuration
-   * @param addr configured address of RPC server
-   * @param protocol RPC protocol provided by RPC server
-   * @param instance RPC protocol implementation instance
-   * @param handlerCount RPC server handler count
-   *
-   * @return RPC server
-   * @throws IOException if there is an I/O error while creating RPC server
-   */
-  private static RPC.Server startRpcServer(OzoneConfiguration conf,
-      InetSocketAddress addr, Class<?> protocol, BlockingService instance,
-      int handlerCount)
-      throws IOException {
-    RPC.Server rpcServer = new RPC.Builder(conf)
-        .setProtocol(protocol)
-        .setInstance(instance)
-        .setBindAddress(addr.getHostString())
-        .setPort(addr.getPort())
-        .setNumHandlers(handlerCount)
-        .setVerbose(false)
-        .setSecretManager(null)
-        .build();
-
-    DFSUtil.addPBProtocol(conf, protocol, instance, rpcServer);
-    return rpcServer;
-  }
-
-  private void registerMXBean() {
-    Map<String, String> jmxProperties = new HashMap<>();
-    jmxProperties.put("component", "ServerRuntime");
-    this.scmInfoBeanName =
-        MBeans.register("StorageContainerManager",
-            "StorageContainerManagerInfo",
-            jmxProperties,
-            this);
-  }
-
-  private void unregisterMXBean() {
-    if(this.scmInfoBeanName != null) {
-      MBeans.unregister(this.scmInfoBeanName);
-      this.scmInfoBeanName = null;
-    }
-  }
-
-  /**
-   * Main entry point for starting StorageContainerManager.
-   *
-   * @param argv arguments
-   * @throws IOException if startup fails due to I/O error
-   */
-  public static void main(String[] argv) throws IOException {
-    if (DFSUtil.parseHelpArgument(argv, USAGE,
-        System.out, true)) {
-      System.exit(0);
-    }
-    try {
-      OzoneConfiguration conf = new OzoneConfiguration();
-      GenericOptionsParser hParser = new GenericOptionsParser(conf, argv);
-      if (!hParser.isParseSuccessful()) {
-        System.err.println("USAGE: " + USAGE + "\n");
-        hParser.printGenericCommandUsage(System.err);
-        System.exit(1);
-      }
-      StringUtils.startupShutdownMessage(StorageContainerManager.class,
-          argv, LOG);
-      StorageContainerManager scm = createSCM(hParser.getRemainingArgs(), conf);
-      if (scm != null) {
-        scm.start();
-        scm.join();
-      }
-    } catch (Throwable t) {
-      LOG.error("Failed to start the StorageContainerManager.", t);
-      terminate(1, t);
-    }
-  }
-
-  private static void printUsage(PrintStream out) {
-    out.println(USAGE + "\n");
-  }
-
-  public static StorageContainerManager createSCM(String[] argv,
-      OzoneConfiguration conf) throws IOException {
-    if (!DFSUtil.isOzoneEnabled(conf)) {
-      System.err.println("SCM cannot be started in secure mode or when " +
-          OZONE_ENABLED + " is set to false");
-      System.exit(1);
-    }
-    StartupOption startOpt = parseArguments(argv);
-    if (startOpt == null) {
-      printUsage(System.err);
-      terminate(1);
-      return null;
-    }
-    switch (startOpt) {
-    case INIT:
-      terminate(scmInit(conf) ? 0 : 1);
-      return null;
-    case GENCLUSTERID:
-      System.out.println("Generating new cluster id:");
-      System.out.println(StorageInfo.newClusterID());
-      terminate(0);
-      return null;
-    case HELP:
-      printUsage(System.err);
-      terminate(0);
-      return null;
-    default:
-      return new StorageContainerManager(conf);
-    }
-  }
-
-  /**
-   * Routine to set up the Version info for StorageContainerManager.
-   *
-   * @param conf OzoneConfiguration
-   * @return true if SCM initialization is successful, false otherwise.
-   * @throws IOException if init fails due to I/O error
-   */
-  public static boolean scmInit(OzoneConfiguration conf) throws IOException {
-    SCMStorage scmStorage = new SCMStorage(conf);
-    StorageState state = scmStorage.getState();
-    if (state != StorageState.INITIALIZED) {
-      try {
-        String clusterId = StartupOption.INIT.getClusterId();
-        if (clusterId != null && !clusterId.isEmpty()) {
-          scmStorage.setClusterId(clusterId);
-        }
-        scmStorage.initialize();
-        System.out.println("SCM initialization succeeded." +
-            "Current cluster id for sd=" + scmStorage.getStorageDir() + ";cid="
-                + scmStorage.getClusterID());
-        return true;
-      } catch (IOException ioe) {
-        LOG.error("Could not initialize SCM version file", ioe);
-        return false;
-      }
-    } else {
-      System.out.println("SCM already initialized. Reusing existing" +
-          " cluster id for sd=" + scmStorage.getStorageDir() + ";cid="
-              + scmStorage.getClusterID());
-      return true;
-    }
-  }
-
-  private static StartupOption parseArguments(String[] args) {
-    int argsLen = (args == null) ? 0 : args.length;
-    StartupOption startOpt = StartupOption.HELP;
-    if (argsLen == 0) {
-      startOpt = StartupOption.REGULAR;
-    }
-    for (int i = 0; i < argsLen; i++) {
-      String cmd = args[i];
-      if (StartupOption.INIT.getName().equalsIgnoreCase(cmd)) {
-        startOpt = StartupOption.INIT;
-        if (argsLen > 3) {
-          return null;
-        }
-        for (i = i + 1; i < argsLen; i++) {
-          if (args[i].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) {
-            i++;
-            if (i < argsLen && !args[i].isEmpty()) {
-              startOpt.setClusterId(args[i]);
-            } else {
-              // if no cluster id specified or is empty string, return null
-              LOG.error("Must specify a valid cluster ID after the "
-                  + StartupOption.CLUSTERID.getName() + " flag");
-              return null;
-            }
-          } else {
-            return null;
-          }
-        }
-      } else if (StartupOption.GENCLUSTERID.getName().equalsIgnoreCase(cmd)) {
-        if (argsLen > 1) {
-          return null;
-        }
-        startOpt = StartupOption.GENCLUSTERID;
-      }
-    }
-    return startOpt;
-  }
-
-  /**
-   * Returns a SCMCommandRepose from the SCM Command.
-   * @param cmd - Cmd
-   * @return SCMCommandResponseProto
-   * @throws InvalidProtocolBufferException
-   */
-  @VisibleForTesting
-  public SCMCommandResponseProto getCommandResponse(SCMCommand cmd,
-      final String datanodID)
-      throws IOException {
-    Type type = cmd.getType();
-    SCMCommandResponseProto.Builder builder =
-        SCMCommandResponseProto.newBuilder()
-        .setDatanodeUUID(datanodID);
-    switch (type) {
-    case registeredCommand:
-      return builder.setCmdType(Type.registeredCommand)
-          .setRegisteredProto(
-              SCMRegisteredCmdResponseProto.getDefaultInstance())
-          .build();
-    case versionCommand:
-      return builder.setCmdType(Type.versionCommand)
-          .setVersionProto(SCMVersionResponseProto.getDefaultInstance())
-          .build();
-    case sendContainerReport:
-      return builder.setCmdType(Type.sendContainerReport)
-          .setSendReport(SendContainerReportProto.getDefaultInstance())
-          .build();
-    case reregisterCommand:
-      return builder.setCmdType(Type.reregisterCommand)
-          .setReregisterProto(SCMReregisterCmdResponseProto
-              .getDefaultInstance())
-          .build();
-    case deleteBlocksCommand:
-      // Once SCM sends out the deletion message, increment the count.
-      // this is done here instead of when SCM receives the ACK, because
-      // DN might not be able to response the ACK for sometime. In case
-      // it times out, SCM needs to re-send the message some more times.
-      List<Long> txs = ((DeleteBlocksCommand) cmd).blocksTobeDeleted()
-          .stream().map(tx -> tx.getTxID()).collect(Collectors.toList());
-      this.getScmBlockManager().getDeletedBlockLog().incrementCount(txs);
-      return builder.setCmdType(Type.deleteBlocksCommand)
-          .setDeleteBlocksProto(((DeleteBlocksCommand) cmd).getProto())
-          .build();
-    case closeContainerCommand:
-      return builder.setCmdType(Type.closeContainerCommand)
-          .setCloseContainerProto(((CloseContainerCommand)cmd).getProto())
-          .build();
-    default:
-      throw new IllegalArgumentException("Not implemented");
-    }
-  }
-
-  @VisibleForTesting
-  public static SCMRegisteredCmdResponseProto getRegisteredResponse(
-      SCMCommand cmd, SCMNodeAddressList addressList) {
-    Preconditions.checkState(cmd.getClass() == RegisteredCommand.class);
-    RegisteredCommand rCmd = (RegisteredCommand) cmd;
-    StorageContainerDatanodeProtocolProtos.Type type = cmd.getType();
-    if (type != Type.registeredCommand) {
-      throw new IllegalArgumentException("Registered command is not well " +
-          "formed. Internal Error.");
-    }
-    return SCMRegisteredCmdResponseProto.newBuilder()
-        //TODO : Fix this later when we have multiple SCM support.
-        //.setAddressList(addressList)
-        .setErrorCode(rCmd.getError())
-        .setClusterID(rCmd.getClusterID())
-        .setDatanodeUUID(rCmd.getDatanodeUUID()).build();
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public Pipeline getContainer(String containerName) throws IOException {
-    checkAdminAccess();
-    return scmContainerManager.getContainer(containerName).getPipeline();
-  }
-
-  @VisibleForTesting
-  public ContainerInfo getContainerInfo(String containerName)
-      throws IOException {
-    return scmContainerManager.getContainer(containerName);
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public List<ContainerInfo> listContainer(String startName,
-      String prefixName, int count) throws IOException {
-    return scmContainerManager.listContainer(startName, prefixName, count);
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public void deleteContainer(String containerName) throws IOException {
-    checkAdminAccess();
-    scmContainerManager.deleteContainer(containerName);
-  }
-
-  /**
-   * Queries a list of Node Statuses.
-   *
-   * @param nodeStatuses
-   * @param queryScope
-   * @param poolName @return List of Datanodes.
-   */
-  @Override
-  public OzoneProtos.NodePool queryNode(EnumSet<NodeState> nodeStatuses,
-      OzoneProtos.QueryScope queryScope, String poolName) throws IOException {
-
-    if (queryScope == OzoneProtos.QueryScope.POOL) {
-      throw new IllegalArgumentException("Not Supported yet");
-    }
-
-    List<DatanodeID> datanodes = queryNode(nodeStatuses);
-    OzoneProtos.NodePool.Builder poolBuilder =
-        OzoneProtos.NodePool.newBuilder();
-
-    for (DatanodeID datanode : datanodes) {
-      OzoneProtos.Node node = OzoneProtos.Node.newBuilder()
-          .setNodeID(datanode.getProtoBufMessage())
-          .addAllNodeStates(nodeStatuses)
-          .build();
-      poolBuilder.addNodes(node);
-    }
-
-    return poolBuilder.build();
-  }
-
-  /**
-   * Notify from client when begin/finish operation for container/pipeline
-   * objects on datanodes.
-   * @param type
-   * @param name
-   * @param op
-   * @param stage
-   */
-  @Override
-  public void notifyObjectStageChange(
-      ObjectStageChangeRequestProto.Type type, String name,
-      ObjectStageChangeRequestProto.Op op,
-      ObjectStageChangeRequestProto.Stage stage) throws IOException {
-
-    LOG.info("Object type {} name {} op {} new stage {}",
-        type, name, op, stage);
-    if (type == ObjectStageChangeRequestProto.Type.container) {
-      if (op == ObjectStageChangeRequestProto.Op.create) {
-        if (stage == ObjectStageChangeRequestProto.Stage.begin) {
-          scmContainerManager.updateContainerState(name,
-              OzoneProtos.LifeCycleEvent.CREATE);
-        } else {
-          scmContainerManager.updateContainerState(name,
-              OzoneProtos.LifeCycleEvent.CREATED);
-        }
-      } else if (op == ObjectStageChangeRequestProto.Op.close) {
-        if (stage == ObjectStageChangeRequestProto.Stage.begin) {
-          scmContainerManager.updateContainerState(name,
-              OzoneProtos.LifeCycleEvent.FINALIZE);
-        } else {
-          scmContainerManager.updateContainerState(name,
-              OzoneProtos.LifeCycleEvent.CLOSE);
-        }
-      }
-    } //else if (type == ObjectStageChangeRequestProto.Type.pipeline) {
-    // TODO: pipeline state update will be addressed in future patch.
-    //}
-  }
-
-  /**
-   * Creates a replication pipeline of a specified type.
-   */
-  @Override
-  public Pipeline createReplicationPipeline(
-      OzoneProtos.ReplicationType replicationType,
-      OzoneProtos.ReplicationFactor factor,
-      OzoneProtos.NodePool nodePool)
-      throws IOException {
-     // TODO: will be addressed in future patch.
-    return null;
-  }
-
-  /**
-   * Queries a list of Node that match a set of statuses.
-   * <p>
-   * For example, if the nodeStatuses is HEALTHY and RAFT_MEMBER,
-   * then this call will return all healthy nodes which members in
-   * Raft pipeline.
-   * <p>
-   * Right now we don't support operations, so we assume it is an AND operation
-   * between the operators.
-   *
-   * @param nodeStatuses - A set of NodeStates.
-   * @return List of Datanodes.
-   */
-
-  public List<DatanodeID> queryNode(EnumSet<NodeState> nodeStatuses) {
-    Preconditions.checkNotNull(nodeStatuses, "Node Query set cannot be null");
-    Preconditions.checkState(nodeStatuses.size() > 0, "No valid arguments " +
-        "in the query set");
-    List<DatanodeID> resultList = new LinkedList<>();
-    Set<DatanodeID> currentSet = new TreeSet<>();
-
-    for (NodeState nodeState : nodeStatuses) {
-      Set<DatanodeID> nextSet = queryNodeState(nodeState);
-      if ((nextSet == null) || (nextSet.size() == 0)) {
-        // Right now we only support AND operation. So intersect with
-        // any empty set is null.
-        return resultList;
-      }
-      // First time we have to add all the elements, next time we have to
-      // do an intersection operation on the set.
-      if (currentSet.size() == 0) {
-        currentSet.addAll(nextSet);
-      } else {
-        currentSet.retainAll(nextSet);
-      }
-    }
-
-    resultList.addAll(currentSet);
-    return resultList;
-  }
-
-  /**
-   * Query the System for Nodes.
-   *
-   * @param nodeState - NodeState that we are interested in matching.
-   * @return Set of Datanodes that match the NodeState.
-   */
-  private Set<DatanodeID> queryNodeState(NodeState nodeState) {
-    if (nodeState == NodeState.RAFT_MEMBER ||
-        nodeState == NodeState.FREE_NODE) {
-      throw new IllegalStateException("Not implemented yet");
-    }
-    Set<DatanodeID> returnSet = new TreeSet<>();
-    List<DatanodeID> tmp = getScmNodeManager().getNodes(nodeState);
-    if ((tmp != null) && (tmp.size() > 0)) {
-      returnSet.addAll(tmp);
-    }
-    return returnSet;
-  }
-
-  /**
-   * Asks SCM where a container should be allocated. SCM responds with the set
-   * of datanodes that should be used creating this container.
-   *
-   * @param containerName - Name of the container.
-   * @param replicationFactor - replication factor.
-   * @return pipeline
-   * @throws IOException
-   */
-  @Override
-  public Pipeline allocateContainer(OzoneProtos.ReplicationType replicationType,
-      OzoneProtos.ReplicationFactor replicationFactor, String containerName,
-      String owner) throws IOException {
-
-    checkAdminAccess();
-    return scmContainerManager
-        .allocateContainer(replicationType, replicationFactor, containerName,
-            owner).getPipeline();
-  }
-
-  /**
-   * Returns listening address of StorageLocation Protocol RPC server.
-   *
-   * @return listen address of StorageLocation RPC server
-   */
-  @VisibleForTesting
-  public InetSocketAddress getClientRpcAddress() {
-    return clientRpcAddress;
-  }
-
-  @Override
-  public String getClientRpcPort() {
-    InetSocketAddress addr = getClientRpcAddress();
-    return addr == null ? "0" : Integer.toString(addr.getPort());
-  }
-
-  /**
-   * Returns listening address of StorageDatanode Protocol RPC server.
-   *
-   * @return Address where datanode are communicating.
-   */
-  public InetSocketAddress getDatanodeRpcAddress() {
-    return datanodeRpcAddress;
-  }
-
-  @Override
-  public String getDatanodeRpcPort() {
-    InetSocketAddress addr = getDatanodeRpcAddress();
-    return addr == null ? "0" : Integer.toString(addr.getPort());
-  }
-
-  /**
-   * Start service.
-   */
-  public void start() throws IOException {
-    LOG.info(buildRpcServerStartMessage(
-        "StorageContainerLocationProtocol RPC server", clientRpcAddress));
-    DefaultMetricsSystem.initialize("StorageContainerManager");
-    clientRpcServer.start();
-    LOG.info(buildRpcServerStartMessage(
-        "ScmBlockLocationProtocol RPC server", blockRpcAddress));
-    blockRpcServer.start();
-    LOG.info(buildRpcServerStartMessage("RPC server for DataNodes",
-        datanodeRpcAddress));
-    datanodeRpcServer.start();
-    httpServer.start();
-    scmBlockManager.start();
-
-    setStartTime();
-
-  }
-
-  /**
-   * Stop service.
-   */
-  public void stop() {
-    try {
-      LOG.info("Stopping block service RPC server");
-      blockRpcServer.stop();
-    } catch (Exception ex) {
-      LOG.error("Storage Container Manager blockRpcServer stop failed.", ex);
-    }
-
-    try {
-      LOG.info("Stopping the StorageContainerLocationProtocol RPC server");
-      clientRpcServer.stop();
-    } catch (Exception ex) {
-      LOG.error("Storage Container Manager clientRpcServer stop failed.", ex);
-    }
-
-    try {
-      LOG.info("Stopping the RPC server for DataNodes");
-      datanodeRpcServer.stop();
-    } catch (Exception ex) {
-      LOG.error("Storage Container Manager datanodeRpcServer stop failed.", ex);
-    }
-
-    try {
-      LOG.info("Stopping Storage Container Manager HTTP server.");
-      httpServer.stop();
-    } catch (Exception ex) {
-      LOG.error("Storage Container Manager HTTP server stop failed.", ex);
-    }
-
-    try {
-      LOG.info("Stopping Block Manager Service.");
-      scmBlockManager.stop();
-    } catch (Exception ex) {
-      LOG.error("SCM block manager service stop failed.", ex);
-    }
-
-    if (containerReportCache != null) {
-      containerReportCache.invalidateAll();
-      containerReportCache.cleanUp();
-    }
-
-    if (metrics != null) {
-      metrics.unRegister();
-    }
-
-    unregisterMXBean();
-    IOUtils.cleanupWithLogger(LOG, scmContainerManager);
-    IOUtils.cleanupWithLogger(LOG, scmNodeManager);
-  }
-
-  /**
-   * Wait until service has completed shutdown.
-   */
-  public void join() {
-    try {
-      blockRpcServer.join();
-      clientRpcServer.join();
-      datanodeRpcServer.join();
-    } catch (InterruptedException e) {
-      Thread.currentThread().interrupt();
-      LOG.info("Interrupted during StorageContainerManager join.");
-    }
-  }
-
-  /**
-   * Returns SCM version.
-   *
-   * @return Version info.
-   */
-  @Override
-  public SCMVersionResponseProto getVersion(
-      SCMVersionRequestProto versionRequest) throws IOException {
-    return getScmNodeManager().getVersion(versionRequest).getProtobufMessage();
-  }
-
-  /**
-   * Used by data node to send a Heartbeat.
-   *
-   * @param datanodeID - Datanode ID.
-   * @param nodeReport - Node Report
-   * @param reportState - Container report ready info.
-   * @return - SCMHeartbeatResponseProto
-   * @throws IOException
-   */
-  @Override
-  public SCMHeartbeatResponseProto sendHeartbeat(DatanodeID datanodeID,
-      SCMNodeReport nodeReport, ReportState reportState) throws IOException {
-    List<SCMCommand> commands =
-        getScmNodeManager().sendHeartbeat(datanodeID, nodeReport, reportState);
-    List<SCMCommandResponseProto> cmdResponses = new LinkedList<>();
-    for (SCMCommand cmd : commands) {
-      cmdResponses.add(getCommandResponse(cmd, datanodeID.getDatanodeUuid()
-          .toString()));
-    }
-    return SCMHeartbeatResponseProto.newBuilder().addAllCommands(cmdResponses)
-        .build();
-  }
-
-  /**
-   * Register Datanode.
-   *
-   * @param datanodeID - DatanodID.
-   * @param scmAddresses - List of SCMs this datanode is configured to
-   * communicate.
-   * @return SCM Command.
-   */
-  @Override
-  public StorageContainerDatanodeProtocolProtos.SCMRegisteredCmdResponseProto
-      register(DatanodeID datanodeID, String[] scmAddresses)
-      throws IOException {
-    // TODO : Return the list of Nodes that forms the SCM HA.
-    return getRegisteredResponse(scmNodeManager.register(datanodeID), null);
-  }
-
-  /**
-   * Send a container report.
-   *
-   * @param reports -- Container report
-   * @return HeartbeatRespose.nullcommand.
-   * @throws IOException
-   */
-  @Override
-  public ContainerReportsResponseProto sendContainerReport(
-      ContainerReportsRequestProto reports) throws IOException {
-    updateContainerReportMetrics(reports);
-
-    // should we process container reports async?
-    scmContainerManager.processContainerReports(reports);
-    return ContainerReportsResponseProto.newBuilder().build();
-  }
-
-  private void updateContainerReportMetrics(
-      ContainerReportsRequestProto reports) {
-    ContainerStat newStat = null;
-    // TODO: We should update the logic once incremental container report
-    // type is supported.
-    if (reports
-        .getType() == ContainerReportsRequestProto.reportType.fullReport) {
-      newStat = new ContainerStat();
-      for (StorageContainerDatanodeProtocolProtos.ContainerInfo info : reports
-          .getReportsList()) {
-        newStat.add(new ContainerStat(info.getSize(), info.getUsed(),
-            info.getKeyCount(), info.getReadBytes(), info.getWriteBytes(),
-            info.getReadCount(), info.getWriteCount()));
-      }
-
-      // update container metrics
-      metrics.setLastContainerStat(newStat);
-    }
-
-    // Update container stat entry, this will trigger a removal operation if it
-    // exists in cache.
-    synchronized (containerReportCache) {
-      String datanodeUuid = reports.getDatanodeID().getDatanodeUuid();
-      if (datanodeUuid != null && newStat != null) {
-        containerReportCache.put(datanodeUuid, newStat);
-        // update global view container metrics
-        metrics.incrContainerStat(newStat);
-      }
-    }
-  }
-
-  /**
-   * Handles the block deletion ACKs sent by datanodes. Once ACKs recieved,
-   * SCM considers the blocks are deleted and update the metadata in SCM DB.
-   *
-   * @param acks
-   * @return
-   * @throws IOException
-   */
-  @Override
-  public ContainerBlocksDeletionACKResponseProto sendContainerBlocksDeletionACK(
-      ContainerBlocksDeletionACKProto acks) throws IOException {
-    if (acks.getResultsCount() > 0) {
-      List<DeleteBlockTransactionResult> resultList = acks.getResultsList();
-      for (DeleteBlockTransactionResult result : resultList) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Got block deletion ACK from datanode, TXIDs={}, "
-                  + "success={}", result.getTxID(), result.getSuccess());
-        }
-        if (result.getSuccess()) {
-          LOG.debug("Purging TXID={} from block deletion log",
-              result.getTxID());
-          this.getScmBlockManager().getDeletedBlockLog()
-              .commitTransactions(Collections.singletonList(result.getTxID()));
-        } else {
-          LOG.warn("Got failed ACK for TXID={}, prepare to resend the "
-              + "TX in next interval", result.getTxID());
-        }
-      }
-    }
-    return ContainerBlocksDeletionACKResponseProto.newBuilder()
-        .getDefaultInstanceForType();
-  }
-
-  /**
-   * Returns the Number of Datanodes that are communicating with SCM.
-   *
-   * @param nodestate Healthy, Dead etc.
-   * @return int -- count
-   */
-  public int getNodeCount(NodeState nodestate) {
-    return scmNodeManager.getNodeCount(nodestate);
-  }
-
-  /**
-   * Returns SCM container manager.
-   */
-  @VisibleForTesting
-  public Mapping getScmContainerManager() {
-    return scmContainerManager;
-  }
-
-  /**
-   * Returns node manager.
-   * @return - Node Manager
-   */
-  @VisibleForTesting
-  public NodeManager getScmNodeManager() {
-    return scmNodeManager;
-  }
-
-  @VisibleForTesting
-  public BlockManager getScmBlockManager() {
-    return scmBlockManager;
-  }
-
-  /**
-   * Get block locations.
-   * @param keys batch of block keys to retrieve.
-   * @return set of allocated blocks.
-   * @throws IOException
-   */
-  @Override
-  public Set<AllocatedBlock> getBlockLocations(final Set<String> keys)
-      throws IOException {
-    Set<AllocatedBlock> locatedBlocks = new HashSet<>();
-    for (String key: keys) {
-      Pipeline pipeline = scmBlockManager.getBlock(key);
-      AllocatedBlock block = new AllocatedBlock.Builder()
-          .setKey(key)
-          .setPipeline(pipeline).build();
-      locatedBlocks.add(block);
-    }
-    return locatedBlocks;
-  }
-
-  /**
-   * Asks SCM where a block should be allocated. SCM responds with the set of
-   * datanodes that should be used creating this block.
-   *
-   * @param size - size of the block.
-   * @param type - Replication type.
-   * @param factor
-   * @return allocated block accessing info (key, pipeline).
-   * @throws IOException
-   */
-  @Override
-  public AllocatedBlock allocateBlock(long size,
-      OzoneProtos.ReplicationType type, OzoneProtos.ReplicationFactor factor,
-      String owner) throws IOException {
-    return scmBlockManager.allocateBlock(size, type, factor, owner);
-  }
-
-  /**
-   * Get the clusterId and SCM Id from the version file in SCM.
-   */
-  @Override
-  public ScmInfo getScmInfo() throws IOException {
-    ScmInfo.Builder builder = new ScmInfo.Builder()
-        .setClusterId(scmStorage.getClusterID())
-        .setScmId(scmStorage.getScmId());
-    return builder.build();
-  }
-  /**
-   * Delete blocks for a set of object keys.
-   *
-   * @param keyBlocksInfoList list of block keys with object keys to delete.
-   * @return deletion results.
-   */
-  public List<DeleteBlockGroupResult> deleteKeyBlocks(
-      List<BlockGroup> keyBlocksInfoList) throws IOException {
-    LOG.info("SCM is informed by KSM to delete {} blocks",
-        keyBlocksInfoList.size());
-    List<DeleteBlockGroupResult> results = new ArrayList<>();
-    for (BlockGroup keyBlocks : keyBlocksInfoList) {
-      Result resultCode;
-      try {
-        // We delete blocks in an atomic operation to prevent getting
-        // into state like only a partial of blocks are deleted,
-        // which will leave key in an inconsistent state.
-        scmBlockManager.deleteBlocks(keyBlocks.getBlockIDList());
-        resultCode = Result.success;
-      } catch (SCMException scmEx) {
-        LOG.warn("Fail to delete block: {}", keyBlocks.getGroupID(), scmEx);
-        switch (scmEx.getResult()) {
-        case CHILL_MODE_EXCEPTION:
-          resultCode = Result.chillMode;
-          break;
-        case FAILED_TO_FIND_BLOCK:
-          resultCode = Result.errorNotFound;
-          break;
-        default:
-          resultCode = Result.unknownFailure;
-        }
-      } catch (IOException ex) {
-        LOG.warn("Fail to delete blocks for object key: {}",
-            keyBlocks.getGroupID(), ex);
-        resultCode = Result.unknownFailure;
-      }
-      List<DeleteBlockResult> blockResultList = new ArrayList<>();
-      for (String blockKey : keyBlocks.getBlockIDList()) {
-        blockResultList.add(new DeleteBlockResult(blockKey, resultCode));
-      }
-      results.add(new DeleteBlockGroupResult(keyBlocks.getGroupID(),
-          blockResultList));
-    }
-    return results;
-  }
-
-  @VisibleForTesting
-  public String getPpcRemoteUsername() {
-    UserGroupInformation user = ProtobufRpcEngine.Server.getRemoteUser();
-    return user == null ? null : user.getUserName();
-  }
-
-  private void checkAdminAccess() throws IOException {
-    String remoteUser = getPpcRemoteUsername();
-    if(remoteUser != null) {
-      if (!scmAdminUsernames.contains(remoteUser)) {
-        throw new IOException(
-            "Access denied for user " + remoteUser
-                + ". Superuser privilege is required.");
-      }
-    }
-  }
-
-  /**
-   * Initialize SCM metrics.
-   */
-  public static void initMetrics() {
-    metrics = SCMMetrics.create();
-  }
-
-  /**
-   * Return SCM metrics instance.
-   */
-  public static SCMMetrics getMetrics() {
-    return metrics == null ? SCMMetrics.create() : metrics;
-  }
-
-  /**
-   * Invalidate container stat entry for given datanode.
-   *
-   * @param datanodeUuid
-   */
-  public void removeContainerReport(String datanodeUuid) {
-    synchronized (containerReportCache) {
-      containerReportCache.invalidate(datanodeUuid);
-    }
-  }
-
-  /**
-   * Get container stat of specified datanode.
-   *
-   * @param datanodeUuid
-   * @return
-   */
-  public ContainerStat getContainerReport(String datanodeUuid) {
-    ContainerStat stat = null;
-    synchronized (containerReportCache) {
-      stat = containerReportCache.getIfPresent(datanodeUuid);
-    }
-
-    return stat;
-  }
-
-  /**
-   * Returns a view of the container stat entries. Modifications made to the
-   * map will directly affect the cache.
-   *
-   * @return
-   */
-  public ConcurrentMap<String, ContainerStat> getContainerReportCache() {
-    return containerReportCache.asMap();
-  }
-
-  @Override
-  public Map<String, String> getContainerReport() {
-    Map<String, String> id2StatMap = new HashMap<>();
-    synchronized (containerReportCache) {
-      ConcurrentMap<String, ContainerStat> map = containerReportCache.asMap();
-      for (Map.Entry<String, ContainerStat> entry : map.entrySet()) {
-        id2StatMap.put(entry.getKey(), entry.getValue().toJsonString());
-      }
-    }
-
-    return id2StatMap;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManagerHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManagerHttpServer.java
deleted file mode 100644
index 62c871c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManagerHttpServer.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.web.OzoneHttpServer;
-import org.apache.hadoop.scm.ScmConfigKeys;
-
-import java.io.IOException;
-
-/**
- * HttpServer2 wrapper for the Ozone Storage Container Manager.
- */
-public class StorageContainerManagerHttpServer extends OzoneHttpServer {
-
-  public StorageContainerManagerHttpServer(Configuration conf)
-      throws IOException {
-    super(conf, "scm");
-  }
-
-  @Override protected String getHttpAddressKey() {
-    return ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY;
-  }
-
-  @Override protected String getHttpBindHostKey() {
-    return ScmConfigKeys.OZONE_SCM_HTTP_BIND_HOST_KEY;
-  }
-
-  @Override protected String getHttpsAddressKey() {
-    return ScmConfigKeys.OZONE_SCM_HTTPS_ADDRESS_KEY;
-  }
-
-  @Override protected String getHttpsBindHostKey() {
-    return ScmConfigKeys.OZONE_SCM_HTTPS_BIND_HOST_KEY;
-  }
-
-  @Override protected String getBindHostDefault() {
-    return ScmConfigKeys.OZONE_SCM_HTTP_BIND_HOST_DEFAULT;
-  }
-
-  @Override protected int getHttpBindPortDefault() {
-    return ScmConfigKeys.OZONE_SCM_HTTP_BIND_PORT_DEFAULT;
-  }
-
-  @Override protected int getHttpsBindPortDefault() {
-    return ScmConfigKeys.OZONE_SCM_HTTPS_BIND_PORT_DEFAULT;
-  }
-
-  @Override protected String getKeytabFile() {
-    return ScmConfigKeys.OZONE_SCM_KEYTAB_FILE;
-  }
-
-  @Override protected String getSpnegoPrincipal() {
-    return OzoneConfigKeys.OZONE_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL;
-  }
-
-  @Override protected String getEnabledKey() {
-    return ScmConfigKeys.OZONE_SCM_HTTP_ENABLED_KEY;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/VersionInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/VersionInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/VersionInfo.java
deleted file mode 100644
index 6bb3a22..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/VersionInfo.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm;
-
-/**
- * This is a class that tracks versions of SCM.
- */
-public final class VersionInfo {
-
-  // We will just be normal and use positive counting numbers for versions.
-  private final static VersionInfo[] VERSION_INFOS =
-      {new VersionInfo("First version of SCM", 1)};
-
-
-  public static final String DESCRIPTION_KEY = "Description";
-  private final String description;
-  private final int version;
-
-  /**
-   * Never created outside this class.
-   *
-   * @param description -- description
-   * @param version     -- version number
-   */
-  private VersionInfo(String description, int version) {
-    this.description = description;
-    this.version = version;
-  }
-
-  /**
-   * Returns all versions.
-   *
-   * @return Version info array.
-   */
-  public static VersionInfo[] getAllVersions() {
-    return VERSION_INFOS.clone();
-  }
-
-  /**
-   * Returns the latest version.
-   *
-   * @return versionInfo
-   */
-  public static VersionInfo getLatestVersion() {
-    return VERSION_INFOS[VERSION_INFOS.length - 1];
-  }
-
-  /**
-   * Return description.
-   *
-   * @return String
-   */
-  public String getDescription() {
-    return description;
-  }
-
-  /**
-   * Return the version.
-   *
-   * @return int.
-   */
-  public int getVersion() {
-    return version;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockManager.java
deleted file mode 100644
index cfed7a8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockManager.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.block;
-
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-/**
- *
- *  Block APIs.
- *  Container is transparent to these APIs.
- */
-public interface BlockManager extends Closeable {
-  /**
-   * Allocates a new block for a given size.
-   * @param size - Block Size
-   * @param type Replication Type
-   * @param factor - Replication Factor
-   * @return AllocatedBlock
-   * @throws IOException
-   */
-  AllocatedBlock allocateBlock(long size, OzoneProtos.ReplicationType type,
-      OzoneProtos.ReplicationFactor factor, String owner) throws IOException;
-
-  /**
-   *  Give the key to the block, get the pipeline info.
-   * @param key - key to the block.
-   * @return - Pipeline that used to access the block.
-   * @throws IOException
-   */
-  Pipeline getBlock(String key) throws IOException;
-
-  /**
-   * Deletes a list of blocks in an atomic operation. Internally, SCM
-   * writes these blocks into a {@link DeletedBlockLog} and deletes them
-   * from SCM DB. If this is successful, given blocks are entering pending
-   * deletion state and becomes invisible from SCM namespace.
-   *
-   * @param blockIDs block IDs. This is often the list of blocks of
-   *                 a particular object key.
-   * @throws IOException if exception happens, non of the blocks is deleted.
-   */
-  void deleteBlocks(List<String> blockIDs) throws IOException;
-
-  /**
-   * @return the block deletion transaction log maintained by SCM.
-   */
-  DeletedBlockLog getDeletedBlockLog();
-
-  /**
-   * Start block manager background services.
-   * @throws IOException
-   */
-  void start() throws IOException;
-
-  /**
-   * Shutdown block manager background services.
-   * @throws IOException
-   */
-  void stop() throws IOException;
-
-  /**
-   * @return the block deleting service executed in SCM.
-   */
-  SCMBlockDeletingService getSCMBlockDeletingService();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockManagerImpl.java
deleted file mode 100644
index 5771080..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockManagerImpl.java
+++ /dev/null
@@ -1,530 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.block;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.scm.container.Mapping;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.utils.BatchOperation;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.ObjectName;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_DB;
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
-    .CHILL_MODE_EXCEPTION;
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
-    .FAILED_TO_FIND_BLOCK;
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
-    .INVALID_BLOCK_SIZE;
-
-/** Block Manager manages the block access for SCM. */
-public class BlockManagerImpl implements BlockManager, BlockmanagerMXBean {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(BlockManagerImpl.class);
-  // TODO : FIX ME : Hard coding the owner.
-  // Currently only user of the block service is Ozone, CBlock manages blocks
-  // by itself and does not rely on the Block service offered by SCM.
-
-  private final NodeManager nodeManager;
-  private final Mapping containerManager;
-  private final MetadataStore blockStore;
-
-  private final Lock lock;
-  private final long containerSize;
-  private final long cacheSize;
-
-  private final DeletedBlockLog deletedBlockLog;
-  private final SCMBlockDeletingService blockDeletingService;
-
-  private final int containerProvisionBatchSize;
-  private final Random rand;
-  private ObjectName mxBean;
-
-  /**
-   * Constructor.
-   *
-   * @param conf - configuration.
-   * @param nodeManager - node manager.
-   * @param containerManager - container manager.
-   * @param cacheSizeMB - cache size for level db store.
-   * @throws IOException
-   */
-  public BlockManagerImpl(final Configuration conf,
-      final NodeManager nodeManager, final Mapping containerManager,
-      final int cacheSizeMB) throws IOException {
-    this.nodeManager = nodeManager;
-    this.containerManager = containerManager;
-    this.cacheSize = cacheSizeMB;
-
-    this.containerSize = OzoneConsts.GB * conf.getInt(
-        ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB,
-        ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT);
-    File metaDir = OzoneUtils.getOzoneMetaDirPath(conf);
-    String scmMetaDataDir = metaDir.getPath();
-
-    // Write the block key to container name mapping.
-    File blockContainerDbPath = new File(scmMetaDataDir, BLOCK_DB);
-    blockStore =
-        MetadataStoreBuilder.newBuilder()
-            .setConf(conf)
-            .setDbFile(blockContainerDbPath)
-            .setCacheSize(this.cacheSize * OzoneConsts.MB)
-            .build();
-
-    this.containerProvisionBatchSize =
-        conf.getInt(
-            ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE,
-            ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE_DEFAULT);
-    rand = new Random();
-    this.lock = new ReentrantLock();
-
-    mxBean = MBeans.register("BlockManager", "BlockManagerImpl", this);
-
-    // SCM block deleting transaction log and deleting service.
-    deletedBlockLog = new DeletedBlockLogImpl(conf);
-    long svcInterval =
-        conf.getTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL,
-            OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT,
-            TimeUnit.MILLISECONDS);
-    long serviceTimeout =
-        conf.getTimeDuration(
-            OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
-            OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT,
-            TimeUnit.MILLISECONDS);
-    blockDeletingService =
-        new SCMBlockDeletingService(
-            deletedBlockLog, containerManager, nodeManager, svcInterval,
-            serviceTimeout, conf);
-  }
-
-  /**
-   * Start block manager services.
-   *
-   * @throws IOException
-   */
-  public void start() throws IOException {
-    this.blockDeletingService.start();
-  }
-
-  /**
-   * Shutdown block manager services.
-   *
-   * @throws IOException
-   */
-  public void stop() throws IOException {
-    this.blockDeletingService.shutdown();
-    this.close();
-  }
-
-  /**
-   * Pre allocate specified count of containers for block creation.
-   *
-   * @param count - Number of containers to allocate.
-   * @param type - Type of containers
-   * @param factor - how many copies needed for this container.
-   * @throws IOException
-   */
-  private void preAllocateContainers(int count, ReplicationType type,
-      ReplicationFactor factor, String owner)
-      throws IOException {
-    lock.lock();
-    try {
-      for (int i = 0; i < count; i++) {
-        String containerName = UUID.randomUUID().toString();
-        ContainerInfo containerInfo = null;
-        try {
-          // TODO: Fix this later when Ratis is made the Default.
-          containerInfo = containerManager.allocateContainer(type, factor,
-              containerName, owner);
-
-          if (containerInfo == null) {
-            LOG.warn("Unable to allocate container.");
-            continue;
-          }
-        } catch (IOException ex) {
-          LOG.warn("Unable to allocate container: {}", ex);
-          continue;
-        }
-      }
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  /**
-   * Allocates a block in a container and returns that info.
-   *
-   * @param size - Block Size
-   * @param type Replication Type
-   * @param factor - Replication Factor
-   * @return Allocated block
-   * @throws IOException on failure.
-   */
-  @Override
-  public AllocatedBlock allocateBlock(final long size,
-      ReplicationType type, ReplicationFactor factor, String owner)
-      throws IOException {
-    LOG.trace("Size;{} , type : {}, factor : {} ", size, type, factor);
-
-    if (size < 0 || size > containerSize) {
-      LOG.warn("Invalid block size requested : {}", size);
-      throw new SCMException("Unsupported block size: " + size,
-          INVALID_BLOCK_SIZE);
-    }
-
-    if (!nodeManager.isOutOfChillMode()) {
-      LOG.warn("Not out of Chill mode.");
-      throw new SCMException("Unable to create block while in chill mode",
-          CHILL_MODE_EXCEPTION);
-    }
-
-    lock.lock();
-    try {
-      /*
-               Here is the high level logic.
-
-               1. First we check if there are containers in ALLOCATED state,
-               that is
-                SCM has allocated them in the SCM namespace but the
-                corresponding
-                container has not been created in the Datanode yet. If we
-                have any
-                in that state, we will return that to the client, which allows
-                client to finish creating those containers. This is a sort of
-                 greedy
-                 algorithm, our primary purpose is to get as many containers as
-                 possible.
-
-                2. If there are no allocated containers -- Then we find a Open
-                container that matches that pattern.
-
-                3. If both of them fail, the we will pre-allocate a bunch of
-                conatainers in SCM and try again.
-
-               TODO : Support random picking of two containers from the list.
-                So we
-               can use different kind of policies.
-      */
-
-      ContainerInfo containerInfo;
-
-      // Look for ALLOCATED container that matches all other parameters.
-      containerInfo =
-          containerManager
-              .getStateManager()
-              .getMatchingContainer(
-                  size, owner, type, factor, OzoneProtos.LifeCycleState
-                      .ALLOCATED);
-      if (containerInfo != null) {
-        containerManager.updateContainerState(containerInfo.getContainerName(),
-            OzoneProtos.LifeCycleEvent.CREATE);
-        return newBlock(containerInfo, OzoneProtos.LifeCycleState.ALLOCATED);
-      }
-
-      // Since we found no allocated containers that match our criteria, let us
-      // look for OPEN containers that match the criteria.
-      containerInfo =
-          containerManager
-              .getStateManager()
-              .getMatchingContainer(size, owner, type, factor, OzoneProtos
-                  .LifeCycleState.OPEN);
-      if (containerInfo != null) {
-        return newBlock(containerInfo, OzoneProtos.LifeCycleState.OPEN);
-      }
-
-      // We found neither ALLOCATED or OPEN Containers. This generally means
-      // that most of our containers are full or we have not allocated
-      // containers of the type and replication factor. So let us go and
-      // allocate some.
-      preAllocateContainers(containerProvisionBatchSize, type, factor, owner);
-
-      // Since we just allocated a set of containers this should work
-      containerInfo =
-          containerManager
-              .getStateManager()
-              .getMatchingContainer(
-                  size, owner, type, factor, OzoneProtos.LifeCycleState
-                      .ALLOCATED);
-      if (containerInfo != null) {
-        containerManager.updateContainerState(containerInfo.getContainerName(),
-            OzoneProtos.LifeCycleEvent.CREATE);
-        return newBlock(containerInfo, OzoneProtos.LifeCycleState.ALLOCATED);
-      }
-
-      // we have tried all strategies we know and but somehow we are not able
-      // to get a container for this block. Log that info and return a null.
-      LOG.error(
-          "Unable to allocate a block for the size: {}, type: {}, " +
-              "factor: {}",
-          size,
-          type,
-          factor);
-      return null;
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  /**
-   * newBlock - returns a new block assigned to a container.
-   *
-   * @param containerInfo - Container Info.
-   * @param state - Current state of the container.
-   * @return AllocatedBlock
-   */
-  private AllocatedBlock newBlock(
-      ContainerInfo containerInfo, OzoneProtos.LifeCycleState state)
-      throws IOException {
-
-    // TODO : Replace this with Block ID.
-    String blockKey = UUID.randomUUID().toString();
-    boolean createContainer = (state == OzoneProtos.LifeCycleState.ALLOCATED);
-
-    AllocatedBlock.Builder abb =
-        new AllocatedBlock.Builder()
-            .setKey(blockKey)
-            // TODO : Use containerinfo instead of pipeline.
-            .setPipeline(containerInfo.getPipeline())
-            .setShouldCreateContainer(createContainer);
-    LOG.trace("New block allocated : {} Container ID: {}", blockKey,
-        containerInfo.toString());
-
-    if (containerInfo.getPipeline().getMachines().size() == 0) {
-      LOG.error("Pipeline Machine count is zero.");
-      return null;
-    }
-
-    // Persist this block info to the blockStore DB, so getBlock(key) can
-    // find which container the block lives.
-    // TODO : Remove this DB in future
-    // and make this a KSM operation. Category: SCALABILITY.
-    if (containerInfo.getPipeline().getMachines().size() > 0) {
-      blockStore.put(
-          DFSUtil.string2Bytes(blockKey),
-          DFSUtil.string2Bytes(containerInfo.getPipeline().getContainerName()));
-    }
-    return abb.build();
-  }
-
-  /**
-   * Given a block key, return the Pipeline information.
-   *
-   * @param key - block key assigned by SCM.
-   * @return Pipeline (list of DNs and leader) to access the block.
-   * @throws IOException
-   */
-  @Override
-  public Pipeline getBlock(final String key) throws IOException {
-    lock.lock();
-    try {
-      byte[] containerBytes = blockStore.get(DFSUtil.string2Bytes(key));
-      if (containerBytes == null) {
-        throw new SCMException(
-            "Specified block key does not exist. key : " + key,
-            FAILED_TO_FIND_BLOCK);
-      }
-
-      String containerName = DFSUtil.bytes2String(containerBytes);
-      ContainerInfo containerInfo = containerManager.getContainer(
-          containerName);
-      if (containerInfo == null) {
-        LOG.debug("Container {} allocated by block service"
-            + "can't be found in SCM", containerName);
-        throw new SCMException(
-            "Unable to find container for the block",
-            SCMException.ResultCodes.FAILED_TO_FIND_CONTAINER);
-      }
-      return containerInfo.getPipeline();
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  /**
-   * Deletes a list of blocks in an atomic operation. Internally, SCM writes
-   * these blocks into a
-   * {@link DeletedBlockLog} and deletes them from SCM DB. If this is
-   * successful, given blocks are
-   * entering pending deletion state and becomes invisible from SCM namespace.
-   *
-   * @param blockIDs block IDs. This is often the list of blocks of a
-   * particular object key.
-   * @throws IOException if exception happens, non of the blocks is deleted.
-   */
-  @Override
-  public void deleteBlocks(List<String> blockIDs) throws IOException {
-    if (!nodeManager.isOutOfChillMode()) {
-      throw new SCMException("Unable to delete block while in chill mode",
-          CHILL_MODE_EXCEPTION);
-    }
-
-    lock.lock();
-    LOG.info("Deleting blocks {}", String.join(",", blockIDs));
-    Map<String, List<String>> containerBlocks = new HashMap<>();
-    BatchOperation batch = new BatchOperation();
-    BatchOperation rollbackBatch = new BatchOperation();
-    // TODO: track the block size info so that we can reclaim the container
-    // TODO: used space when the block is deleted.
-    try {
-      for (String blockKey : blockIDs) {
-        byte[] blockKeyBytes = DFSUtil.string2Bytes(blockKey);
-        byte[] containerBytes = blockStore.get(blockKeyBytes);
-        if (containerBytes == null) {
-          throw new SCMException(
-              "Specified block key does not exist. key : " + blockKey,
-              FAILED_TO_FIND_BLOCK);
-        }
-        batch.delete(blockKeyBytes);
-        rollbackBatch.put(blockKeyBytes, containerBytes);
-
-        // Merge blocks to a container to blocks mapping,
-        // prepare to persist this info to the deletedBlocksLog.
-        String containerName = DFSUtil.bytes2String(containerBytes);
-        if (containerBlocks.containsKey(containerName)) {
-          containerBlocks.get(containerName).add(blockKey);
-        } else {
-          List<String> item = new ArrayList<>();
-          item.add(blockKey);
-          containerBlocks.put(containerName, item);
-        }
-      }
-
-      // We update SCM DB first, so if this step fails, we end up here,
-      // nothing gets into the delLog so no blocks will be accidentally
-      // removed. If we write the log first, once log is written, the
-      // async deleting service will start to scan and might be picking
-      // up some blocks to do real deletions, that might cause data loss.
-      blockStore.writeBatch(batch);
-      try {
-        deletedBlockLog.addTransactions(containerBlocks);
-      } catch (IOException e) {
-        try {
-          // If delLog update is failed, we need to rollback the changes.
-          blockStore.writeBatch(rollbackBatch);
-        } catch (IOException rollbackException) {
-          // This is a corner case. AddTX fails and rollback also fails,
-          // this will leave these blocks in inconsistent state. They were
-          // moved to pending deletion state in SCM DB but were not written
-          // into delLog so real deletions would not be done. Blocks become
-          // to be invisible from namespace but actual data are not removed.
-          // We log an error here so admin can manually check and fix such
-          // errors.
-          LOG.error(
-              "Blocks might be in inconsistent state because"
-                  + " they were moved to pending deletion state in SCM DB but"
-                  + " not written into delLog. Admin can manually add them"
-                  + " into delLog for deletions. Inconsistent block list: {}",
-              String.join(",", blockIDs),
-              e);
-          throw rollbackException;
-        }
-        throw new IOException(
-            "Skip writing the deleted blocks info to"
-                + " the delLog because addTransaction fails. Batch skipped: "
-                + String.join(",", blockIDs),
-            e);
-      }
-      // TODO: Container report handling of the deleted blocks:
-      // Remove tombstone and update open container usage.
-      // We will revisit this when the closed container replication is done.
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  @Override
-  public DeletedBlockLog getDeletedBlockLog() {
-    return this.deletedBlockLog;
-  }
-
-  @VisibleForTesting
-  public String getDeletedKeyName(String key) {
-    return StringUtils.format(".Deleted/%s", key);
-  }
-
-  /**
-   * Close the resources for BlockManager.
-   *
-   * @throws IOException
-   */
-  @Override
-  public void close() throws IOException {
-    if (blockStore != null) {
-      blockStore.close();
-    }
-    if (deletedBlockLog != null) {
-      deletedBlockLog.close();
-    }
-    blockDeletingService.shutdown();
-    if (mxBean != null) {
-      MBeans.unregister(mxBean);
-      mxBean = null;
-    }
-  }
-
-  @Override
-  public int getOpenContainersNo() {
-    return 0;
-    // TODO : FIX ME : The open container being a single number does not make
-    // sense.
-    // We have to get open containers by Replication Type and Replication
-    // factor. Hence returning 0 for now.
-    // containers.get(OzoneProtos.LifeCycleState.OPEN).size();
-  }
-
-  @Override
-  public SCMBlockDeletingService getSCMBlockDeletingService() {
-    return this.blockDeletingService;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockmanagerMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockmanagerMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockmanagerMXBean.java
deleted file mode 100644
index efcfc63..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockmanagerMXBean.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.block;
-
-
-/**
- * JMX interface for the block manager.
- */
-public interface BlockmanagerMXBean {
-
-  /**
-   * Number of open containers manager by the block manager.
-   */
-  int getOpenContainersNo();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DatanodeDeletedBlockTransactions.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DatanodeDeletedBlockTransactions.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DatanodeDeletedBlockTransactions.java
deleted file mode 100644
index 31c61dd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DatanodeDeletedBlockTransactions.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.block;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-import org.apache.hadoop.ozone.scm.container.Mapping;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-
-import com.google.common.collect.ArrayListMultimap;
-
-/**
- * A wrapper class to hold info about datanode and all deleted block
- * transactions that will be sent to this datanode.
- */
-public class DatanodeDeletedBlockTransactions {
-  private int nodeNum;
-  // The throttle size for each datanode.
-  private int maximumAllowedTXNum;
-  // Current counter of inserted TX.
-  private int currentTXNum;
-  private Mapping mappingService;
-  // A list of TXs mapped to a certain datanode ID.
-  private final ArrayListMultimap<DatanodeID, DeletedBlocksTransaction>
-      transactions;
-
-  DatanodeDeletedBlockTransactions(Mapping mappingService,
-      int maximumAllowedTXNum, int nodeNum) {
-    this.transactions = ArrayListMultimap.create();
-    this.mappingService = mappingService;
-    this.maximumAllowedTXNum = maximumAllowedTXNum;
-    this.nodeNum = nodeNum;
-  }
-
-  public void addTransaction(DeletedBlocksTransaction tx) throws IOException {
-    ContainerInfo info = null;
-    try {
-      info = mappingService.getContainer(tx.getContainerName());
-    } catch (IOException e) {
-      SCMBlockDeletingService.LOG.warn("Got container info error.", e);
-    }
-
-    if (info == null) {
-      SCMBlockDeletingService.LOG.warn(
-          "Container {} not found, continue to process next",
-          tx.getContainerName());
-      return;
-    }
-
-    for (DatanodeID dnID : info.getPipeline().getMachines()) {
-      if (transactions.containsKey(dnID)) {
-        List<DeletedBlocksTransaction> txs = transactions.get(dnID);
-        if (txs != null && txs.size() < maximumAllowedTXNum) {
-          boolean hasContained = false;
-          for (DeletedBlocksTransaction t : txs) {
-            if (t.getContainerName().equals(tx.getContainerName())) {
-              hasContained = true;
-              break;
-            }
-          }
-
-          if (!hasContained) {
-            txs.add(tx);
-            currentTXNum++;
-          }
-        }
-      } else {
-        currentTXNum++;
-        transactions.put(dnID, tx);
-      }
-      SCMBlockDeletingService.LOG.debug("Transaction added: {} <- TX({})", dnID,
-          tx.getTxID());
-    }
-  }
-
-  Set<DatanodeID> getDatanodes() {
-    return transactions.keySet();
-  }
-
-  boolean isEmpty() {
-    return transactions.isEmpty();
-  }
-
-  boolean hasTransactions(DatanodeID dnID) {
-    return transactions.containsKey(dnID) && !transactions.get(dnID).isEmpty();
-  }
-
-  List<DeletedBlocksTransaction> getDatanodeTransactions(
-      DatanodeID dnID) {
-    return transactions.get(dnID);
-  }
-
-  List<String> getTransactionIDList(DatanodeID dnID) {
-    if (hasTransactions(dnID)) {
-      return transactions.get(dnID).stream()
-          .map(DeletedBlocksTransaction::getTxID).map(String::valueOf)
-          .collect(Collectors.toList());
-    } else {
-      return Collections.emptyList();
-    }
-  }
-
-  boolean isFull() {
-    return currentTXNum >= maximumAllowedTXNum * nodeNum;
-  }
-
-  int getTXNum() {
-    return currentTXNum;
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DeletedBlockLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DeletedBlockLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DeletedBlockLog.java
deleted file mode 100644
index d8af853..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DeletedBlockLog.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.block;
-
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-/**
- * The DeletedBlockLog is a persisted log in SCM to keep tracking
- * container blocks which are under deletion. It maintains info
- * about under-deletion container blocks that notified by KSM,
- * and the state how it is processed.
- */
-public interface DeletedBlockLog extends Closeable {
-
-  /**
-   *  A limit size list of transactions. Note count is the max number
-   *  of TXs to return, we might not be able to always return this
-   *  number. and the processCount of those transactions
-   *  should be [0, MAX_RETRY).
-   *
-   * @param count - number of transactions.
-   * @return a list of BlockDeletionTransaction.
-   */
-  List<DeletedBlocksTransaction> getTransactions(int count)
-      throws IOException;
-
-  /**
-   * Scan entire log once and returns TXs to DatanodeDeletedBlockTransactions.
-   * Once DatanodeDeletedBlockTransactions is full, the scan behavior will
-   * stop.
-   * @param transactions a list of TXs will be set into.
-   * @throws IOException
-   */
-  void getTransactions(DatanodeDeletedBlockTransactions transactions)
-      throws IOException;
-
-  /**
-   * Return all failed transactions in the log. A transaction is considered
-   * to be failed if it has been sent more than MAX_RETRY limit and its
-   * count is reset to -1.
-   *
-   * @return a list of failed deleted block transactions.
-   * @throws IOException
-   */
-  List<DeletedBlocksTransaction> getFailedTransactions()
-      throws IOException;
-
-  /**
-   * Increments count for given list of transactions by 1.
-   * The log maintains a valid range of counts for each transaction
-   * [0, MAX_RETRY]. If exceed this range, resets it to -1 to indicate
-   * the transaction is no longer valid.
-   *
-   * @param txIDs - transaction ID.
-   */
-  void incrementCount(List<Long> txIDs)
-      throws IOException;
-
-  /**
-   * Commits a transaction means to delete all footprints of a transaction
-   * from the log. This method doesn't guarantee all transactions can be
-   * successfully deleted, it tolerate failures and tries best efforts to.
-   *
-   * @param txIDs - transaction IDs.
-   */
-  void commitTransactions(List<Long> txIDs) throws IOException;
-
-  /**
-   * Creates a block deletion transaction and adds that into the log.
-   *
-   * @param containerName - container name.
-   * @param blocks - blocks that belong to the same container.
-   *
-   * @throws IOException
-   */
-  void addTransaction(String containerName, List<String> blocks)
-      throws IOException;
-
-  /**
-   * Creates block deletion transactions for a set of containers,
-   * add into the log and persist them atomically. An object key
-   * might be stored in multiple containers and multiple blocks,
-   * this API ensures that these updates are done in atomic manner
-   * so if any of them fails, the entire operation fails without
-   * any updates to the log. Note, this doesn't mean to create only
-   * one transaction, it creates multiple transactions (depends on the
-   * number of containers) together (on success) or non (on failure).
-   *
-   * @param containerBlocksMap a map of containerBlocks.
-   * @throws IOException
-   */
-  void addTransactions(Map<String, List<String>> containerBlocksMap)
-      throws IOException;
-
-  /**
-   * Returns the total number of valid transactions. A transaction is
-   * considered to be valid as long as its count is in range [0, MAX_RETRY].
-   *
-   * @return number of a valid transactions.
-   * @throws IOException
-   */
-  int getNumOfValidTransactions() throws IOException;
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[46/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java
new file mode 100644
index 0000000..90fe802
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.scm.client.ScmClient;
+import org.apache.hadoop.cblock.util.MockStorageClient;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This class tests the basics of CBlock server. Mainly about the four
+ * operations on volumes: create, delete, info and list.
+ */
+public class TestCBlockServer {
+  private static CBlockManager cBlockManager;
+  private static OzoneConfiguration conf;
+
+  @Before
+  public void setup() throws Exception {
+    ScmClient storageClient = new MockStorageClient();
+    conf = new OzoneConfiguration();
+    conf.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0");
+    conf.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0");
+    cBlockManager = new CBlockManager(conf, storageClient);
+    cBlockManager.start();
+  }
+
+  @After
+  public void clean() {
+    cBlockManager.stop();
+    cBlockManager.join();
+    cBlockManager.clean();
+  }
+
+  /**
+   * Test create volume for different users.
+   * @throws Exception
+   */
+  @Test
+  public void testCreateVolume() throws Exception {
+    String userName1 = "user" + RandomStringUtils.randomNumeric(5);
+    String userName2 = "user" + RandomStringUtils.randomNumeric(5);
+    String volumeName1 = "volume" +  RandomStringUtils.randomNumeric(5);
+    String volumeName2 = "volume" +   RandomStringUtils.randomNumeric(5);
+    long volumeSize = 1L*1024*1024;
+    int blockSize = 4096;
+    cBlockManager.createVolume(userName1, volumeName1, volumeSize, blockSize);
+    List<VolumeInfo> volumes = cBlockManager.listVolume(userName1);
+    assertEquals(1, volumes.size());
+    VolumeInfo existingVolume = volumes.get(0);
+    assertEquals(userName1, existingVolume.getUserName());
+    assertEquals(volumeName1, existingVolume.getVolumeName());
+    assertEquals(volumeSize, existingVolume.getVolumeSize());
+    assertEquals(blockSize, existingVolume.getBlockSize());
+
+    cBlockManager.createVolume(userName1, volumeName2, volumeSize, blockSize);
+    cBlockManager.createVolume(userName2, volumeName1, volumeSize, blockSize);
+    volumes = cBlockManager.listVolume(userName1);
+    assertEquals(2, volumes.size());
+    volumes = cBlockManager.listVolume(userName2);
+    assertEquals(1, volumes.size());
+  }
+
+  /**
+   * Test delete volume.
+   * @throws Exception
+   */
+  @Test
+  public void testDeleteVolume() throws Exception {
+    String userName = "user" + RandomStringUtils.randomNumeric(5);
+    String volumeName1 = "volume" +  RandomStringUtils.randomNumeric(5);
+    String volumeName2 = "volume" +  RandomStringUtils.randomNumeric(5);
+    long volumeSize = 1L*1024*1024;
+    int blockSize = 4096;
+    cBlockManager.createVolume(userName, volumeName1, volumeSize, blockSize);
+    cBlockManager.createVolume(userName, volumeName2, volumeSize, blockSize);
+    cBlockManager.deleteVolume(userName, volumeName1, true);
+    List<VolumeInfo> volumes = cBlockManager.listVolume(userName);
+    assertEquals(1, volumes.size());
+
+    VolumeInfo existingVolume = volumes.get(0);
+    assertEquals(userName, existingVolume.getUserName());
+    assertEquals(volumeName2, existingVolume.getVolumeName());
+    assertEquals(volumeSize, existingVolume.getVolumeSize());
+    assertEquals(blockSize, existingVolume.getBlockSize());
+  }
+
+  /**
+   * Test info volume.
+   *
+   * TODO : usage field is not being tested (as it is not implemented yet)
+   * @throws Exception
+   */
+  @Test
+  public void testInfoVolume() throws Exception {
+    String userName = "user" + RandomStringUtils.randomNumeric(5);
+    String volumeName = "volume" +  RandomStringUtils.randomNumeric(5);
+    long volumeSize = 1L*1024*1024;
+    int blockSize = 4096;
+    cBlockManager.createVolume(userName, volumeName, volumeSize, blockSize);
+    VolumeInfo info = cBlockManager.infoVolume(userName, volumeName);
+    assertEquals(userName, info.getUserName());
+    assertEquals(volumeName, info.getVolumeName());
+    assertEquals(volumeSize, info.getVolumeSize());
+    assertEquals(blockSize, info.getBlockSize());
+  }
+
+  /**
+   * Test listing a number of volumes.
+   * @throws Exception
+   */
+  @Test
+  public void testListVolume() throws Exception {
+    String userName = "user" + RandomStringUtils.randomNumeric(5);
+    String volumeName ="volume" +  RandomStringUtils.randomNumeric(5);
+    long volumeSize = 1L*1024*1024;
+    int blockSize = 4096;
+    int volumeNum = 100;
+    for (int i = 0; i<volumeNum; i++) {
+      cBlockManager.createVolume(userName, volumeName + i,
+          volumeSize, blockSize);
+    }
+    List<VolumeInfo> volumes = cBlockManager.listVolume(userName);
+    assertEquals(volumeNum, volumes.size());
+    Set<String> volumeIds = new HashSet<>();
+    for (int i = 0; i<volumeNum; i++) {
+      VolumeInfo volumeInfo = volumes.get(i);
+      assertEquals(userName, volumeInfo.getUserName());
+      assertFalse(volumeIds.contains(volumeName + i));
+      volumeIds.add(volumeName + i);
+      assertEquals(volumeSize, volumeInfo.getVolumeSize());
+      assertEquals(blockSize, volumeInfo.getBlockSize());
+    }
+    for (int i = 0; i<volumeNum; i++) {
+      assertTrue(volumeIds.contains(volumeName + i));
+    }
+  }
+
+  /**
+   * Test listing a number of volumes.
+   * @throws Exception
+   */
+  @Test
+  public void testListVolumes() throws Exception {
+    String volumeName ="volume" +  RandomStringUtils.randomNumeric(5);
+    long volumeSize = 1L*1024*1024;
+    int blockSize = 4096;
+    int volumeNum = 100;
+    int userCount = 10;
+
+    assertTrue("We need at least one volume for each user",
+        userCount < volumeNum);
+
+    for (int i = 0; i<volumeNum; i++) {
+      String userName =
+          "user-" + (i % userCount);
+      cBlockManager.createVolume(userName, volumeName + i,
+          volumeSize, blockSize);
+    }
+    List<VolumeInfo> allVolumes = cBlockManager.listVolumes();
+    //check if we have the volumes from all the users.
+
+    Set<String> volumeIds = new HashSet<>();
+    Set<String> usernames = new HashSet<>();
+    for (int i = 0; i < allVolumes.size(); i++) {
+      VolumeInfo volumeInfo = allVolumes.get(i);
+      assertFalse(volumeIds.contains(volumeName + i));
+      usernames.add(volumeInfo.getUserName());
+      volumeIds.add(volumeName + i);
+      assertEquals(volumeSize, volumeInfo.getVolumeSize());
+      assertEquals(blockSize, volumeInfo.getBlockSize());
+    }
+
+    assertEquals(volumeNum, volumeIds.size());
+    for (int i = 0; i<volumeNum; i++) {
+      assertTrue(volumeIds.contains(volumeName + i));
+    }
+
+    assertEquals(userCount, usernames.size());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServerPersistence.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServerPersistence.java
new file mode 100644
index 0000000..1ea183b
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServerPersistence.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import org.apache.hadoop.cblock.meta.VolumeDescriptor;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.scm.client.ScmClient;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.cblock.util.MockStorageClient;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test the CBlock server state is maintained in persistent storage and can be
+ * recovered on CBlock server restart.
+ */
+public class TestCBlockServerPersistence {
+
+  /**
+   * Test when cblock server fails with volume meta data, the meta data can be
+   * restored correctly.
+   * @throws Exception
+   */
+  @Test
+  public void testWriteToPersistentStore() throws Exception {
+    OzoneConfiguration conf = new OzoneConfiguration();
+    String userName = "testWriteToPersistentStore";
+    String volumeName1 = "testVolume1";
+    String volumeName2 = "testVolume2";
+    long volumeSize1 = 30L*1024*1024*1024;
+    long volumeSize2 = 15L*1024*1024*1024;
+    int blockSize = 4096;
+    CBlockManager cBlockManager = null;
+    CBlockManager cBlockManager1 = null;
+    String path = GenericTestUtils
+        .getTempPath(TestCBlockServerPersistence.class.getSimpleName());
+    File filePath = new File(path);
+    if(!filePath.exists() && !filePath.mkdirs()) {
+      throw new IOException("Unable to create test DB dir");
+    }
+    conf.set(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY, path.concat(
+        "/testCblockPersistence.dat"));
+    try {
+      ScmClient storageClient = new MockStorageClient();
+      conf.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0");
+      conf.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0");
+      cBlockManager = new CBlockManager(conf, storageClient);
+      cBlockManager.start();
+      cBlockManager.createVolume(userName, volumeName1, volumeSize1, blockSize);
+      cBlockManager.createVolume(userName, volumeName2, volumeSize2, blockSize);
+      List<VolumeDescriptor> allVolumes = cBlockManager.getAllVolumes();
+      // close the cblock server. Since meta data is written to disk on volume
+      // creation, closing server here is the same as a cblock server crash.
+      cBlockManager.close();
+      cBlockManager.stop();
+      cBlockManager.join();
+      cBlockManager = null;
+      assertEquals(2, allVolumes.size());
+      VolumeDescriptor volumeDescriptor1 = allVolumes.get(0);
+      VolumeDescriptor volumeDescriptor2 = allVolumes.get(1);
+
+      // create a new cblock server instance. This is just the
+      // same as restarting cblock server.
+      ScmClient storageClient1 = new MockStorageClient();
+      OzoneConfiguration conf1 = new OzoneConfiguration();
+      conf1.set(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY, path.concat(
+          "/testCblockPersistence.dat"));
+      conf1.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0");
+      conf1.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0");
+      cBlockManager1 = new CBlockManager(conf1, storageClient1);
+      cBlockManager1.start();
+      List<VolumeDescriptor> allVolumes1 = cBlockManager1.getAllVolumes();
+      assertEquals(2, allVolumes1.size());
+      VolumeDescriptor newvolumeDescriptor1 = allVolumes1.get(0);
+      VolumeDescriptor newvolumeDescriptor2 = allVolumes1.get(1);
+
+      // It seems levelDB iterator gets keys in the same order as keys
+      // are inserted, in which case the else clause should never happen.
+      // But still kept the second clause if it is possible to get different
+      // key ordering from leveldb. And we do not rely on the ordering of keys
+      // here.
+      if (volumeDescriptor1.getVolumeName().equals(
+          newvolumeDescriptor1.getVolumeName())) {
+        assertEquals(volumeDescriptor1.toString(),
+            newvolumeDescriptor1.toString());
+        assertEquals(volumeDescriptor2.toString(),
+            newvolumeDescriptor2.toString());
+      } else {
+        assertEquals(volumeDescriptor1.toString(),
+            newvolumeDescriptor2.toString());
+        assertEquals(volumeDescriptor2.toString(),
+            newvolumeDescriptor1.toString());
+      }
+    } finally {
+      if (cBlockManager != null) {
+        cBlockManager.clean();
+      }
+      if (cBlockManager1 != null) {
+        cBlockManager1.close();
+        cBlockManager1.stop();
+        cBlockManager1.join();
+        cBlockManager1.clean();
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java
new file mode 100644
index 0000000..6eb7ea6
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import com.google.common.primitives.Longs;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.cblock.jscsiHelper.CBlockIStorageImpl;
+import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics;
+import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher;
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.MiniOzoneClassicCluster;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.apache.hadoop.scm.XceiverClientSpi;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static java.lang.Math.abs;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+    DFS_CBLOCK_DISK_CACHE_PATH_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+    DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+    DFS_CBLOCK_TRACE_IO;
+
+/**
+ * Tests for local cache.
+ */
+public class TestLocalBlockCache {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestLocalBlockCache.class);
+  private final static long GB = 1024 * 1024 * 1024;
+  private final static int KB = 1024;
+  private static MiniOzoneCluster cluster;
+  private static OzoneConfiguration config;
+  private static StorageContainerLocationProtocolClientSideTranslatorPB
+      storageContainerLocationClient;
+  private static XceiverClientManager xceiverClientManager;
+
+  @BeforeClass
+  public static void init() throws IOException {
+    config = new OzoneConfiguration();
+    String path = GenericTestUtils
+        .getTempPath(TestLocalBlockCache.class.getSimpleName());
+    config.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+    config.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+    config.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+    cluster = new MiniOzoneClassicCluster.Builder(config)
+        .numDataNodes(1)
+        .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
+    storageContainerLocationClient = cluster
+        .createStorageContainerLocationClient();
+    xceiverClientManager = new XceiverClientManager(config);
+  }
+
+  @AfterClass
+  public static void shutdown() throws InterruptedException {
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+    IOUtils.cleanupWithLogger(null, storageContainerLocationClient, cluster);
+  }
+
+  /**
+   * getContainerPipelines creates a set of containers and returns the
+   * Pipelines that define those containers.
+   *
+   * @param count - Number of containers to create.
+   * @return - List of Pipelines.
+   * @throws IOException throws Exception
+   */
+  private List<Pipeline> getContainerPipeline(int count) throws IOException {
+    List<Pipeline> containerPipelines = new LinkedList<>();
+    for (int x = 0; x < count; x++) {
+      String traceID = "trace" + RandomStringUtils.randomNumeric(4);
+      String containerName = "container" + RandomStringUtils.randomNumeric(10);
+      Pipeline pipeline =
+          storageContainerLocationClient.allocateContainer(
+              xceiverClientManager.getType(),
+              xceiverClientManager.getFactor(), containerName, "CBLOCK");
+      XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline);
+      ContainerProtocolCalls.createContainer(client, traceID);
+      // This step is needed since we set private data on pipelines, when we
+      // read the list from CBlockServer. So we mimic that action here.
+      pipeline.setData(Longs.toByteArray(x));
+      containerPipelines.add(pipeline);
+      xceiverClientManager.releaseClient(client);
+    }
+    return containerPipelines;
+  }
+
+  /**
+   * This test creates a cache and performs a simple write / read.
+   * Due to the cache - we have Read-after-write consistency for cBlocks.
+   *
+   * @throws IOException throws Exception
+   */
+  @Test
+  public void testCacheWriteRead() throws IOException,
+      InterruptedException, TimeoutException {
+    final long blockID = 0;
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+    String userName = "user" + RandomStringUtils.randomNumeric(4);
+    String data = RandomStringUtils.random(4 * KB);
+    String dataHash = DigestUtils.sha256Hex(data);
+    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
+        xceiverClientManager, metrics);
+    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+        .setConfiguration(this.config)
+        .setVolumeName(volumeName)
+        .setUserName(userName)
+        .setPipelines(getContainerPipeline(10))
+        .setClientManager(xceiverClientManager)
+        .setBlockSize(4 * KB)
+        .setVolumeSize(50 * GB)
+        .setFlusher(flusher)
+        .setCBlockTargetMetrics(metrics)
+        .build();
+    cache.start();
+    cache.put(blockID, data.getBytes(StandardCharsets.UTF_8));
+    Assert.assertEquals(1, metrics.getNumWriteOps());
+    // Please note that this read is from the local cache.
+    LogicalBlock block = cache.get(blockID);
+    Assert.assertEquals(1, metrics.getNumReadOps());
+    Assert.assertEquals(1, metrics.getNumReadCacheHits());
+    Assert.assertEquals(0, metrics.getNumReadCacheMiss());
+    Assert.assertEquals(0, metrics.getNumReadLostBlocks());
+
+    cache.put(blockID + 1, data.getBytes(StandardCharsets.UTF_8));
+    Assert.assertEquals(2, metrics.getNumWriteOps());
+    // Please note that this read is from the local cache.
+    block = cache.get(blockID + 1);
+    Assert.assertEquals(2, metrics.getNumReadOps());
+    Assert.assertEquals(2, metrics.getNumReadCacheHits());
+    Assert.assertEquals(0, metrics.getNumReadCacheMiss());
+    Assert.assertEquals(0, metrics.getNumReadLostBlocks());
+    String readHash = DigestUtils.sha256Hex(block.getData().array());
+    Assert.assertEquals("File content does not match.", dataHash, readHash);
+    GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
+    cache.close();
+
+  }
+
+  @Test
+  public void testCacheWriteToRemoteContainer() throws IOException,
+      InterruptedException, TimeoutException {
+    final long blockID = 0;
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+    String userName = "user" + RandomStringUtils.randomNumeric(4);
+    String data = RandomStringUtils.random(4 * KB);
+    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
+        xceiverClientManager, metrics);
+    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+        .setConfiguration(this.config)
+        .setVolumeName(volumeName)
+        .setUserName(userName)
+        .setPipelines(getContainerPipeline(10))
+        .setClientManager(xceiverClientManager)
+        .setBlockSize(4 * KB)
+        .setVolumeSize(50 * GB)
+        .setFlusher(flusher)
+        .setCBlockTargetMetrics(metrics)
+        .build();
+    cache.start();
+    cache.put(blockID, data.getBytes(StandardCharsets.UTF_8));
+    GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
+    cache.close();
+  }
+
+  @Test
+  public void testCacheWriteToRemote50KBlocks() throws IOException,
+      InterruptedException, TimeoutException {
+    final long totalBlocks = 50 * 1000;
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+    String userName = "user" + RandomStringUtils.randomNumeric(4);
+    String data = RandomStringUtils.random(4 * KB);
+    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
+        xceiverClientManager, metrics);
+    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+        .setConfiguration(this.config)
+        .setVolumeName(volumeName)
+        .setUserName(userName)
+        .setPipelines(getContainerPipeline(10))
+        .setClientManager(xceiverClientManager)
+        .setBlockSize(4 * 1024)
+        .setVolumeSize(50 * GB)
+        .setFlusher(flusher)
+        .setCBlockTargetMetrics(metrics)
+        .build();
+    cache.start();
+    long startTime = Time.monotonicNow();
+    for (long blockid = 0; blockid < totalBlocks; blockid++) {
+      cache.put(blockid, data.getBytes(StandardCharsets.UTF_8));
+    }
+    Assert.assertEquals(totalBlocks, metrics.getNumWriteOps());
+    Assert.assertEquals(totalBlocks,  metrics.getNumBlockBufferUpdates());
+    LOG.info("Wrote 50K blocks, waiting for replication to finish.");
+    GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
+    long endTime = Time.monotonicNow();
+    LOG.info("Time taken for writing {} blocks is {} seconds", totalBlocks,
+        TimeUnit.MILLISECONDS.toSeconds(endTime - startTime));
+    // TODO: Read this data back.
+    cache.close();
+  }
+
+  @Test
+  public void testCacheInvalidBlock() throws IOException {
+    final int blockID = 1024;
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+    String userName = "user" + RandomStringUtils.randomNumeric(4);
+    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
+        xceiverClientManager, metrics);
+    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+        .setConfiguration(this.config)
+        .setVolumeName(volumeName)
+        .setUserName(userName)
+        .setPipelines(getContainerPipeline(10))
+        .setClientManager(xceiverClientManager)
+        .setBlockSize(4 * KB)
+        .setVolumeSize(50 * GB)
+        .setFlusher(flusher)
+        .setCBlockTargetMetrics(metrics)
+        .build();
+    cache.start();
+    // Read a non-existent block ID.
+    LogicalBlock block = cache.get(blockID);
+    Assert.assertNotNull(block);
+    Assert.assertEquals(4 * 1024, block.getData().array().length);
+    Assert.assertEquals(1, metrics.getNumReadOps());
+    Assert.assertEquals(1, metrics.getNumReadLostBlocks());
+    Assert.assertEquals(1, metrics.getNumReadCacheMiss());
+    cache.close();
+  }
+
+  @Test
+  public void testReadWriteCorrectness() throws IOException,
+      InterruptedException, TimeoutException {
+    Random r = new Random();
+    final int maxBlock = 12500000;
+    final int blockCount = 10 * 1000;
+    Map<Long, String> blockShaMap = new HashMap<>();
+    List<Pipeline> pipelines = getContainerPipeline(10);
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+    String userName = "user" + RandomStringUtils.randomNumeric(4);
+    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
+        xceiverClientManager, metrics);
+    final CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+        .setConfiguration(this.config)
+        .setVolumeName(volumeName)
+        .setUserName(userName)
+        .setPipelines(pipelines)
+        .setClientManager(xceiverClientManager)
+        .setBlockSize(4 * KB)
+        .setVolumeSize(50 * GB)
+        .setFlusher(flusher)
+        .setCBlockTargetMetrics(metrics)
+        .build();
+    cache.start();
+    for (int x = 0; x < blockCount; x++) {
+      String data = RandomStringUtils.random(4 * 1024);
+      String dataHash = DigestUtils.sha256Hex(data);
+      long blockId = abs(r.nextInt(maxBlock));
+      blockShaMap.put(blockId, dataHash);
+      cache.put(blockId, data.getBytes(StandardCharsets.UTF_8));
+    }
+    Assert.assertEquals(blockCount, metrics.getNumWriteOps());
+    GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
+    LOG.info("Finished with putting blocks ..starting reading blocks back. " +
+        "unique blocks : {}", blockShaMap.size());
+    // Test reading from local cache.
+    for (Map.Entry<Long, String> entry : blockShaMap.entrySet()) {
+      LogicalBlock block = cache.get(entry.getKey());
+      String blockSha = DigestUtils.sha256Hex(block.getData().array());
+      Assert.assertEquals("Block data is not equal", entry.getValue(),
+          blockSha);
+    }
+    Assert.assertEquals(blockShaMap.size(), metrics.getNumReadOps());
+    Assert.assertEquals(blockShaMap.size(), metrics.getNumReadCacheHits());
+    Assert.assertEquals(0, metrics.getNumReadCacheMiss());
+    Assert.assertEquals(0, metrics.getNumReadLostBlocks());
+
+    LOG.info("Finished with reading blocks, SUCCESS.");
+    // Close and discard local cache.
+    cache.close();
+    LOG.info("Closing the and destroying local cache");
+    CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher newflusher = new ContainerCacheFlusher(this.config,
+        xceiverClientManager, newMetrics);
+    Assert.assertEquals(0, newMetrics.getNumReadCacheHits());
+    CBlockLocalCache newCache = null;
+    try {
+      newCache = CBlockLocalCache.newBuilder()
+          .setConfiguration(this.config)
+          .setVolumeName(volumeName)
+          .setUserName(userName)
+          .setPipelines(pipelines)
+          .setClientManager(xceiverClientManager)
+          .setBlockSize(4 * KB)
+          .setVolumeSize(50 * GB)
+          .setFlusher(newflusher)
+          .setCBlockTargetMetrics(newMetrics)
+          .build();
+      newCache.start();
+      for (Map.Entry<Long, String> entry : blockShaMap.entrySet()) {
+        LogicalBlock block = newCache.get(entry.getKey());
+        String blockSha = DigestUtils.sha256Hex(block.getData().array());
+        Assert.assertEquals("Block data is not equal", entry.getValue(),
+            blockSha);
+      }
+
+      Assert.assertEquals(blockShaMap.size(), newMetrics.getNumReadOps());
+      Assert.assertEquals(blockShaMap.size(), newMetrics.getNumReadCacheHits());
+      Assert.assertEquals(0, newMetrics.getNumReadCacheMiss());
+      Assert.assertEquals(0, newMetrics.getNumReadLostBlocks());
+
+      LOG.info("Finished with reading blocks from remote cache, SUCCESS.");
+    } finally {
+      if (newCache != null) {
+        newCache.close();
+      }
+    }
+  }
+
+  @Test
+  public void testStorageImplReadWrite() throws IOException,
+      InterruptedException, TimeoutException {
+    String userName = "user" + RandomStringUtils.randomNumeric(5);
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+    long volumeSize = 50L * (1024L * 1024L * 1024L);
+    int blockSize = 4096;
+    byte[] data =
+        RandomStringUtils.randomAlphanumeric(10 * (1024 * 1024))
+            .getBytes(StandardCharsets.UTF_8);
+    String hash = DigestUtils.sha256Hex(data);
+    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
+        xceiverClientManager, metrics);
+    CBlockIStorageImpl ozoneStore = CBlockIStorageImpl.newBuilder()
+        .setUserName(userName)
+        .setVolumeName(volumeName)
+        .setVolumeSize(volumeSize)
+        .setBlockSize(blockSize)
+        .setContainerList(getContainerPipeline(10))
+        .setClientManager(xceiverClientManager)
+        .setConf(this.config)
+        .setFlusher(flusher)
+        .setCBlockTargetMetrics(metrics)
+        .build();
+    ozoneStore.write(data, 0);
+
+    byte[] newData = new byte[10 * 1024 * 1024];
+    ozoneStore.read(newData, 0);
+    String newHash = DigestUtils.sha256Hex(newData);
+    Assert.assertEquals("hashes don't match.", hash, newHash);
+    GenericTestUtils.waitFor(() -> !ozoneStore.getCache().isDirtyCache(),
+        100, 20 * 1000);
+    ozoneStore.close();
+  }
+
+  //@Test
+  // Disabling this test for time being since the bug in JSCSI
+  // forces us always to have a local cache.
+  public void testStorageImplNoLocalCache() throws IOException,
+      InterruptedException, TimeoutException {
+    OzoneConfiguration oConfig = new OzoneConfiguration();
+    oConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, false);
+    oConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+    String userName = "user" + RandomStringUtils.randomNumeric(5);
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+    long volumeSize = 50L * (1024L * 1024L * 1024L);
+    int blockSize = 4096;
+    byte[] data =
+        RandomStringUtils.randomAlphanumeric(10 * (1024 * 1024))
+            .getBytes(StandardCharsets.UTF_8);
+    String hash = DigestUtils.sha256Hex(data);
+    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher flusher = new ContainerCacheFlusher(oConfig,
+        xceiverClientManager, metrics);
+    CBlockIStorageImpl ozoneStore = CBlockIStorageImpl.newBuilder()
+        .setUserName(userName)
+        .setVolumeName(volumeName)
+        .setVolumeSize(volumeSize)
+        .setBlockSize(blockSize)
+        .setContainerList(getContainerPipeline(10))
+        .setClientManager(xceiverClientManager)
+        .setConf(oConfig)
+        .setFlusher(flusher)
+        .setCBlockTargetMetrics(metrics)
+        .build();
+    ozoneStore.write(data, 0);
+
+    byte[] newData = new byte[10 * 1024 * 1024];
+    ozoneStore.read(newData, 0);
+    String newHash = DigestUtils.sha256Hex(newData);
+    Assert.assertEquals("hashes don't match.", hash, newHash);
+    GenericTestUtils.waitFor(() -> !ozoneStore.getCache().isDirtyCache(),
+        100, 20 * 1000);
+    ozoneStore.close();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java
new file mode 100644
index 0000000..8d1a865
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.cblock.kubernetes;
+
+import io.kubernetes.client.JSON;
+import io.kubernetes.client.models.V1PersistentVolume;
+import io.kubernetes.client.models.V1PersistentVolumeClaim;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_ISCSI_ADVERTISED_IP;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+
+/**
+ * Test the resource generation of Dynamic Provisioner.
+ */
+public class TestDynamicProvisioner {
+
+  @Test
+  public void persitenceVolumeBuilder() throws Exception {
+
+    OzoneConfiguration conf = new OzoneConfiguration();
+    conf.setStrings(DFS_CBLOCK_ISCSI_ADVERTISED_IP, "1.2.3.4");
+
+    DynamicProvisioner provisioner =
+        new DynamicProvisioner(conf, null);
+
+    String pvc = new String(Files.readAllBytes(
+        Paths.get(getClass().getResource(
+                "/dynamicprovisioner/input1-pvc.json").toURI())));
+
+    String pv = new String(Files.readAllBytes(
+        Paths.get(getClass().getResource(
+            "/dynamicprovisioner/expected1-pv.json").toURI())));
+
+    JSON json = new io.kubernetes.client.JSON();
+
+    V1PersistentVolumeClaim claim =
+        json.getGson().fromJson(pvc, V1PersistentVolumeClaim.class);
+
+    String volumeName = provisioner.createVolumeName(claim);
+
+    V1PersistentVolume volume =
+        provisioner.persitenceVolumeBuilder(claim, volumeName);
+
+    //remove the data which should not been compared
+    V1PersistentVolume expectedVolume =
+        json.getGson().fromJson(pv, V1PersistentVolume.class);
+
+
+    Assert.assertEquals(expectedVolume, volume);
+  }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java
new file mode 100644
index 0000000..8cb57d6
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.util;
+
+import org.apache.hadoop.cblock.meta.ContainerDescriptor;
+import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+
+import java.io.IOException;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * NOTE : This class is only for testing purpose.
+ *
+ * Mock an underlying container storage layer, expose to CBlock to perform
+ * IO. While in this mock implementation, a container is nothing more than
+ * a in memory hashmap.
+ *
+ * This is to allow volume creation call and perform standalone tests.
+ */
+public final class ContainerLookUpService {
+  private static ConcurrentHashMap<String, ContainerDescriptor>
+      containers = new ConcurrentHashMap<>();
+
+  /**
+   * Return an *existing* container with given Id.
+   *
+   * TODO : for testing purpose, return a new container if the given Id
+   * is not found
+   *
+   * found
+   * @param containerID
+   * @return the corresponding pipeline instance (create if not exist)
+   */
+  public static ContainerDescriptor lookUp(String containerID)
+      throws IOException {
+    if (!containers.containsKey(containerID)) {
+      Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline(
+          containerID);
+      ContainerDescriptor cd = new ContainerDescriptor(containerID);
+      cd.setPipeline(pipeline);
+      containers.put(containerID, cd);
+    }
+    return containers.get(containerID);
+  }
+
+  public static void addContainer(String containerID) throws IOException {
+    Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline(
+        containerID);
+    ContainerDescriptor cd = new ContainerDescriptor(containerID);
+    cd.setPipeline(pipeline);
+    containers.put(containerID, cd);
+  }
+
+  private ContainerLookUpService() {
+
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/MockStorageClient.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/MockStorageClient.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/MockStorageClient.java
new file mode 100644
index 0000000..59c8e01
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/MockStorageClient.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.util;
+
+import org.apache.hadoop.cblock.meta.ContainerDescriptor;
+import org.apache.hadoop.hdsl.protocol.proto.ContainerProtos.ContainerData;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.hdsl.protocol.proto.HdslProtos;
+import org.apache.hadoop.scm.client.ScmClient;
+import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * This class is the one that directly talks to SCM server.
+ *
+ * NOTE : this is only a mock class, only to allow testing volume
+ * creation without actually creating containers. In real world, need to be
+ * replaced with actual container look up calls.
+ *
+ */
+public class MockStorageClient implements ScmClient {
+  private static AtomicInteger currentContainerId =
+      new AtomicInteger(0);
+
+  /**
+   * Ask SCM to get a exclusive container.
+   *
+   * @return A container descriptor object to locate this container
+   * @throws Exception
+   */
+  @Override
+  public Pipeline createContainer(String containerId, String owner)
+      throws IOException {
+    int contId = currentContainerId.getAndIncrement();
+    ContainerLookUpService.addContainer(Long.toString(contId));
+    return ContainerLookUpService.lookUp(Long.toString(contId))
+        .getPipeline();
+  }
+
+  /**
+   * As this is only a testing class, with all "container" maintained in
+   * memory, no need to really delete anything for now.
+   * @throws IOException
+   */
+  @Override
+  public void deleteContainer(Pipeline pipeline, boolean force)
+      throws IOException {
+
+  }
+
+  /**
+   * This is a mock class, so returns the container infos of start container
+   * and end container.
+   *
+   * @param startName start container name.
+   * @param prefixName prefix container name.
+   * @param count count.
+   * @return a list of pipeline.
+   * @throws IOException
+   */
+  @Override
+  public List<ContainerInfo> listContainer(String startName,
+      String prefixName, int count) throws IOException {
+    List<ContainerInfo> containerList = new ArrayList<>();
+    ContainerDescriptor containerDescriptor =
+        ContainerLookUpService.lookUp(startName);
+    ContainerInfo container = new ContainerInfo.Builder()
+        .setContainerName(containerDescriptor.getContainerID())
+        .setPipeline(containerDescriptor.getPipeline())
+        .setState(HdslProtos.LifeCycleState.ALLOCATED)
+        .build();
+    containerList.add(container);
+    return containerList;
+  }
+
+  /**
+   * Create a instance of ContainerData by a given container id,
+   * since this is a testing class, there is no need set up the hold
+   * env to get the meta data of the container.
+   * @param pipeline
+   * @return
+   * @throws IOException
+   */
+  @Override
+  public ContainerData readContainer(Pipeline pipeline) throws IOException {
+    return ContainerData.newBuilder()
+        .setName(pipeline.getContainerName())
+        .build();
+  }
+
+  /**
+   * Return reference to an *existing* container with given ID.
+   *
+   * @param containerId
+   * @return
+   * @throws IOException
+   */
+  public Pipeline getContainer(String containerId)
+      throws IOException {
+    return ContainerLookUpService.lookUp(containerId).getPipeline();
+  }
+
+  @Override
+  public void closeContainer(Pipeline container) throws IOException {
+    // Do nothing, because the mock container does not have the notion of
+    // "open" and "close".
+  }
+
+  @Override
+  public long getContainerSize(Pipeline pipeline) throws IOException {
+    // just return a constant value for now
+    return 5L * OzoneConsts.GB; // 5GB
+  }
+
+  @Override
+  public Pipeline createContainer(HdslProtos.ReplicationType type,
+      HdslProtos.ReplicationFactor replicationFactor, String containerId,
+      String owner) throws IOException {
+    int contId = currentContainerId.getAndIncrement();
+    ContainerLookUpService.addContainer(Long.toString(contId));
+    return ContainerLookUpService.lookUp(Long.toString(contId))
+        .getPipeline();
+  }
+
+  /**
+   * Returns a set of Nodes that meet a query criteria.
+   *
+   * @param nodeStatuses - A set of criteria that we want the node to have.
+   * @param queryScope - Query scope - Cluster or pool.
+   * @param poolName - if it is pool, a pool name is required.
+   * @return A set of nodes that meet the requested criteria.
+   * @throws IOException
+   */
+  @Override
+  public HdslProtos.NodePool queryNode(EnumSet<HdslProtos.NodeState>
+      nodeStatuses, HdslProtos.QueryScope queryScope, String poolName)
+      throws IOException {
+    return null;
+  }
+
+  /**
+   * Creates a specified replication pipeline.
+   *
+   * @param type - Type
+   * @param factor - Replication factor
+   * @param nodePool - Set of machines.
+   * @throws IOException
+   */
+  @Override
+  public Pipeline createReplicationPipeline(HdslProtos.ReplicationType type,
+      HdslProtos.ReplicationFactor factor, HdslProtos.NodePool nodePool)
+      throws IOException {
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/resources/dynamicprovisioner/expected1-pv.json
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/resources/dynamicprovisioner/expected1-pv.json b/hadoop-cblock/server/src/test/resources/dynamicprovisioner/expected1-pv.json
new file mode 100644
index 0000000..2f8a193
--- /dev/null
+++ b/hadoop-cblock/server/src/test/resources/dynamicprovisioner/expected1-pv.json
@@ -0,0 +1,37 @@
+{
+    "apiVersion": "v1",
+    "kind": "PersistentVolume",
+    "metadata": {
+        "annotations": {
+            "volume.beta.kubernetes.io/storage-class": "cblock",
+            "pv.kubernetes.io/provisioned-by": "hadoop.apache.org/cblock"
+        },
+        "name": "volume1-b65d053d-f92e-11e7-be3b-84b261c34638",
+        "namespace": "ns"
+    },
+    "spec": {
+        "accessModes": [
+            "ReadWriteOnce"
+        ],
+        "capacity": {
+            "storage": "1Gi"
+        },
+        "claimRef": {
+            "apiVersion": "v1",
+            "kind": "PersistentVolumeClaim",
+            "name": "volume1",
+            "namespace": "ns",
+            "uid": "b65d053d-f92e-11e7-be3b-84b261c34638"
+        },
+        "iscsi": {
+            "fsType": "ext4",
+            "iqn": "iqn.2001-04.org.apache.hadoop:volume1-b65d053d-f92e-11e7-be3b-84b261c34638",
+            "lun": 0,
+            "portals": [
+                "1.2.3.4:3260"
+            ],
+            "targetPortal": "1.2.3.4:3260"
+        },
+        "persistentVolumeReclaimPolicy": "Delete"
+    }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/resources/dynamicprovisioner/input1-pvc.json
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/resources/dynamicprovisioner/input1-pvc.json b/hadoop-cblock/server/src/test/resources/dynamicprovisioner/input1-pvc.json
new file mode 100644
index 0000000..e40ba71
--- /dev/null
+++ b/hadoop-cblock/server/src/test/resources/dynamicprovisioner/input1-pvc.json
@@ -0,0 +1,38 @@
+{
+    "apiVersion": "v1",
+    "kind": "PersistentVolumeClaim",
+    "metadata": {
+        "annotations": {
+            "pv.kubernetes.io/bind-completed": "yes",
+            "pv.kubernetes.io/bound-by-controller": "yes",
+            "volume.beta.kubernetes.io/storage-provisioner": "hadoop.apache.org/cblock"
+        },
+        "creationTimestamp": "2018-01-14T13:27:48Z",
+        "name": "volume1",
+        "namespace": "ns",
+        "resourceVersion": "5532691",
+        "selfLink": "/api/v1/namespaces/demo1/persistentvolumeclaims/persistent",
+        "uid": "b65d053d-f92e-11e7-be3b-84b261c34638"
+    },
+    "spec": {
+        "accessModes": [
+            "ReadWriteOnce"
+        ],
+        "resources": {
+            "requests": {
+                "storage": "1Gi"
+            }
+        },
+        "storageClassName": "cblock",
+        "volumeName": "persistent-b65d053d-f92e-11e7-be3b-84b261c34638"
+    },
+    "status": {
+        "accessModes": [
+            "ReadWriteOnce"
+        ],
+        "capacity": {
+            "storage": "1Gi"
+        },
+        "phase": "Bound"
+    }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-cblock/tools/pom.xml b/hadoop-cblock/tools/pom.xml
new file mode 100644
index 0000000..150f2e5
--- /dev/null
+++ b/hadoop-cblock/tools/pom.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-cblock</artifactId>
+    <version>3.2.0-SNAPSHOT</version>
+  </parent>
+  <artifactId>hadoop-cblock-tools</artifactId>
+  <version>3.2.0-SNAPSHOT</version>
+  <description>Apache Hadoop CBlock Tools</description>
+  <name>Apache Hadoop CBlock Tools</name>
+  <packaging>jar</packaging>
+
+  <properties>
+    <hadoop.component>cblock</hadoop.component>
+    <is.hadoop.component>true</is.hadoop.component>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-cblock-server</artifactId>
+    </dependency>
+  </dependencies>
+</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java b/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java
new file mode 100644
index 0000000..224c908
--- /dev/null
+++ b/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.cli;
+
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import org.apache.hadoop.cblock.CblockUtils;
+import org.apache.hadoop.cblock.client.CBlockVolumeClient;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * The command line tool class.
+ */
+public class CBlockCli extends Configured implements Tool {
+
+  private static final String CREATE_VOLUME = "createVolume";
+
+  private static final String DELETE_VOLUME = "deleteVolume";
+
+  private static final String INFO_VOLUME = "infoVolume";
+
+  private static final String LIST_VOLUME = "listVolume";
+
+  private static final String SERVER_ADDR = "serverAddr";
+
+  private static final String HELP = "help";
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(CBlockCli.class);
+  private OzoneConfiguration conf;
+
+  private PrintStream printStream;
+
+  private Options options;
+
+  private BasicParser parser;
+
+  private CBlockVolumeClient localProxy;
+
+  public CBlockCli(OzoneConfiguration conf, PrintStream printStream)
+      throws IOException {
+    this.printStream = printStream;
+    this.conf = conf;
+    this.options = getOptions();
+    this.parser = new BasicParser();
+  }
+
+  public CBlockCli(OzoneConfiguration conf) throws IOException{
+    this(conf, System.out);
+  }
+
+  private CommandLine parseArgs(String[] argv)
+      throws ParseException {
+    return parser.parse(options, argv);
+  }
+
+  private static Options getOptions() {
+    Options options = new Options();
+    Option serverAddress = OptionBuilder
+        .withArgName("serverAddress>:<serverPort")
+        .withLongOpt(SERVER_ADDR)
+        .withValueSeparator(':')
+        .hasArgs(2)
+        .withDescription("specify server address:port")
+        .create("s");
+    options.addOption(serverAddress);
+
+    // taking 4 args: userName, volumeName, volumeSize, blockSize
+    Option createVolume = OptionBuilder
+        .withArgName("user> <volume> <volumeSize in [GB/TB]> <blockSize")
+        .withLongOpt(CREATE_VOLUME)
+        .withValueSeparator(' ')
+        .hasArgs(4)
+        .withDescription("create a fresh new volume")
+        .create("c");
+    options.addOption(createVolume);
+
+    // taking 2 args: userName, volumeName
+    Option deleteVolume = OptionBuilder
+        .withArgName("user> <volume")
+        .withLongOpt(DELETE_VOLUME)
+        .hasArgs(2)
+        .withDescription("delete a volume")
+        .create("d");
+    options.addOption(deleteVolume);
+
+    // taking 2 args: userName, volumeName
+    Option infoVolume = OptionBuilder
+        .withArgName("user> <volume")
+        .withLongOpt(INFO_VOLUME)
+        .hasArgs(2)
+        .withDescription("info a volume")
+        .create("i");
+    options.addOption(infoVolume);
+
+    // taking 1 arg: userName
+    Option listVolume = OptionBuilder
+        .withArgName("user")
+        .withLongOpt(LIST_VOLUME)
+        .hasOptionalArgs(1)
+        .withDescription("list all volumes")
+        .create("l");
+    options.addOption(listVolume);
+
+    Option help = OptionBuilder
+        .withLongOpt(HELP)
+        .withDescription("help")
+        .create("h");
+    options.addOption(help);
+
+    return options;
+  }
+
+  @Override
+  public int run(String[] args) throws ParseException, IOException {
+    CommandLine commandLine = parseArgs(args);
+    if (commandLine.hasOption("s")) {
+      String[] serverAddrArgs = commandLine.getOptionValues("s");
+      LOG.info("server address" + Arrays.toString(serverAddrArgs));
+      String serverHost = serverAddrArgs[0];
+      int serverPort = Integer.parseInt(serverAddrArgs[1]);
+      InetSocketAddress serverAddress =
+          new InetSocketAddress(serverHost, serverPort);
+      this.localProxy = new CBlockVolumeClient(conf, serverAddress);
+    } else {
+      this.localProxy = new CBlockVolumeClient(conf);
+    }
+
+    if (commandLine.hasOption("h")) {
+      LOG.info("help");
+      help();
+    }
+
+    if (commandLine.hasOption("c")) {
+      String[] createArgs = commandLine.getOptionValues("c");
+      LOG.info("create volume:" + Arrays.toString(createArgs));
+      createVolume(createArgs);
+    }
+
+    if (commandLine.hasOption("d")) {
+      String[] deleteArgs = commandLine.getOptionValues("d");
+      LOG.info("delete args:" + Arrays.toString(deleteArgs));
+      deleteVolume(deleteArgs);
+    }
+
+    if (commandLine.hasOption("l")) {
+      String[] listArg = commandLine.getOptionValues("l");
+      LOG.info("list args:" + Arrays.toString(listArg));
+      listVolume(listArg);
+    }
+
+    if (commandLine.hasOption("i")) {
+      String[] infoArgs = commandLine.getOptionValues("i");
+      LOG.info("info args:" + Arrays.toString(infoArgs));
+      infoVolume(infoArgs);
+    }
+    return 0;
+  }
+
+  public static void main(String[] argv) throws Exception {
+    CblockUtils.activateConfigs();
+    OzoneConfiguration cblockConf = new OzoneConfiguration();
+    RPC.setProtocolEngine(cblockConf, CBlockServiceProtocolPB.class,
+        ProtobufRpcEngine.class);
+    int res = 0;
+    Tool shell = new CBlockCli(cblockConf, System.out);
+    try {
+      ToolRunner.run(shell, argv);
+    } catch (Exception ex) {
+      LOG.error(ex.toString());
+      res = 1;
+    }
+    System.exit(res);
+  }
+
+
+
+  private void createVolume(String[] createArgs) throws IOException {
+    String userName = createArgs[0];
+    String volumeName = createArgs[1];
+    long volumeSize = CblockUtils.parseSize(createArgs[2]);
+    int blockSize = Integer.parseInt(createArgs[3])*1024;
+    localProxy.createVolume(userName, volumeName, volumeSize, blockSize);
+  }
+
+  private void deleteVolume(String[] deleteArgs) throws IOException {
+    String userName = deleteArgs[0];
+    String volumeName = deleteArgs[1];
+    boolean force = false;
+    if (deleteArgs.length > 2) {
+      force = Boolean.parseBoolean(deleteArgs[2]);
+    }
+    localProxy.deleteVolume(userName, volumeName, force);
+  }
+
+  private void infoVolume(String[] infoArgs) throws IOException {
+    String userName = infoArgs[0];
+    String volumeName = infoArgs[1];
+    VolumeInfo volumeInfo = localProxy.infoVolume(userName, volumeName);
+    printStream.println(volumeInfo.toString());
+  }
+
+  private void listVolume(String[] listArgs) throws IOException {
+    StringBuilder stringBuilder = new StringBuilder();
+    List<VolumeInfo> volumeResponse;
+    if (listArgs == null) {
+      volumeResponse = localProxy.listVolume(null);
+    } else {
+      volumeResponse = localProxy.listVolume(listArgs[0]);
+    }
+    for (int i = 0; i<volumeResponse.size(); i++) {
+      stringBuilder.append(
+          String.format("%s:%s\t%d\t%d", volumeResponse.get(i).getUserName(),
+              volumeResponse.get(i).getVolumeName(),
+          volumeResponse.get(i).getVolumeSize(),
+          volumeResponse.get(i).getBlockSize()));
+      if (i < volumeResponse.size() - 1) {
+        stringBuilder.append("\n");
+      }
+    }
+    printStream.println(stringBuilder);
+  }
+
+  private void help() {
+    HelpFormatter formatter = new HelpFormatter();
+    formatter.printHelp(100, "cblock", "", options, "");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/package-info.java b/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/package-info.java
new file mode 100644
index 0000000..b8b1889
--- /dev/null
+++ b/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.cli;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/tools/src/test/org/apache/hadoop/cblock/TestCBlockCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/tools/src/test/org/apache/hadoop/cblock/TestCBlockCLI.java b/hadoop-cblock/tools/src/test/org/apache/hadoop/cblock/TestCBlockCLI.java
new file mode 100644
index 0000000..f8b05ed
--- /dev/null
+++ b/hadoop-cblock/tools/src/test/org/apache/hadoop/cblock/TestCBlockCLI.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import org.apache.hadoop.cblock.cli.CBlockCli;
+import org.apache.hadoop.cblock.meta.VolumeDescriptor;
+import org.apache.hadoop.cblock.util.MockStorageClient;
+import org.apache.hadoop.conf.OzoneConfiguration;
+import org.apache.hadoop.scm.client.ScmClient;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.List;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * A testing class for cblock command line tool.
+ */
+public class TestCBlockCLI {
+  private static final long GB = 1 * 1024 * 1024 * 1024L;
+  private static final int KB = 1024;
+  private static CBlockCli cmd;
+  private static OzoneConfiguration conf;
+  private static CBlockManager cBlockManager;
+  private static ByteArrayOutputStream outContent;
+  private static PrintStream testPrintOut;
+
+  @BeforeClass
+  public static void setup() throws IOException {
+    outContent = new ByteArrayOutputStream();
+    ScmClient storageClient = new MockStorageClient();
+    conf = new OzoneConfiguration();
+    String path = GenericTestUtils
+        .getTempPath(TestCBlockCLI.class.getSimpleName());
+    File filePath = new File(path);
+    if (!filePath.exists() && !filePath.mkdirs()) {
+      throw new IOException("Unable to create test DB dir");
+    }
+    conf.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0");
+    conf.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0");
+    conf.set(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY, path.concat(
+        "/testCblockCli.dat"));
+    cBlockManager = new CBlockManager(conf, storageClient);
+    cBlockManager.start();
+    testPrintOut = new PrintStream(outContent);
+    cmd = new CBlockCli(conf, testPrintOut);
+  }
+
+  @AfterClass
+  public static void clean() {
+    if (cBlockManager != null) {
+      cBlockManager.stop();
+      cBlockManager.join();
+      cBlockManager.clean();
+    }
+  }
+
+  @After
+  public void reset() {
+    outContent.reset();
+  }
+
+  /**
+   * Test the help command.
+   * @throws Exception
+   */
+  @Test
+  public void testCliHelp() throws Exception {
+    PrintStream initialStdOut = System.out;
+    System.setOut(testPrintOut);
+    String[] args = {"-h"};
+    cmd.run(args);
+    String helpPrints =
+        "usage: cblock\n" +
+        " -c,--createVolume <user> <volume> <volumeSize in [GB/TB]> " +
+            "<blockSize>   create a fresh new volume\n" +
+        " -d,--deleteVolume <user> <volume>                         " +
+            "              delete a volume\n" +
+        " -h,--help                                                 " +
+            "              help\n" +
+        " -i,--infoVolume <user> <volume>                           " +
+            "              info a volume\n" +
+        " -l,--listVolume <user>                                    " +
+            "              list all volumes\n" +
+        " -s,--serverAddr <serverAddress>:<serverPort>              " +
+            "              specify server address:port\n";
+    assertEquals(helpPrints, outContent.toString());
+    outContent.reset();
+    System.setOut(initialStdOut);
+  }
+
+  /**
+   * Test volume listing command.
+   * @throws Exception
+   */
+  @Test
+  public void testCliList() throws Exception {
+    String userName0 = "userTestCliList0";
+    String userName1 = "userTestCliList1";
+    String userTestNotExist = "userTestNotExist";
+    String volumeName0 = "volumeTest0";
+    String volumeName1 = "volumeTest1";
+    String volumeSize0 = "30GB";
+    String volumeSize1 = "40GB";
+    String blockSize = Integer.toString(4);
+    String[] argsCreate0 =
+        {"-c", userName0, volumeName0, volumeSize0, blockSize};
+    cmd.run(argsCreate0);
+    String[] argsCreate1 =
+        {"-c", userName0, volumeName1, volumeSize1, blockSize};
+    cmd.run(argsCreate1);
+    String[] argsCreate2 =
+        {"-c", userName1, volumeName0, volumeSize0, blockSize};
+    cmd.run(argsCreate2);
+    String[] argsList0 = {"-l"};
+    cmd.run(argsList0);
+    String[] outExpected1 = {
+        "userTestCliList1:volumeTest0\t32212254720\t4096\n",
+        "userTestCliList0:volumeTest0\t32212254720\t4096\n",
+        "userTestCliList0:volumeTest1\t42949672960\t4096\n"};
+    int length = 0;
+    for (String str : outExpected1) {
+      assertTrue(outContent.toString().contains(str));
+      length += str.length();
+    }
+    assertEquals(length, outContent.toString().length());
+    outContent.reset();
+
+    String[] argsList1 = {"-l", userName1};
+    cmd.run(argsList1);
+    String outExpected2 = "userTestCliList1:volumeTest0\t32212254720\t4096\n";
+    assertEquals(outExpected2, outContent.toString());
+    outContent.reset();
+
+    String[] argsList2 = {"-l", userTestNotExist};
+    cmd.run(argsList2);
+    String outExpected3 = "\n";
+    assertEquals(outExpected3, outContent.toString());
+  }
+
+  /**
+   * Test create volume command.
+   * @throws Exception
+   */
+  @Test
+  public void testCliCreate() throws Exception {
+    String userName = "userTestCliCreate";
+    String volumeName = "volumeTest";
+    String volumeSize = "30GB";
+    String blockSize = "4";
+    String[] argsCreate = {"-c", userName, volumeName, volumeSize, blockSize};
+    cmd.run(argsCreate);
+    List<VolumeDescriptor> allVolumes = cBlockManager.getAllVolumes(userName);
+    assertEquals(1, allVolumes.size());
+    VolumeDescriptor volume = allVolumes.get(0);
+    assertEquals(userName, volume.getUserName());
+    assertEquals(volumeName, volume.getVolumeName());
+    long volumeSizeB = volume.getVolumeSize();
+    assertEquals(30, (int)(volumeSizeB/ GB));
+    assertEquals(4, volume.getBlockSize()/ KB);
+  }
+
+  /**
+   * Test delete volume command.
+   * @throws Exception
+   */
+  @Test
+  public void testCliDelete() throws Exception {
+    String userName = "userTestCliDelete";
+    String volumeName = "volumeTest";
+    String volumeSize = "30GB";
+    String blockSize = "4";
+    String[] argsCreate = {"-c", userName, volumeName, volumeSize, blockSize};
+    cmd.run(argsCreate);
+    List<VolumeDescriptor> allVolumes = cBlockManager.getAllVolumes(userName);
+    assertEquals(1, allVolumes.size());
+    VolumeDescriptor volume = allVolumes.get(0);
+    assertEquals(userName, volume.getUserName());
+    assertEquals(volumeName, volume.getVolumeName());
+    long volumeSizeB = volume.getVolumeSize();
+    assertEquals(30, (int)(volumeSizeB/ GB));
+    assertEquals(4, volume.getBlockSize()/ KB);
+
+    String[] argsDelete = {"-d", userName, volumeName};
+    cmd.run(argsDelete);
+    allVolumes = cBlockManager.getAllVolumes(userName);
+    assertEquals(0, allVolumes.size());
+  }
+
+  /**
+   * Test info volume command.
+   * @throws Exception
+   */
+  @Test
+  public void testCliInfoVolume() throws Exception {
+    String userName0 = "userTestCliInfo";
+    String volumeName0 = "volumeTest0";
+    String volumeSize = "8000GB";
+    String blockSize = "4";
+    String[] argsCreate0 = {
+        "-c", userName0, volumeName0, volumeSize, blockSize};
+    cmd.run(argsCreate0);
+    String[] argsInfo = {"-i", userName0, volumeName0};
+    cmd.run(argsInfo);
+    // TODO : the usage field is not implemented yet, always 0 now.
+    String outExpected = " userName:userTestCliInfo " +
+        "volumeName:volumeTest0 " +
+        "volumeSize:8589934592000 " +
+        "blockSize:4096 (sizeInBlocks:2097152000) usageInBlocks:0\n";
+    assertEquals(outExpected, outContent.toString());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index 9ea4587..c3e7113 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -596,6 +596,13 @@ function hadoop_bootstrap
   YARN_LIB_JARS_DIR=${YARN_LIB_JARS_DIR:-"share/hadoop/yarn/lib"}
   MAPRED_DIR=${MAPRED_DIR:-"share/hadoop/mapreduce"}
   MAPRED_LIB_JARS_DIR=${MAPRED_LIB_JARS_DIR:-"share/hadoop/mapreduce/lib"}
+  HDSL_DIR=${HDSL_DIR:-"share/hadoop/hdsl"}
+  HDSL_LIB_JARS_DIR=${HDSL_LIB_JARS_DIR:-"share/hadoop/hdsl/lib"}
+  OZONE_DIR=${OZONE_DIR:-"share/hadoop/ozone"}
+  OZONE_LIB_JARS_DIR=${OZONE_LIB_JARS_DIR:-"share/hadoop/ozone/lib"}
+  CBLOCK_DIR=${CBLOCK_DIR:-"share/hadoop/cblock"}
+  CBLOCK_LIB_JARS_DIR=${CBLOCK_LIB_JARS_DIR:-"share/hadoop/cblock/lib"}
+
   HADOOP_TOOLS_HOME=${HADOOP_TOOLS_HOME:-${HADOOP_HOME}}
   HADOOP_TOOLS_DIR=${HADOOP_TOOLS_DIR:-"share/hadoop/tools"}
   HADOOP_TOOLS_LIB_JARS_DIR=${HADOOP_TOOLS_LIB_JARS_DIR:-"${HADOOP_TOOLS_DIR}/lib"}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
index b19e54d..2128de7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
@@ -17,15 +17,9 @@
  */
 package org.apache.hadoop.conf;
 
-import com.google.gson.Gson;
 import java.io.IOException;
 import java.io.Writer;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
 import javax.servlet.ServletContext;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServlet;
@@ -38,8 +32,6 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.http.HttpServer2;
 
 import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * A servlet to print out the running configuration data.
@@ -51,12 +43,6 @@ public class ConfServlet extends HttpServlet {
 
   protected static final String FORMAT_JSON = "json";
   protected static final String FORMAT_XML = "xml";
-  private static final String COMMAND = "cmd";
-  private static final Logger LOG = LoggerFactory.getLogger(ConfServlet.class);
-  private transient static final Configuration OZONE_CONFIG = new
-      OzoneConfiguration();
-  private transient Map<String, OzoneConfiguration.Property> propertyMap = null;
-
 
   /**
    * Return the Configuration of the daemon hosting this servlet.
@@ -79,27 +65,21 @@ public class ConfServlet extends HttpServlet {
     final ServletContext servletContext = getServletContext();
     if (!HttpServer2.isStaticUserAndNoneAuthType(servletContext, request) &&
         !HttpServer2.isInstrumentationAccessAllowed(servletContext,
-            request, response)) {
+                                                   request, response)) {
       return;
     }
 
     String format = parseAcceptHeader(request);
-    String cmd = request.getParameter(COMMAND);
-    Writer out = response.getWriter();
+    if (FORMAT_XML.equals(format)) {
+      response.setContentType("text/xml; charset=utf-8");
+    } else if (FORMAT_JSON.equals(format)) {
+      response.setContentType("application/json; charset=utf-8");
+    }
 
+    String name = request.getParameter("name");
+    Writer out = response.getWriter();
     try {
-      if (cmd == null) {
-        if (FORMAT_XML.equals(format)) {
-          response.setContentType("text/xml; charset=utf-8");
-        } else if (FORMAT_JSON.equals(format)) {
-          response.setContentType("application/json; charset=utf-8");
-        }
-
-        String name = request.getParameter("name");
-        writeResponse(getConfFromContext(), out, format, name);
-      } else {
-        processConfigTagRequest(request, out);
-      }
+      writeResponse(getConfFromContext(), out, format, name);
     } catch (BadFormatException bfe) {
       response.sendError(HttpServletResponse.SC_BAD_REQUEST, bfe.getMessage());
     } catch (IllegalArgumentException iae) {
@@ -108,67 +88,6 @@ public class ConfServlet extends HttpServlet {
     out.close();
   }
 
-  private void processConfigTagRequest(HttpServletRequest request,
-      Writer out) throws IOException {
-    String cmd = request.getParameter(COMMAND);
-    Gson gson = new Gson();
-    Configuration config = getOzoneConfig();
-
-    config.get("ozone.enabled");
-    switch (cmd) {
-    case "getPropertyByTag":
-      String tags = request.getParameter("tags");
-      String tagGroup = request.getParameter("group");
-      LOG.debug("Getting all properties for tags:" + tags + " group:" +
-          tagGroup);
-      List<String> tagList = new ArrayList<>();
-      for (String tag : tags.split(",")) {
-        if (config.isPropertyTag(tag)) {
-          tagList.add(tag);
-        }
-      }
-
-      Properties properties = config.getAllPropertiesByTags(tagList);
-      if (propertyMap == null) {
-        loadDescriptions();
-      }
-
-      List<OzoneConfiguration.Property> filteredProperties = new ArrayList<>();
-
-      properties.stringPropertyNames().stream().forEach(key -> {
-        if (config.get(key) != null) {
-          propertyMap.get(key).setValue(config.get(key));
-          filteredProperties.add(propertyMap.get(key));
-        }
-      });
-      out.write(gson.toJsonTree(filteredProperties).toString());
-      break;
-    default:
-      throw new IllegalArgumentException(cmd + " is not a valid command.");
-    }
-
-  }
-
-  private void loadDescriptions() {
-    OzoneConfiguration config = (OzoneConfiguration) getOzoneConfig();
-    List<OzoneConfiguration.Property> propList = null;
-    propertyMap = new HashMap<>();
-    try {
-      propList = config.readPropertyFromXml(config.getResource("ozone-site"
-          + ".xml"));
-      propList.stream().map(p -> propertyMap.put(p.getName(), p));
-      propList = config.readPropertyFromXml(config.getResource("ozone-default"
-          + ".xml"));
-      propList.stream().forEach(p -> {
-        if (!propertyMap.containsKey(p.getName().trim())) {
-          propertyMap.put(p.getName().trim(), p);
-        }
-      });
-    } catch (Exception e) {
-      LOG.error("Error while reading description from xml files", e);
-    }
-  }
-
   @VisibleForTesting
   static String parseAcceptHeader(HttpServletRequest request) {
     String format = request.getHeader(HttpHeaders.ACCEPT);
@@ -204,8 +123,4 @@ public class ConfServlet extends HttpServlet {
     }
   }
 
-  private static Configuration getOzoneConfig() {
-    return OZONE_CONFIG;
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/OzoneConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/OzoneConfiguration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/OzoneConfiguration.java
deleted file mode 100644
index ed118bf..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/OzoneConfiguration.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.conf;
-
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Unmarshaller;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * Configuration for ozone.
- */
-@InterfaceAudience.Private
-public class OzoneConfiguration extends Configuration {
-  static {
-    // adds the default resources
-    Configuration.addDefaultResource("hdfs-default.xml");
-    Configuration.addDefaultResource("hdfs-site.xml");
-    Configuration.addDefaultResource("ozone-default.xml");
-    Configuration.addDefaultResource("ozone-site.xml");
-  }
-
-  public OzoneConfiguration() {
-  }
-
-  public OzoneConfiguration(Configuration conf) {
-    super(conf);
-  }
-
-  public List<Property> readPropertyFromXml(URL url) throws JAXBException {
-    JAXBContext context = JAXBContext.newInstance(XMLConfiguration.class);
-    Unmarshaller um = context.createUnmarshaller();
-
-    XMLConfiguration config = (XMLConfiguration) um.unmarshal(url);
-    return config.getProperties();
-  }
-
-  /**
-   * Class to marshall/un-marshall configuration from xml files.
-   */
-  @XmlAccessorType(XmlAccessType.FIELD)
-  @XmlRootElement(name = "configuration")
-  public static class XMLConfiguration {
-
-    @XmlElement(name = "property", type = Property.class)
-    private List<Property> properties = new ArrayList<>();
-
-    public XMLConfiguration() {
-    }
-
-    public XMLConfiguration(List<Property> properties) {
-      this.properties = properties;
-    }
-
-    public List<Property> getProperties() {
-      return properties;
-    }
-
-    public void setProperties(List<Property> properties) {
-      this.properties = properties;
-    }
-  }
-
-  /**
-   * Class to marshall/un-marshall configuration properties from xml files.
-   */
-  @XmlAccessorType(XmlAccessType.FIELD)
-  @XmlRootElement(name = "property")
-  public static class Property implements Comparable<Property> {
-
-    private String name;
-    private String value;
-    private String tag;
-    private String description;
-
-    public String getName() {
-      return name;
-    }
-
-    public void setName(String name) {
-      this.name = name;
-    }
-
-    public String getValue() {
-      return value;
-    }
-
-    public void setValue(String value) {
-      this.value = value;
-    }
-
-    public String getTag() {
-      return tag;
-    }
-
-    public void setTag(String tag) {
-      this.tag = tag;
-    }
-
-    public String getDescription() {
-      return description;
-    }
-
-    public void setDescription(String description) {
-      this.description = description;
-    }
-
-    @Override
-    public int compareTo(Property o) {
-      if (this == o) {
-        return 0;
-      }
-      return this.getName().compareTo(o.getName());
-    }
-
-    @Override
-    public String toString() {
-      return this.getName() + " " + this.getValue() + this.getTag();
-    }
-
-    @Override
-    public int hashCode(){
-      return this.getName().hashCode();
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-      return (obj instanceof Property) && (((Property) obj).getName())
-          .equals(this.getName());
-    }
-  }
-}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[42/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/Header.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/Header.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/Header.java
deleted file mode 100644
index 00d4857..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/Header.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest.headers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * OZONE specific HTTP headers.
- */
-@InterfaceAudience.Private
-public final class Header {
-  public static final String OZONE_QUOTA_BYTES = "BYTES";
-  public static final String OZONE_QUOTA_MB = "MB";
-  public static final String OZONE_QUOTA_GB = "GB";
-  public static final String OZONE_QUOTA_TB = "TB";
-  public static final String OZONE_QUOTA_REMOVE = "remove";
-  public static final String OZONE_QUOTA_UNDEFINED = "undefined";
-  public static final String OZONE_EMPTY_STRING="";
-  public static final String OZONE_DEFAULT_LIST_SIZE = "1000";
-
-  public static final String OZONE_USER = "x-ozone-user";
-  public static final String OZONE_SIMPLE_AUTHENTICATION_SCHEME = "OZONE";
-  public static final String OZONE_VERSION_HEADER = "x-ozone-version";
-  public static final String OZONE_V1_VERSION_HEADER ="v1";
-
-  public static final String OZONE_LIST_QUERY_SERVICE = "service";
-
-  public static final String OZONE_INFO_QUERY_VOLUME = "volume";
-  public static final String OZONE_INFO_QUERY_BUCKET = "bucket";
-  public static final String OZONE_INFO_QUERY_KEY = "key";
-
-  public static final String OZONE_REQUEST_ID = "x-ozone-request-id";
-  public static final String OZONE_SERVER_NAME = "x-ozone-server-name";
-
-  public static final String OZONE_STORAGE_TYPE = "x-ozone-storage-type";
-
-  public static final String OZONE_BUCKET_VERSIONING =
-      "x-ozone-bucket-versioning";
-
-  public static final String OZONE_ACLS = "x-ozone-acls";
-  public static final String OZONE_ACL_ADD = "ADD";
-  public static final String OZONE_ACL_REMOVE = "REMOVE";
-
-  public static final String OZONE_INFO_QUERY_TAG ="info";
-  public static final String OZONE_QUOTA_QUERY_TAG ="quota";
-  public static final String CONTENT_MD5 = "Content-MD5";
-  public static final String OZONE_LIST_QUERY_PREFIX="prefix";
-  public static final String OZONE_LIST_QUERY_MAXKEYS="max-keys";
-  public static final String OZONE_LIST_QUERY_PREVKEY="prev-key";
-  public static final String OZONE_LIST_QUERY_ROOTSCAN="root-scan";
-
-  private Header() {
-    // Never constructed.
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java
deleted file mode 100644
index 340709f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-@InterfaceAudience.Private
-package org.apache.hadoop.ozone.client.rest.headers;
-
-import org.apache.hadoop.classification.InterfaceAudience;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java
deleted file mode 100644
index ebcc104..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest;
-
-/**
- * This package contains Ozone rest client library classes.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/BucketInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/BucketInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/BucketInfo.java
deleted file mode 100644
index 3c65df5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/BucketInfo.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.client.rest.response;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.OzoneConsts;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * BucketInfo class is used used for parsing json response
- * when BucketInfo Call is made.
- */
-public class BucketInfo implements Comparable<BucketInfo> {
-
-  private static final ObjectReader READER =
-      new ObjectMapper().readerFor(BucketInfo.class);
-
-  private String volumeName;
-  private String bucketName;
-  private String createdOn;
-  private List<OzoneAcl> acls;
-  private OzoneConsts.Versioning versioning;
-  private StorageType storageType;
-
-  /**
-   * Constructor for BucketInfo.
-   *
-   * @param volumeName
-   * @param bucketName
-   */
-  public BucketInfo(String volumeName, String bucketName) {
-    this.volumeName = volumeName;
-    this.bucketName = bucketName;
-  }
-
-
-  /**
-   * Default constructor for BucketInfo.
-   */
-  public BucketInfo() {
-    acls = new LinkedList<>();
-  }
-
-  /**
-   * Parse a JSON string into BucketInfo Object.
-   *
-   * @param jsonString Json String
-   * @return BucketInfo
-   * @throws IOException
-   */
-  public static BucketInfo parse(String jsonString) throws IOException {
-    return READER.readValue(jsonString);
-  }
-
-  /**
-   * Returns a List of ACLs set on the Bucket.
-   *
-   * @return List of Acl
-   */
-  public List<OzoneAcl> getAcls() {
-    return acls;
-  }
-
-  /**
-   * Sets ACls.
-   *
-   * @param acls Acl list
-   */
-  public void setAcls(List<OzoneAcl> acls) {
-    this.acls = acls;
-  }
-
-  /**
-   * Returns Storage Type info.
-   *
-   * @return Storage Type of the bucket
-   */
-  public StorageType getStorageType() {
-    return storageType;
-  }
-
-  /**
-   * Sets the Storage Type.
-   *
-   * @param storageType Storage Type
-   */
-  public void setStorageType(StorageType storageType) {
-    this.storageType = storageType;
-  }
-
-  /**
-   * Returns versioning.
-   *
-   * @return versioning Enum
-   */
-  public OzoneConsts.Versioning getVersioning() {
-    return versioning;
-  }
-
-  /**
-   * Sets Versioning.
-   *
-   * @param versioning
-   */
-  public void setVersioning(OzoneConsts.Versioning versioning) {
-    this.versioning = versioning;
-  }
-
-
-  /**
-   * Gets bucket Name.
-   *
-   * @return String
-   */
-  public String getBucketName() {
-    return bucketName;
-  }
-
-  /**
-   * Sets bucket Name.
-   *
-   * @param bucketName Name of the bucket
-   */
-  public void setBucketName(String bucketName) {
-    this.bucketName = bucketName;
-  }
-
-  /**
-   * Sets creation time of the bucket.
-   *
-   * @param creationTime Date String
-   */
-  public void setCreatedOn(String creationTime) {
-    this.createdOn = creationTime;
-  }
-
-  /**
-   * Returns creation time.
-   *
-   * @return creation time of bucket.
-   */
-  public String getCreatedOn() {
-    return createdOn;
-  }
-
-  /**
-   * Returns Volume Name.
-   *
-   * @return String volume name
-   */
-  public String getVolumeName() {
-    return volumeName;
-  }
-
-  /**
-   * Sets the Volume Name of bucket.
-   *
-   * @param volumeName volumeName
-   */
-  public void setVolumeName(String volumeName) {
-    this.volumeName = volumeName;
-  }
-
-  /**
-   * Compares this object with the specified object for order.  Returns a
-   * negative integer, zero, or a positive integer as this object is less
-   * than, equal to, or greater than the specified object.
-   *
-   * Please note : BucketInfo compare functions are used only within the
-   * context of a volume, hence volume name is purposefully ignored in
-   * compareTo, equal and hashcode functions of this class.
-   */
-  @Override
-  public int compareTo(BucketInfo o) {
-    Preconditions.checkState(o.getVolumeName().equals(this.getVolumeName()));
-    return this.bucketName.compareTo(o.getBucketName());
-  }
-
-  /**
-   * Checks if two bucketInfo's are equal.
-   * @param o Object BucketInfo
-   * @return  True or False
-   */
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (!(o instanceof BucketInfo)) {
-      return false;
-    }
-
-    BucketInfo that = (BucketInfo) o;
-    Preconditions.checkState(that.getVolumeName().equals(this.getVolumeName()));
-    return bucketName.equals(that.bucketName);
-
-  }
-
-  /**
-   * Hash Code for this object.
-   * @return int
-   */
-  @Override
-  public int hashCode() {
-    return bucketName.hashCode();
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
deleted file mode 100644
index 5699fda..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest.response;
-
-
-import java.io.IOException;
-
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-
-/**
- * KeyInfo class is used used for parsing json response
- * when KeyInfo Call is made.
- */
-public class KeyInfo implements Comparable<KeyInfo> {
-
-  private static final ObjectReader READER =
-      new ObjectMapper().readerFor(KeyInfo.class);
-
-  private long version;
-  private String md5hash;
-  private String createdOn;
-  private String modifiedOn;
-  private long size;
-  private String keyName;
-
-  /**
-   * When this key was created.
-   *
-   * @return Date String
-   */
-  public String getCreatedOn() {
-    return createdOn;
-  }
-
-  /**
-   * When this key was modified.
-   *
-   * @return Date String
-   */
-  public String getModifiedOn() {
-    return modifiedOn;
-  }
-
-  /**
-   * When this key was created.
-   *
-   * @param createdOn Date String
-   */
-  public void setCreatedOn(String createdOn) {
-    this.createdOn = createdOn;
-  }
-
-  /**
-   * When this key was modified.
-   *
-   * @param modifiedOn Date String
-   */
-  public void setModifiedOn(String modifiedOn) {
-    this.modifiedOn = modifiedOn;
-  }
-
-  /**
-   * Gets the Key name of this object.
-   *
-   * @return String
-   */
-  public String getKeyName() {
-    return keyName;
-  }
-
-  /**
-   * Sets the Key name of this object.
-   *
-   * @param keyName String
-   */
-  public void setKeyName(String keyName) {
-    this.keyName = keyName;
-  }
-
-  /**
-   * Returns the MD5 Hash for the data of this key.
-   *
-   * @return String MD5
-   */
-  public String getMd5hash() {
-    return md5hash;
-  }
-
-  /**
-   * Sets the MD5 value of this key.
-   *
-   * @param md5hash Md5 of this file
-   */
-  public void setMd5hash(String md5hash) {
-    this.md5hash = md5hash;
-  }
-
-  /**
-   * Number of bytes stored in the data part of this key.
-   *
-   * @return long size of the data file
-   */
-  public long getSize() {
-    return size;
-  }
-
-  /**
-   * Sets the size of the data part of this key.
-   *
-   * @param size Size in long
-   */
-  public void setSize(long size) {
-    this.size = size;
-  }
-
-  /**
-   * Version of this key.
-   *
-   * @return returns the version of this key.
-   */
-  public long getVersion() {
-    return version;
-  }
-
-  /**
-   * Sets the version of this key.
-   *
-   * @param version - Version String
-   */
-  public void setVersion(long version) {
-    this.version = version;
-  }
-
-  /**
-   * Compares this object with the specified object for order.  Returns a
-   * negative integer, zero, or a positive integer as this object is less
-   * than, equal to, or greater than the specified object.
-   *
-   * @param o the object to be compared.
-   * @return a negative integer, zero, or a positive integer as this object
-   * is less than, equal to, or greater than the specified object.
-   * @throws NullPointerException if the specified object is null
-   * @throws ClassCastException   if the specified object's type prevents it
-   *                              from being compared to this object.
-   */
-  @Override
-  public int compareTo(KeyInfo o) {
-    if (this.keyName.compareTo(o.getKeyName()) != 0) {
-      return this.keyName.compareTo(o.getKeyName());
-    }
-
-    if (this.getVersion() == o.getVersion()) {
-      return 0;
-    }
-    if (this.getVersion() < o.getVersion()) {
-      return -1;
-    }
-    return 1;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-
-    KeyInfo keyInfo = (KeyInfo) o;
-
-    return new EqualsBuilder()
-        .append(version, keyInfo.version)
-        .append(keyName, keyInfo.keyName)
-        .isEquals();
-  }
-
-  @Override
-  public int hashCode() {
-    return new HashCodeBuilder(17, 37)
-        .append(version)
-        .append(keyName)
-        .toHashCode();
-  }
-
-  /**
-   * Parse a string to return KeyInfo Object.
-   *
-   * @param jsonString Json String
-   * @return keyInfo
-   * @throws IOException
-   */
-  public static KeyInfo parse(String jsonString) throws IOException {
-    return READER.readValue(jsonString);
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeInfo.java
deleted file mode 100644
index 8db3ac2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeInfo.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest.response;
-
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.client.OzoneQuota;
-
-import java.io.IOException;
-
-/**
- * VolumeInfo Class is used for parsing json response
- * when VolumeInfo Call is made.
- */
-@InterfaceAudience.Private
-public class VolumeInfo implements Comparable<VolumeInfo> {
-
-
-  private static final ObjectReader READER =
-      new ObjectMapper().readerFor(VolumeInfo.class);
-
-  private VolumeOwner owner;
-  private OzoneQuota quota;
-  private String volumeName;
-  private String createdOn;
-  private String createdBy;
-
-
-  /**
-   * Constructor for VolumeInfo.
-   *
-   * @param volumeName - Name of the Volume
-   * @param createdOn _ Date String
-   * @param createdBy - Person who created it
-   */
-  public VolumeInfo(String volumeName, String createdOn,
-                    String createdBy) {
-    this.volumeName = volumeName;
-    this.createdOn = createdOn;
-    this.createdBy = createdBy;
-  }
-
-  /**
-   * Constructor for VolumeInfo.
-   */
-  public VolumeInfo() {
-  }
-
-  /**
-   * gets the volume name.
-   *
-   * @return Volume Name
-   */
-  public String getVolumeName() {
-    return volumeName;
-  }
-
-  /**
-   * Sets the volume name.
-   *
-   * @param volumeName Volume Name
-   */
-  public void setVolumeName(String volumeName) {
-    this.volumeName = volumeName;
-  }
-
-
-  /**
-   * Returns the name of the person who created this volume.
-   *
-   * @return Name of Admin who created this
-   */
-  public String getCreatedBy() {
-    return createdBy;
-  }
-
-  /**
-   * Sets the user name of the person who created this volume.
-   *
-   * @param createdBy UserName
-   */
-  public void setCreatedBy(String createdBy) {
-    this.createdBy = createdBy;
-  }
-
-  /**
-   * Gets the date on which this volume was created.
-   *
-   * @return Date String
-   */
-  public String getCreatedOn() {
-    return createdOn;
-  }
-
-  /**
-   * Sets the date string.
-   *
-   * @param createdOn Date String
-   */
-  public void setCreatedOn(String createdOn) {
-    this.createdOn = createdOn;
-  }
-
-  /**
-   * Returns the owner info.
-   *
-   * @return OwnerInfo
-   */
-  public VolumeOwner getOwner() {
-    return owner;
-  }
-
-  /**
-   * Sets the owner.
-   *
-   * @param owner OwnerInfo
-   */
-  public void setOwner(VolumeOwner owner) {
-    this.owner = owner;
-  }
-
-  /**
-   * Returns the quota information on a volume.
-   *
-   * @return Quota
-   */
-  public OzoneQuota getQuota() {
-    return quota;
-  }
-
-  /**
-   * Sets the quota info.
-   *
-   * @param quota Quota Info
-   */
-  public void setQuota(OzoneQuota quota) {
-    this.quota = quota;
-  }
-
-  /**
-   * Comparable Interface.
-   * @param o VolumeInfo Object.
-   * @return Result of comparison
-   */
-  @Override
-  public int compareTo(VolumeInfo o) {
-    return this.volumeName.compareTo(o.getVolumeName());
-  }
-
-  /**
-   * Returns VolumeInfo class from json string.
-   *
-   * @param data Json String
-   *
-   * @return VolumeInfo
-   *
-   * @throws IOException
-   */
-  public static VolumeInfo parse(String data) throws IOException {
-    return READER.readValue(data);
-  }
-
-  /**
-   * Indicates whether some other object is "equal to" this one.
-   *
-   * @param obj the reference object with which to compare.
-   *
-   * @return {@code true} if this object is the same as the obj
-   * argument; {@code false} otherwise.
-   */
-  @Override
-  public boolean equals(Object obj) {
-    if (obj == null) {
-      return false;
-    }
-    if (getClass() != obj.getClass()) {
-      return false;
-    }
-    VolumeInfo otherInfo = (VolumeInfo) obj;
-    return otherInfo.getVolumeName().equals(this.getVolumeName());
-  }
-
-  /**
-   * Returns a hash code value for the object. This method is
-   * supported for the benefit of hash tables such as those provided by
-   * HashMap.
-   * @return a hash code value for this object.
-   *
-   * @see Object#equals(Object)
-   * @see System#identityHashCode
-   */
-  @Override
-  public int hashCode() {
-    return getVolumeName().hashCode();
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeOwner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeOwner.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeOwner.java
deleted file mode 100644
index 132bc2e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeOwner.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest.response;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * Volume Owner represents the owner of a volume.
- *
- * This is a class instead of a string since we might need to extend this class
- * to support other forms of authentication.
- */
-@InterfaceAudience.Private
-public class VolumeOwner {
-  @JsonInclude(JsonInclude.Include.NON_NULL)
-  private String name;
-
-  /**
-   * Constructor for VolumeOwner.
-   *
-   * @param name name of the User
-   */
-  public VolumeOwner(String name) {
-    this.name = name;
-  }
-
-  /**
-   * Constructs Volume Owner.
-   */
-  public VolumeOwner() {
-    name = null;
-  }
-
-  /**
-   * Returns the user name.
-   *
-   * @return Name
-   */
-  public String getName() {
-    return name;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/package-info.java
deleted file mode 100644
index 432b029b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest.response;
-
-/**
- * This package contains class for ozone rest client library.
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
deleted file mode 100644
index 7b48f49..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ /dev/null
@@ -1,567 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rpc;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.BucketArgs;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneKey;
-import org.apache.hadoop.ozone.client.OzoneQuota;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.ReplicationFactor;
-import org.apache.hadoop.ozone.client.ReplicationType;
-import org.apache.hadoop.ozone.client.VolumeArgs;
-import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream;
-import org.apache.hadoop.ozone.client.io.ChunkGroupOutputStream;
-import org.apache.hadoop.ozone.client.io.LengthInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
-import org.apache.hadoop.ozone.ksm.protocolPB
-    .KeySpaceManagerProtocolClientSideTranslatorPB;
-import org.apache.hadoop.ozone.ksm.protocolPB
-    .KeySpaceManagerProtocolPB;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ServicePort;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.protocolPB
-    .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.protocolPB
-    .StorageContainerLocationProtocolPB;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.UUID;
-import java.util.stream.Collectors;
-
-/**
- * Ozone RPC Client Implementation, it connects to KSM, SCM and DataNode
- * to execute client calls. This uses RPC protocol for communication
- * with the servers.
- */
-public class RpcClient implements ClientProtocol {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(RpcClient.class);
-
-  private final OzoneConfiguration conf;
-  private final StorageContainerLocationProtocolClientSideTranslatorPB
-      storageContainerLocationClient;
-  private final KeySpaceManagerProtocolClientSideTranslatorPB
-      keySpaceManagerClient;
-  private final XceiverClientManager xceiverClientManager;
-  private final int chunkSize;
-  private final UserGroupInformation ugi;
-  private final OzoneAcl.OzoneACLRights userRights;
-  private final OzoneAcl.OzoneACLRights groupRights;
-
-   /**
-    * Creates RpcClient instance with the given configuration.
-    * @param conf
-    * @throws IOException
-    */
-  public RpcClient(Configuration conf) throws IOException {
-    Preconditions.checkNotNull(conf);
-    this.conf = new OzoneConfiguration(conf);
-    this.ugi = UserGroupInformation.getCurrentUser();
-    this.userRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_USER_RIGHTS,
-        KSMConfigKeys.OZONE_KSM_USER_RIGHTS_DEFAULT);
-    this.groupRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_GROUP_RIGHTS,
-        KSMConfigKeys.OZONE_KSM_GROUP_RIGHTS_DEFAULT);
-    long ksmVersion =
-        RPC.getProtocolVersion(KeySpaceManagerProtocolPB.class);
-    InetSocketAddress ksmAddress = OzoneClientUtils
-        .getKsmAddressForClients(conf);
-    RPC.setProtocolEngine(conf, KeySpaceManagerProtocolPB.class,
-        ProtobufRpcEngine.class);
-    this.keySpaceManagerClient =
-        new KeySpaceManagerProtocolClientSideTranslatorPB(
-            RPC.getProxy(KeySpaceManagerProtocolPB.class, ksmVersion,
-                ksmAddress, UserGroupInformation.getCurrentUser(), conf,
-                NetUtils.getDefaultSocketFactory(conf),
-                Client.getRpcTimeout(conf)));
-
-    long scmVersion =
-        RPC.getProtocolVersion(StorageContainerLocationProtocolPB.class);
-    InetSocketAddress scmAddress = getScmAddressForClient();
-    RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class,
-        ProtobufRpcEngine.class);
-    this.storageContainerLocationClient =
-        new StorageContainerLocationProtocolClientSideTranslatorPB(
-            RPC.getProxy(StorageContainerLocationProtocolPB.class, scmVersion,
-                scmAddress, UserGroupInformation.getCurrentUser(), conf,
-                NetUtils.getDefaultSocketFactory(conf),
-                Client.getRpcTimeout(conf)));
-
-    this.xceiverClientManager = new XceiverClientManager(conf);
-
-    int configuredChunkSize = conf.getInt(
-        ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
-        ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
-    if(configuredChunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
-      LOG.warn("The chunk size ({}) is not allowed to be more than"
-              + " the maximum size ({}),"
-              + " resetting to the maximum size.",
-          configuredChunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE);
-      chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
-    } else {
-      chunkSize = configuredChunkSize;
-    }
-  }
-
-  private InetSocketAddress getScmAddressForClient() throws IOException {
-    List<ServiceInfo> services = keySpaceManagerClient.getServiceList();
-    ServiceInfo scmInfo = services.stream().filter(
-        a -> a.getNodeType().equals(OzoneProtos.NodeType.SCM))
-        .collect(Collectors.toList()).get(0);
-    return NetUtils.createSocketAddr(scmInfo.getHostname()+ ":" +
-        scmInfo.getPort(ServicePort.Type.RPC));
-  }
-
-  @Override
-  public void createVolume(String volumeName) throws IOException {
-    createVolume(volumeName, VolumeArgs.newBuilder().build());
-  }
-
-  @Override
-  public void createVolume(String volumeName, VolumeArgs volArgs)
-      throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(volArgs);
-
-    String admin = volArgs.getAdmin() == null ?
-        ugi.getUserName() : volArgs.getAdmin();
-    String owner = volArgs.getOwner() == null ?
-        ugi.getUserName() : volArgs.getOwner();
-    long quota = volArgs.getQuota() == null ?
-        OzoneConsts.MAX_QUOTA_IN_BYTES :
-        OzoneQuota.parseQuota(volArgs.getQuota()).sizeInBytes();
-    List<OzoneAcl> listOfAcls = new ArrayList<>();
-    //User ACL
-    listOfAcls.add(new OzoneAcl(OzoneAcl.OzoneACLType.USER,
-            owner, userRights));
-    //Group ACLs of the User
-    List<String> userGroups = Arrays.asList(UserGroupInformation
-        .createRemoteUser(owner).getGroupNames());
-    userGroups.stream().forEach((group) -> listOfAcls.add(
-        new OzoneAcl(OzoneAcl.OzoneACLType.GROUP, group, groupRights)));
-    //ACLs from VolumeArgs
-    if(volArgs.getAcls() != null) {
-      listOfAcls.addAll(volArgs.getAcls());
-    }
-
-    KsmVolumeArgs.Builder builder = KsmVolumeArgs.newBuilder();
-    builder.setVolume(volumeName);
-    builder.setAdminName(admin);
-    builder.setOwnerName(owner);
-    builder.setQuotaInBytes(quota);
-
-    //Remove duplicates and add ACLs
-    for (OzoneAcl ozoneAcl :
-        listOfAcls.stream().distinct().collect(Collectors.toList())) {
-      builder.addOzoneAcls(KSMPBHelper.convertOzoneAcl(ozoneAcl));
-    }
-
-    LOG.info("Creating Volume: {}, with {} as owner and quota set to {} bytes.",
-        volumeName, owner, quota);
-    keySpaceManagerClient.createVolume(builder.build());
-  }
-
-  @Override
-  public void setVolumeOwner(String volumeName, String owner)
-      throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(owner);
-    keySpaceManagerClient.setOwner(volumeName, owner);
-  }
-
-  @Override
-  public void setVolumeQuota(String volumeName, OzoneQuota quota)
-      throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(quota);
-    long quotaInBytes = quota.sizeInBytes();
-    keySpaceManagerClient.setQuota(volumeName, quotaInBytes);
-  }
-
-  @Override
-  public OzoneVolume getVolumeDetails(String volumeName)
-      throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    KsmVolumeArgs volume = keySpaceManagerClient.getVolumeInfo(volumeName);
-    return new OzoneVolume(
-        conf,
-        this,
-        volume.getVolume(),
-        volume.getAdminName(),
-        volume.getOwnerName(),
-        volume.getQuotaInBytes(),
-        volume.getCreationTime(),
-        volume.getAclMap().ozoneAclGetProtobuf().stream().
-            map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
-  }
-
-  @Override
-  public boolean checkVolumeAccess(String volumeName, OzoneAcl acl)
-      throws IOException {
-    throw new UnsupportedOperationException("Not yet implemented.");
-  }
-
-  @Override
-  public void deleteVolume(String volumeName) throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    keySpaceManagerClient.deleteVolume(volumeName);
-  }
-
-  @Override
-  public List<OzoneVolume> listVolumes(String volumePrefix, String prevVolume,
-                                       int maxListResult)
-      throws IOException {
-    List<KsmVolumeArgs> volumes = keySpaceManagerClient.listAllVolumes(
-        volumePrefix, prevVolume, maxListResult);
-
-    return volumes.stream().map(volume -> new OzoneVolume(
-        conf,
-        this,
-        volume.getVolume(),
-        volume.getAdminName(),
-        volume.getOwnerName(),
-        volume.getQuotaInBytes(),
-        volume.getCreationTime(),
-        volume.getAclMap().ozoneAclGetProtobuf().stream().
-            map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
-        .collect(Collectors.toList());
-  }
-
-  @Override
-  public List<OzoneVolume> listVolumes(String user, String volumePrefix,
-                                       String prevVolume, int maxListResult)
-      throws IOException {
-    List<KsmVolumeArgs> volumes = keySpaceManagerClient.listVolumeByUser(
-        user, volumePrefix, prevVolume, maxListResult);
-
-    return volumes.stream().map(volume -> new OzoneVolume(
-        conf,
-        this,
-        volume.getVolume(),
-        volume.getAdminName(),
-        volume.getOwnerName(),
-        volume.getQuotaInBytes(),
-        volume.getCreationTime(),
-        volume.getAclMap().ozoneAclGetProtobuf().stream().
-            map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
-        .collect(Collectors.toList());
-  }
-
-  @Override
-  public void createBucket(String volumeName, String bucketName)
-      throws IOException {
-    createBucket(volumeName, bucketName, BucketArgs.newBuilder().build());
-  }
-
-  @Override
-  public void createBucket(
-      String volumeName, String bucketName, BucketArgs bucketArgs)
-      throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(bucketName);
-    Preconditions.checkNotNull(bucketArgs);
-
-    Boolean isVersionEnabled = bucketArgs.getVersioning() == null ?
-        Boolean.FALSE : bucketArgs.getVersioning();
-    StorageType storageType = bucketArgs.getStorageType() == null ?
-        StorageType.DEFAULT : bucketArgs.getStorageType();
-    List<OzoneAcl> listOfAcls = new ArrayList<>();
-    //User ACL
-    listOfAcls.add(new OzoneAcl(OzoneAcl.OzoneACLType.USER,
-        ugi.getUserName(), userRights));
-    //Group ACLs of the User
-    List<String> userGroups = Arrays.asList(UserGroupInformation
-        .createRemoteUser(ugi.getUserName()).getGroupNames());
-    userGroups.stream().forEach((group) -> listOfAcls.add(
-        new OzoneAcl(OzoneAcl.OzoneACLType.GROUP, group, groupRights)));
-    //ACLs from BucketArgs
-    if(bucketArgs.getAcls() != null) {
-      listOfAcls.addAll(bucketArgs.getAcls());
-    }
-
-    KsmBucketInfo.Builder builder = KsmBucketInfo.newBuilder();
-    builder.setVolumeName(volumeName)
-        .setBucketName(bucketName)
-        .setIsVersionEnabled(isVersionEnabled)
-        .setStorageType(storageType)
-        .setAcls(listOfAcls.stream().distinct().collect(Collectors.toList()));
-
-    LOG.info("Creating Bucket: {}/{}, with Versioning {} and " +
-            "Storage Type set to {}", volumeName, bucketName, isVersionEnabled,
-            storageType);
-    keySpaceManagerClient.createBucket(builder.build());
-  }
-
-  @Override
-  public void addBucketAcls(
-      String volumeName, String bucketName, List<OzoneAcl> addAcls)
-      throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(bucketName);
-    Preconditions.checkNotNull(addAcls);
-    KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
-    builder.setVolumeName(volumeName)
-        .setBucketName(bucketName)
-        .setAddAcls(addAcls);
-    keySpaceManagerClient.setBucketProperty(builder.build());
-  }
-
-  @Override
-  public void removeBucketAcls(
-      String volumeName, String bucketName, List<OzoneAcl> removeAcls)
-      throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(bucketName);
-    Preconditions.checkNotNull(removeAcls);
-    KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
-    builder.setVolumeName(volumeName)
-        .setBucketName(bucketName)
-        .setRemoveAcls(removeAcls);
-    keySpaceManagerClient.setBucketProperty(builder.build());
-  }
-
-  @Override
-  public void setBucketVersioning(
-      String volumeName, String bucketName, Boolean versioning)
-      throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(bucketName);
-    Preconditions.checkNotNull(versioning);
-    KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
-    builder.setVolumeName(volumeName)
-        .setBucketName(bucketName)
-        .setIsVersionEnabled(versioning);
-    keySpaceManagerClient.setBucketProperty(builder.build());
-  }
-
-  @Override
-  public void setBucketStorageType(
-      String volumeName, String bucketName, StorageType storageType)
-      throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(bucketName);
-    Preconditions.checkNotNull(storageType);
-    KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
-    builder.setVolumeName(volumeName)
-        .setBucketName(bucketName)
-        .setStorageType(storageType);
-    keySpaceManagerClient.setBucketProperty(builder.build());
-  }
-
-  @Override
-  public void deleteBucket(
-      String volumeName, String bucketName) throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(bucketName);
-    keySpaceManagerClient.deleteBucket(volumeName, bucketName);
-  }
-
-  @Override
-  public void checkBucketAccess(
-      String volumeName, String bucketName) throws IOException {
-
-  }
-
-  @Override
-  public OzoneBucket getBucketDetails(
-      String volumeName, String bucketName) throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(bucketName);
-    KsmBucketInfo bucketArgs =
-        keySpaceManagerClient.getBucketInfo(volumeName, bucketName);
-    return new OzoneBucket(
-        conf,
-        this,
-        bucketArgs.getVolumeName(),
-        bucketArgs.getBucketName(),
-        bucketArgs.getAcls(),
-        bucketArgs.getStorageType(),
-        bucketArgs.getIsVersionEnabled(),
-        bucketArgs.getCreationTime());
-  }
-
-  @Override
-  public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
-                                       String prevBucket, int maxListResult)
-      throws IOException {
-    List<KsmBucketInfo> buckets = keySpaceManagerClient.listBuckets(
-        volumeName, prevBucket, bucketPrefix, maxListResult);
-
-    return buckets.stream().map(bucket -> new OzoneBucket(
-        conf,
-        this,
-        bucket.getVolumeName(),
-        bucket.getBucketName(),
-        bucket.getAcls(),
-        bucket.getStorageType(),
-        bucket.getIsVersionEnabled(),
-        bucket.getCreationTime()))
-        .collect(Collectors.toList());
-  }
-
-  @Override
-  public OzoneOutputStream createKey(
-      String volumeName, String bucketName, String keyName, long size,
-      ReplicationType type, ReplicationFactor factor)
-      throws IOException {
-    String requestId = UUID.randomUUID().toString();
-    KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
-        .setVolumeName(volumeName)
-        .setBucketName(bucketName)
-        .setKeyName(keyName)
-        .setDataSize(size)
-        .setType(OzoneProtos.ReplicationType.valueOf(type.toString()))
-        .setFactor(OzoneProtos.ReplicationFactor.valueOf(factor.getValue()))
-        .build();
-
-    OpenKeySession openKey = keySpaceManagerClient.openKey(keyArgs);
-    ChunkGroupOutputStream groupOutputStream =
-        new ChunkGroupOutputStream.Builder()
-            .setHandler(openKey)
-            .setXceiverClientManager(xceiverClientManager)
-            .setScmClient(storageContainerLocationClient)
-            .setKsmClient(keySpaceManagerClient)
-            .setChunkSize(chunkSize)
-            .setRequestID(requestId)
-            .setType(OzoneProtos.ReplicationType.valueOf(type.toString()))
-            .setFactor(OzoneProtos.ReplicationFactor.valueOf(factor.getValue()))
-            .build();
-    groupOutputStream.addPreallocateBlocks(
-        openKey.getKeyInfo().getLatestVersionLocations(),
-        openKey.getOpenVersion());
-    return new OzoneOutputStream(groupOutputStream);
-  }
-
-  @Override
-  public OzoneInputStream getKey(
-      String volumeName, String bucketName, String keyName)
-      throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(bucketName);
-    Preconditions.checkNotNull(keyName);
-    String requestId = UUID.randomUUID().toString();
-    KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
-        .setVolumeName(volumeName)
-        .setBucketName(bucketName)
-        .setKeyName(keyName)
-        .build();
-    KsmKeyInfo keyInfo = keySpaceManagerClient.lookupKey(keyArgs);
-    LengthInputStream lengthInputStream =
-        ChunkGroupInputStream.getFromKsmKeyInfo(
-            keyInfo, xceiverClientManager, storageContainerLocationClient,
-            requestId);
-    return new OzoneInputStream(
-        (ChunkGroupInputStream)lengthInputStream.getWrappedStream());
-  }
-
-  @Override
-  public void deleteKey(
-      String volumeName, String bucketName, String keyName)
-      throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(bucketName);
-    Preconditions.checkNotNull(keyName);
-    KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
-        .setVolumeName(volumeName)
-        .setBucketName(bucketName)
-        .setKeyName(keyName)
-        .build();
-    keySpaceManagerClient.deleteKey(keyArgs);
-  }
-
-  @Override
-  public List<OzoneKey> listKeys(String volumeName, String bucketName,
-                                 String keyPrefix, String prevKey,
-                                 int maxListResult)
-      throws IOException {
-    List<KsmKeyInfo> keys = keySpaceManagerClient.listKeys(
-        volumeName, bucketName, prevKey, keyPrefix, maxListResult);
-
-    return keys.stream().map(key -> new OzoneKey(
-        key.getVolumeName(),
-        key.getBucketName(),
-        key.getKeyName(),
-        key.getDataSize(),
-        key.getCreationTime(),
-        key.getModificationTime()))
-        .collect(Collectors.toList());
-  }
-
-  @Override
-  public OzoneKey getKeyDetails(
-      String volumeName, String bucketName, String keyName)
-      throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(bucketName);
-    Preconditions.checkNotNull(keyName);
-    KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
-        .setVolumeName(volumeName)
-        .setBucketName(bucketName)
-        .setKeyName(keyName)
-        .build();
-    KsmKeyInfo keyInfo = keySpaceManagerClient.lookupKey(keyArgs);
-    return new OzoneKey(keyInfo.getVolumeName(),
-                        keyInfo.getBucketName(),
-                        keyInfo.getKeyName(),
-                        keyInfo.getDataSize(),
-                        keyInfo.getCreationTime(),
-                        keyInfo.getModificationTime());
-  }
-
-  @Override
-  public void close() throws IOException {
-    IOUtils.cleanupWithLogger(LOG, storageContainerLocationClient);
-    IOUtils.cleanupWithLogger(LOG, keySpaceManagerClient);
-    IOUtils.cleanupWithLogger(LOG, xceiverClientManager);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/package-info.java
deleted file mode 100644
index 0fcc3fc..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rpc;
-
-/**
- * This package contains Ozone rpc client library classes.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java
deleted file mode 100644
index 5571f0a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.common;
-
-import org.apache.hadoop.ozone.protocol.proto
-    .ScmBlockLocationProtocolProtos.KeyBlocks;
-
-import java.util.List;
-
-/**
- * A group of blocks relations relevant, e.g belong to a certain object key.
- */
-public final class BlockGroup {
-
-  private String groupID;
-  private List<String> blockIDs;
-  private BlockGroup(String groupID, List<String> blockIDs) {
-    this.groupID = groupID;
-    this.blockIDs = blockIDs;
-  }
-
-  public List<String> getBlockIDList() {
-    return blockIDs;
-  }
-
-  public String getGroupID() {
-    return groupID;
-  }
-
-  public KeyBlocks getProto() {
-    return KeyBlocks.newBuilder().setKey(groupID)
-        .addAllBlocks(blockIDs).build();
-  }
-
-  /**
-   * Parses a KeyBlocks proto to a group of blocks.
-   * @param proto KeyBlocks proto.
-   * @return a group of blocks.
-   */
-  public static BlockGroup getFromProto(KeyBlocks proto) {
-    return BlockGroup.newBuilder().setKeyName(proto.getKey())
-        .addAllBlockIDs(proto.getBlocksList()).build();
-  }
-
-  public static Builder newBuilder() {
-    return new Builder();
-  }
-
-  /**
-   * BlockGroup instance builder.
-   */
-  public static class Builder {
-
-    private String groupID;
-    private List<String> blockIDs;
-
-    public Builder setKeyName(String blockGroupID) {
-      this.groupID = blockGroupID;
-      return this;
-    }
-
-    public Builder addAllBlockIDs(List<String> keyBlocks) {
-      this.blockIDs = keyBlocks;
-      return this;
-    }
-
-    public BlockGroup build() {
-      return new BlockGroup(groupID, blockIDs);
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/DeleteBlockGroupResult.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/DeleteBlockGroupResult.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/DeleteBlockGroupResult.java
deleted file mode 100644
index f56f832..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/DeleteBlockGroupResult.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.common;
-
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.DeleteScmBlockResult;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.DeleteScmBlockResult.Result;
-import org.apache.hadoop.scm.container.common.helpers.DeleteBlockResult;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Result to delete a group of blocks.
- */
-public class DeleteBlockGroupResult {
-  private String objectKey;
-  private List<DeleteBlockResult> blockResultList;
-  public DeleteBlockGroupResult(String objectKey,
-      List<DeleteBlockResult> blockResultList) {
-    this.objectKey = objectKey;
-    this.blockResultList = blockResultList;
-  }
-
-  public String getObjectKey() {
-    return objectKey;
-  }
-
-  public List<DeleteBlockResult> getBlockResultList() {
-    return blockResultList;
-  }
-
-  public List<DeleteScmBlockResult> getBlockResultProtoList() {
-    List<DeleteScmBlockResult> resultProtoList =
-        new ArrayList<>(blockResultList.size());
-    for (DeleteBlockResult result : blockResultList) {
-      DeleteScmBlockResult proto = DeleteScmBlockResult.newBuilder()
-          .setKey(result.getKey())
-          .setResult(result.getResult()).build();
-      resultProtoList.add(proto);
-    }
-    return resultProtoList;
-  }
-
-  public static List<DeleteBlockResult> convertBlockResultProto(
-      List<DeleteScmBlockResult> results) {
-    List<DeleteBlockResult> protoResults = new ArrayList<>(results.size());
-    for (DeleteScmBlockResult result : results) {
-      protoResults.add(new DeleteBlockResult(result.getKey(),
-          result.getResult()));
-    }
-    return protoResults;
-  }
-
-  /**
-   * Only if all blocks are successfully deleted, this group is considered
-   * to be successfully executed.
-   *
-   * @return true if all blocks are successfully deleted, false otherwise.
-   */
-  public boolean isSuccess() {
-    for (DeleteBlockResult result : blockResultList) {
-      if (result.getResult() != Result.success) {
-        return false;
-      }
-    }
-    return true;
-  }
-
-  /**
-   * @return A list of deletion failed block IDs.
-   */
-  public List<String> getFailedBlocks() {
-    List<String> failedBlocks = blockResultList.stream()
-        .filter(result -> result.getResult() != Result.success)
-        .map(DeleteBlockResult::getKey).collect(Collectors.toList());
-    return failedBlocks;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/package-info.java
deleted file mode 100644
index 1e9fd59..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.common;
-/**
-  ozone common shared by SCM, KSM, etc.
- **/
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/InvalidStateTransitionException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/InvalidStateTransitionException.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/InvalidStateTransitionException.java
deleted file mode 100644
index 9aeff24..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/InvalidStateTransitionException.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.common.statemachine;
-
-/**
- * Class wraps invalid state transition exception.
- */
-public class InvalidStateTransitionException extends Exception {
-  private Enum<?> currentState;
-  private Enum<?> event;
-
-  public InvalidStateTransitionException(Enum<?> currentState, Enum<?> event) {
-    super("Invalid event: " + event + " at " + currentState + " state.");
-    this.currentState = currentState;
-    this.event = event;
-  }
-
-  public Enum<?> getCurrentState() {
-    return currentState;
-  }
-
-  public Enum<?> getEvent() {
-    return event;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/StateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/StateMachine.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/StateMachine.java
deleted file mode 100644
index bf8cbd5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/StateMachine.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.common.statemachine;
-
-import com.google.common.base.Supplier;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Template class that wraps simple event driven state machine.
- * @param <STATE> states allowed
- * @param <EVENT> events allowed
- */
-public class StateMachine<STATE extends Enum<?>, EVENT extends Enum<?>> {
-  private STATE initialState;
-  private Set<STATE> finalStates;
-
-  private final LoadingCache<EVENT, Map<STATE, STATE>> transitions =
-      CacheBuilder.newBuilder().build(
-          CacheLoader.from((Supplier<Map<STATE, STATE>>) () -> new HashMap()));
-
-  public StateMachine(STATE initState, Set<STATE> finalStates) {
-    this.initialState = initState;
-    this.finalStates = finalStates;
-  }
-
-  public STATE getInitialState() {
-    return initialState;
-  }
-
-  public Set<STATE> getFinalStates() {
-    return finalStates;
-  }
-
-  public STATE getNextState(STATE from, EVENT e)
-      throws InvalidStateTransitionException {
-    STATE target = transitions.getUnchecked(e).get(from);
-    if (target == null) {
-      throw new InvalidStateTransitionException(from, e);
-    }
-    return target;
-  }
-
-  public void addTransition(STATE from, STATE to, EVENT e) {
-    transitions.getUnchecked(e).put(from, to);
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/package-info.java
deleted file mode 100644
index 045409e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.common.statemachine;
-/**
- state machine template class for ozone.
- **/
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
deleted file mode 100644
index 75cf613..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-/**
- * KSM Constants.
- */
-public final class KSMConfigKeys {
-  /**
-   * Never constructed.
-   */
-  private KSMConfigKeys() {
-  }
-
-
-  public static final String OZONE_KSM_HANDLER_COUNT_KEY =
-      "ozone.ksm.handler.count.key";
-  public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 20;
-
-  public static final String OZONE_KSM_ADDRESS_KEY =
-      "ozone.ksm.address";
-  public static final String OZONE_KSM_BIND_HOST_DEFAULT =
-      "0.0.0.0";
-  public static final int OZONE_KSM_PORT_DEFAULT = 9862;
-
-  public static final String OZONE_KSM_HTTP_ENABLED_KEY =
-      "ozone.ksm.http.enabled";
-  public static final String OZONE_KSM_HTTP_BIND_HOST_KEY =
-      "ozone.ksm.http-bind-host";
-  public static final String OZONE_KSM_HTTPS_BIND_HOST_KEY =
-      "ozone.ksm.https-bind-host";
-  public static final String OZONE_KSM_HTTP_ADDRESS_KEY =
-      "ozone.ksm.http-address";
-  public static final String OZONE_KSM_HTTPS_ADDRESS_KEY =
-      "ozone.ksm.https-address";
-  public static final String OZONE_KSM_KEYTAB_FILE =
-      "ozone.ksm.keytab.file";
-  public static final String OZONE_KSM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
-  public static final int OZONE_KSM_HTTP_BIND_PORT_DEFAULT = 9874;
-  public static final int OZONE_KSM_HTTPS_BIND_PORT_DEFAULT = 9875;
-
-  // LevelDB cache file uses an off-heap cache in LevelDB of 128 MB.
-  public static final String OZONE_KSM_DB_CACHE_SIZE_MB =
-      "ozone.ksm.db.cache.size.mb";
-  public static final int OZONE_KSM_DB_CACHE_SIZE_DEFAULT = 128;
-
-  public static final String OZONE_KSM_USER_MAX_VOLUME =
-      "ozone.ksm.user.max.volume";
-  public static final int OZONE_KSM_USER_MAX_VOLUME_DEFAULT = 1024;
-
-  // KSM Default user/group permissions
-  public static final String OZONE_KSM_USER_RIGHTS =
-      "ozone.ksm.user.rights";
-  public static final OzoneAcl.OzoneACLRights OZONE_KSM_USER_RIGHTS_DEFAULT =
-      OzoneAcl.OzoneACLRights.READ_WRITE;
-
-  public static final String OZONE_KSM_GROUP_RIGHTS =
-      "ozone.ksm.group.rights";
-  public static final OzoneAcl.OzoneACLRights OZONE_KSM_GROUP_RIGHTS_DEFAULT =
-      OzoneAcl.OzoneACLRights.READ_WRITE;
-
-  public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK =
-      "ozone.key.deleting.limit.per.task";
-  public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketArgs.java
deleted file mode 100644
index 1211b50..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketArgs.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.BucketArgs;
-import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
-
-/**
- * A class that encapsulates Bucket Arguments.
- */
-public final class KsmBucketArgs {
-  /**
-   * Name of the volume in which the bucket belongs to.
-   */
-  private final String volumeName;
-  /**
-   * Name of the bucket.
-   */
-  private final String bucketName;
-  /**
-   * ACL's that are to be added for the bucket.
-   */
-  private List<OzoneAcl> addAcls;
-  /**
-   * ACL's that are to be removed from the bucket.
-   */
-  private List<OzoneAcl> removeAcls;
-  /**
-   * Bucket Version flag.
-   */
-  private Boolean isVersionEnabled;
-  /**
-   * Type of storage to be used for this bucket.
-   * [RAM_DISK, SSD, DISK, ARCHIVE]
-   */
-  private StorageType storageType;
-
-  /**
-   * Private constructor, constructed via builder.
-   * @param volumeName - Volume name.
-   * @param bucketName - Bucket name.
-   * @param addAcls - ACL's to be added.
-   * @param removeAcls - ACL's to be removed.
-   * @param isVersionEnabled - Bucket version flag.
-   * @param storageType - Storage type to be used.
-   */
-  private KsmBucketArgs(String volumeName, String bucketName,
-      List<OzoneAcl> addAcls, List<OzoneAcl> removeAcls,
-      Boolean isVersionEnabled, StorageType storageType) {
-    this.volumeName = volumeName;
-    this.bucketName = bucketName;
-    this.addAcls = addAcls;
-    this.removeAcls = removeAcls;
-    this.isVersionEnabled = isVersionEnabled;
-    this.storageType = storageType;
-  }
-
-  /**
-   * Returns the Volume Name.
-   * @return String.
-   */
-  public String getVolumeName() {
-    return volumeName;
-  }
-
-  /**
-   * Returns the Bucket Name.
-   * @return String
-   */
-  public String getBucketName() {
-    return bucketName;
-  }
-
-  /**
-   * Returns the ACL's that are to be added.
-   * @return List<OzoneAclInfo>
-   */
-  public List<OzoneAcl> getAddAcls() {
-    return addAcls;
-  }
-
-  /**
-   * Returns the ACL's that are to be removed.
-   * @return List<OzoneAclInfo>
-   */
-  public List<OzoneAcl> getRemoveAcls() {
-    return removeAcls;
-  }
-
-  /**
-   * Returns true if bucket version is enabled, else false.
-   * @return isVersionEnabled
-   */
-  public Boolean getIsVersionEnabled() {
-    return isVersionEnabled;
-  }
-
-  /**
-   * Returns the type of storage to be used.
-   * @return StorageType
-   */
-  public StorageType getStorageType() {
-    return storageType;
-  }
-
-  /**
-   * Returns new builder class that builds a KsmBucketArgs.
-   *
-   * @return Builder
-   */
-  public static Builder newBuilder() {
-    return new Builder();
-  }
-
-  /**
-   * Builder for KsmBucketArgs.
-   */
-  public static class Builder {
-    private String volumeName;
-    private String bucketName;
-    private List<OzoneAcl> addAcls;
-    private List<OzoneAcl> removeAcls;
-    private Boolean isVersionEnabled;
-    private StorageType storageType;
-
-    public Builder setVolumeName(String volume) {
-      this.volumeName = volume;
-      return this;
-    }
-
-    public Builder setBucketName(String bucket) {
-      this.bucketName = bucket;
-      return this;
-    }
-
-    public Builder setAddAcls(List<OzoneAcl> acls) {
-      this.addAcls = acls;
-      return this;
-    }
-
-    public Builder setRemoveAcls(List<OzoneAcl> acls) {
-      this.removeAcls = acls;
-      return this;
-    }
-
-    public Builder setIsVersionEnabled(Boolean versionFlag) {
-      this.isVersionEnabled = versionFlag;
-      return this;
-    }
-
-    public Builder setStorageType(StorageType storage) {
-      this.storageType = storage;
-      return this;
-    }
-
-    /**
-     * Constructs the KsmBucketArgs.
-     * @return instance of KsmBucketArgs.
-     */
-    public KsmBucketArgs build() {
-      Preconditions.checkNotNull(volumeName);
-      Preconditions.checkNotNull(bucketName);
-      return new KsmBucketArgs(volumeName, bucketName, addAcls,
-          removeAcls, isVersionEnabled, storageType);
-    }
-  }
-
-  /**
-   * Creates BucketArgs protobuf from KsmBucketArgs.
-   */
-  public BucketArgs getProtobuf() {
-    BucketArgs.Builder builder = BucketArgs.newBuilder();
-    builder.setVolumeName(volumeName)
-        .setBucketName(bucketName);
-    if(addAcls != null && !addAcls.isEmpty()) {
-      builder.addAllAddAcls(addAcls.stream().map(
-          KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
-    }
-    if(removeAcls != null && !removeAcls.isEmpty()) {
-      builder.addAllRemoveAcls(removeAcls.stream().map(
-          KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
-    }
-    if(isVersionEnabled != null) {
-      builder.setIsVersionEnabled(isVersionEnabled);
-    }
-    if(storageType != null) {
-      builder.setStorageType(
-          PBHelperClient.convertStorageType(storageType));
-    }
-    return builder.build();
-  }
-
-  /**
-   * Parses BucketInfo protobuf and creates KsmBucketArgs.
-   * @param bucketArgs
-   * @return instance of KsmBucketArgs
-   */
-  public static KsmBucketArgs getFromProtobuf(BucketArgs bucketArgs) {
-    return new KsmBucketArgs(bucketArgs.getVolumeName(),
-        bucketArgs.getBucketName(),
-        bucketArgs.getAddAclsList().stream().map(
-            KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
-        bucketArgs.getRemoveAclsList().stream().map(
-            KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
-        bucketArgs.hasIsVersionEnabled() ?
-            bucketArgs.getIsVersionEnabled() : null,
-        bucketArgs.hasStorageType() ? PBHelperClient.convertStorageType(
-            bucketArgs.getStorageType()) : null);
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java
deleted file mode 100644
index a49137a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.BucketInfo;
-import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * A class that encapsulates Bucket Info.
- */
-public final class KsmBucketInfo {
-  /**
-   * Name of the volume in which the bucket belongs to.
-   */
-  private final String volumeName;
-  /**
-   * Name of the bucket.
-   */
-  private final String bucketName;
-  /**
-   * ACL Information.
-   */
-  private List<OzoneAcl> acls;
-  /**
-   * Bucket Version flag.
-   */
-  private Boolean isVersionEnabled;
-  /**
-   * Type of storage to be used for this bucket.
-   * [RAM_DISK, SSD, DISK, ARCHIVE]
-   */
-  private StorageType storageType;
-  /**
-   * Creation time of bucket.
-   */
-  private final long creationTime;
-
-  /**
-   * Private constructor, constructed via builder.
-   * @param volumeName - Volume name.
-   * @param bucketName - Bucket name.
-   * @param acls - list of ACLs.
-   * @param isVersionEnabled - Bucket version flag.
-   * @param storageType - Storage type to be used.
-   * @param creationTime - Bucket creation time.
-   */
-  private KsmBucketInfo(String volumeName, String bucketName,
-                        List<OzoneAcl> acls, boolean isVersionEnabled,
-                        StorageType storageType, long creationTime) {
-    this.volumeName = volumeName;
-    this.bucketName = bucketName;
-    this.acls = acls;
-    this.isVersionEnabled = isVersionEnabled;
-    this.storageType = storageType;
-    this.creationTime = creationTime;
-  }
-
-  /**
-   * Returns the Volume Name.
-   * @return String.
-   */
-  public String getVolumeName() {
-    return volumeName;
-  }
-
-  /**
-   * Returns the Bucket Name.
-   * @return String
-   */
-  public String getBucketName() {
-    return bucketName;
-  }
-
-  /**
-   * Returns the ACL's associated with this bucket.
-   * @return List<OzoneAcl>
-   */
-  public List<OzoneAcl> getAcls() {
-    return acls;
-  }
-
-  /**
-   * Returns true if bucket version is enabled, else false.
-   * @return isVersionEnabled
-   */
-  public boolean getIsVersionEnabled() {
-    return isVersionEnabled;
-  }
-
-  /**
-   * Returns the type of storage to be used.
-   * @return StorageType
-   */
-  public StorageType getStorageType() {
-    return storageType;
-  }
-
-  /**
-   * Returns creation time.
-   *
-   * @return long
-   */
-  public long getCreationTime() {
-    return creationTime;
-  }
-
-  /**
-   * Returns new builder class that builds a KsmBucketInfo.
-   *
-   * @return Builder
-   */
-  public static Builder newBuilder() {
-    return new Builder();
-  }
-
-  /**
-   * Builder for KsmBucketInfo.
-   */
-  public static class Builder {
-    private String volumeName;
-    private String bucketName;
-    private List<OzoneAcl> acls;
-    private Boolean isVersionEnabled;
-    private StorageType storageType;
-    private long creationTime;
-
-    Builder() {
-      //Default values
-      this.acls = new LinkedList<>();
-      this.isVersionEnabled = false;
-      this.storageType = StorageType.DISK;
-    }
-
-    public Builder setVolumeName(String volume) {
-      this.volumeName = volume;
-      return this;
-    }
-
-    public Builder setBucketName(String bucket) {
-      this.bucketName = bucket;
-      return this;
-    }
-
-    public Builder setAcls(List<OzoneAcl> listOfAcls) {
-      this.acls = listOfAcls;
-      return this;
-    }
-
-    public Builder setIsVersionEnabled(Boolean versionFlag) {
-      this.isVersionEnabled = versionFlag;
-      return this;
-    }
-
-    public Builder setStorageType(StorageType storage) {
-      this.storageType = storage;
-      return this;
-    }
-
-    public Builder setCreationTime(long createdOn) {
-      this.creationTime = createdOn;
-      return this;
-    }
-
-    /**
-     * Constructs the KsmBucketInfo.
-     * @return instance of KsmBucketInfo.
-     */
-    public KsmBucketInfo build() {
-      Preconditions.checkNotNull(volumeName);
-      Preconditions.checkNotNull(bucketName);
-      Preconditions.checkNotNull(acls);
-      Preconditions.checkNotNull(isVersionEnabled);
-      Preconditions.checkNotNull(storageType);
-
-      return new KsmBucketInfo(volumeName, bucketName, acls,
-          isVersionEnabled, storageType, creationTime);
-    }
-  }
-
-  /**
-   * Creates BucketInfo protobuf from KsmBucketInfo.
-   */
-  public BucketInfo getProtobuf() {
-    return BucketInfo.newBuilder()
-        .setVolumeName(volumeName)
-        .setBucketName(bucketName)
-        .addAllAcls(acls.stream().map(
-            KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()))
-        .setIsVersionEnabled(isVersionEnabled)
-        .setStorageType(PBHelperClient.convertStorageType(
-            storageType))
-        .setCreationTime(creationTime)
-        .build();
-  }
-
-  /**
-   * Parses BucketInfo protobuf and creates KsmBucketInfo.
-   * @param bucketInfo
-   * @return instance of KsmBucketInfo
-   */
-  public static KsmBucketInfo getFromProtobuf(BucketInfo bucketInfo) {
-    return new KsmBucketInfo(
-        bucketInfo.getVolumeName(),
-        bucketInfo.getBucketName(),
-        bucketInfo.getAclsList().stream().map(
-            KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
-        bucketInfo.getIsVersionEnabled(),
-        PBHelperClient.convertStorageType(
-            bucketInfo.getStorageType()), bucketInfo.getCreationTime());
-  }
-}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[11/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java
deleted file mode 100644
index cfdf025..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java
+++ /dev/null
@@ -1,303 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.utils;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.client.io.LengthInputStream;
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.util.Time;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.MediaType;
-import java.io.File;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.nio.charset.Charset;
-import java.nio.file.Paths;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.List;
-import java.util.Locale;
-import java.util.TimeZone;
-import java.util.UUID;
-
-/**
- * Set of Utility functions used in ozone.
- */
-@InterfaceAudience.Private
-public final class OzoneUtils {
-
-  public static final String ENCODING_NAME = "UTF-8";
-  public static final Charset ENCODING = Charset.forName(ENCODING_NAME);
-
-  private OzoneUtils() {
-    // Never constructed
-  }
-
-  /**
-   * Date format that used in ozone. Here the format is thread safe to use.
-   */
-  private static final ThreadLocal<SimpleDateFormat> DATE_FORMAT =
-      new ThreadLocal<SimpleDateFormat>() {
-    @Override
-    protected SimpleDateFormat initialValue() {
-      SimpleDateFormat format = new SimpleDateFormat(
-          OzoneConsts.OZONE_DATE_FORMAT, Locale.US);
-      format.setTimeZone(TimeZone.getTimeZone(OzoneConsts.OZONE_TIME_ZONE));
-
-      return format;
-    }
-  };
-
-  /**
-   * verifies that bucket name / volume name is a valid DNS name.
-   *
-   * @param resName Bucket or volume Name to be validated
-   *
-   * @throws IllegalArgumentException
-   */
-  public static void verifyResourceName(String resName)
-      throws IllegalArgumentException {
-    OzoneClientUtils.verifyResourceName(resName);
-  }
-
-  /**
-   * Verifies that max key length is a valid value.
-   *
-   * @param length
-   *          The max key length to be validated
-   *
-   * @throws IllegalArgumentException
-   */
-  public static void verifyMaxKeyLength(String length)
-      throws IllegalArgumentException {
-    int maxKey = 0;
-    try {
-      maxKey = Integer.parseInt(length);
-    } catch (NumberFormatException nfe) {
-      throw new IllegalArgumentException(
-          "Invalid max key length, the vaule should be digital.");
-    }
-
-    if (maxKey <= 0) {
-      throw new IllegalArgumentException(
-          "Invalid max key length, the vaule should be a positive number.");
-    }
-  }
-
-  /**
-   * Returns a random Request ID.
-   *
-   * Request ID is returned to the client as well as flows through the system
-   * facilitating debugging on why a certain request failed.
-   *
-   * @return String random request ID
-   */
-  public static String getRequestID() {
-    return UUID.randomUUID().toString();
-  }
-
-  /**
-   * Return host name if possible.
-   *
-   * @return Host Name or localhost
-   */
-  public static String getHostName() {
-    String host = "localhost";
-    try {
-      host = InetAddress.getLocalHost().getHostName();
-    } catch (UnknownHostException e) {
-      // Ignore the error
-    }
-    return host;
-  }
-
-  /**
-   * Basic validate routine to make sure that all the
-   * required headers are in place.
-   *
-   * @param request - http request
-   * @param headers - http headers
-   * @param reqId - request id
-   * @param resource - Resource Name
-   * @param hostname - Hostname
-   *
-   * @throws OzoneException
-   */
-  public static void validate(Request request, HttpHeaders headers,
-                              String reqId, String resource, String hostname)
-      throws OzoneException {
-
-    List<String> ozHeader =
-        headers.getRequestHeader(Header.OZONE_VERSION_HEADER);
-    if (ozHeader == null) {
-      throw ErrorTable
-          .newError(ErrorTable.MISSING_VERSION, reqId, resource, hostname);
-    }
-
-    List<String> date = headers.getRequestHeader(HttpHeaders.DATE);
-    if (date == null) {
-      throw ErrorTable
-          .newError(ErrorTable.MISSING_DATE, reqId, resource, hostname);
-    }
-
-    /*
-    TODO :
-    Ignore the results for time being. Eventually we can validate if the
-    request Date time is too skewed and reject if it is so.
-    */
-    parseDate(date.get(0), reqId, resource, hostname);
-
-  }
-
-  /**
-   * Parses the Date String coming from the Users.
-   *
-   * @param dateString - Date String
-   * @param reqID - Ozone Request ID
-   * @param resource - Resource Name
-   * @param hostname - HostName
-   *
-   * @return - Date
-   *
-   * @throws OzoneException - in case of parsing error
-   */
-  public static synchronized Date parseDate(String dateString, String reqID,
-                                            String resource, String hostname)
-      throws OzoneException {
-    try {
-      return DATE_FORMAT.get().parse(dateString);
-    } catch (ParseException ex) {
-      OzoneException exp =
-          ErrorTable.newError(ErrorTable.BAD_DATE, reqID, resource, hostname);
-      exp.setMessage(ex.getMessage());
-      throw exp;
-    }
-  }
-
-  /**
-   * Returns a response with appropriate OZONE headers and payload.
-   *
-   * @param args - UserArgs or Inherited class
-   * @param statusCode - HttpStatus code
-   * @param payload - Content Body
-   *
-   * @return JAX-RS Response
-   */
-  public static Response getResponse(UserArgs args, int statusCode,
-                                     String payload) {
-    String date = DATE_FORMAT.get().format(new Date(Time.now()));
-    return Response.ok(payload)
-        .header(Header.OZONE_SERVER_NAME, args.getHostName())
-        .header(Header.OZONE_REQUEST_ID, args.getRequestID())
-        .header(HttpHeaders.DATE, date).status(statusCode).build();
-  }
-
-  /**
-   * Returns a response with appropriate OZONE headers and payload.
-   *
-   * @param args - UserArgs or Inherited class
-   * @param statusCode - HttpStatus code
-   * @param stream InputStream
-   *
-   * @return JAX-RS Response
-   */
-  public static Response getResponse(UserArgs args, int statusCode,
-                                     LengthInputStream stream) {
-    String date = DATE_FORMAT.get().format(new Date(Time.now()));
-    return Response.ok(stream, MediaType.APPLICATION_OCTET_STREAM)
-        .header(Header.OZONE_SERVER_NAME, args.getHostName())
-        .header(Header.OZONE_REQUEST_ID, args.getRequestID())
-        .header(HttpHeaders.DATE, date).status(statusCode)
-        .header(HttpHeaders.CONTENT_LENGTH, stream.getLength())
-        .build();
-
-  }
-
-  /**
-   * Checks and creates Ozone Metadir Path if it does not exist.
-   *
-   * @param conf - Configuration
-   *
-   * @return File MetaDir
-   */
-  public static File getOzoneMetaDirPath(Configuration conf) {
-    String metaDirPath = conf.getTrimmed(OzoneConfigKeys
-        .OZONE_METADATA_DIRS);
-    Preconditions.checkNotNull(metaDirPath);
-    File dirPath = new File(metaDirPath);
-    if (!dirPath.exists() && !dirPath.mkdirs()) {
-      throw new IllegalArgumentException("Unable to create paths. Path: " +
-          dirPath);
-    }
-    return dirPath;
-  }
-
-  /**
-   * Get the path for datanode id file.
-   *
-   * @param conf - Configuration
-   * @return the path of datanode id as string
-   */
-  public static String getDatanodeIDPath(Configuration conf) {
-    String dataNodeIDPath = conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID);
-    if (dataNodeIDPath == null) {
-      String metaPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
-      if (Strings.isNullOrEmpty(metaPath)) {
-        // this means meta data is not found, in theory should not happen at
-        // this point because should've failed earlier.
-        throw new IllegalArgumentException("Unable to locate meta data" +
-            "directory when getting datanode id path");
-      }
-      dataNodeIDPath = Paths.get(metaPath,
-          ScmConfigKeys.OZONE_SCM_DATANODE_ID_PATH_DEFAULT).toString();
-    }
-    return dataNodeIDPath;
-  }
-
-  /**
-   * Convert time in millisecond to a human readable format required in ozone.
-   * @return a human readable string for the input time
-   */
-  public static String formatTime(long millis) {
-    return DATE_FORMAT.get().format(millis);
-  }
-
-  /**
-   * Convert time in ozone date format to millisecond.
-   * @return time in milliseconds
-   */
-  public static long formatDate(String date) throws ParseException {
-    Preconditions.checkNotNull(date, "Date string should not be null.");
-    return DATE_FORMAT.get().parse(date).getTime();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/package-info.java
deleted file mode 100644
index 178157f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.utils;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundService.java
deleted file mode 100644
index 2ff4e55..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundService.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.utils;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.Future;
-import java.util.concurrent.ExecutionException;
-
-/**
- * An abstract class for a background service in ozone.
- * A background service schedules multiple child tasks in parallel
- * in a certain period. In each interval, it waits until all the tasks
- * finish execution and then schedule next interval.
- */
-public abstract class BackgroundService {
-
-  @VisibleForTesting
-  public static final Logger LOG =
-      LoggerFactory.getLogger(BackgroundService.class);
-
-  // Executor to launch child tasks
-  private final ScheduledExecutorService exec;
-  private final ThreadGroup threadGroup;
-  private final ThreadFactory threadFactory;
-  private final String serviceName;
-  private final long interval;
-  private final long serviceTimeout;
-  private final TimeUnit unit;
-  private final PeriodicalTask service;
-
-  public BackgroundService(String serviceName, long interval,
-      TimeUnit unit, int threadPoolSize, long serviceTimeout) {
-    this.interval = interval;
-    this.unit = unit;
-    this.serviceName = serviceName;
-    this.serviceTimeout = serviceTimeout;
-    threadGroup = new ThreadGroup(serviceName);
-    ThreadFactory tf = r -> new Thread(threadGroup, r);
-    threadFactory = new ThreadFactoryBuilder()
-        .setThreadFactory(tf)
-        .setDaemon(true)
-        .setNameFormat(serviceName + "#%d")
-        .build();
-    exec = Executors.newScheduledThreadPool(threadPoolSize, threadFactory);
-    service = new PeriodicalTask();
-  }
-
-  protected ExecutorService getExecutorService() {
-    return this.exec;
-  }
-
-  @VisibleForTesting
-  public int getThreadCount() {
-    return threadGroup.activeCount();
-  }
-
-  @VisibleForTesting
-  public void triggerBackgroundTaskForTesting() {
-    service.run();
-  }
-
-  // start service
-  public void start() {
-    exec.scheduleWithFixedDelay(service, 0, interval, unit);
-  }
-
-  public abstract BackgroundTaskQueue getTasks();
-
-  /**
-   * Run one or more background tasks concurrently.
-   * Wait until all tasks to return the result.
-   */
-  public class PeriodicalTask implements Runnable {
-    @Override
-    public synchronized void run() {
-      LOG.debug("Running background service : {}", serviceName);
-      BackgroundTaskQueue tasks = getTasks();
-      if (tasks.isEmpty()) {
-        // No task found, or some problems to init tasks
-        // return and retry in next interval.
-        return;
-      }
-
-      LOG.debug("Number of background tasks to execute : {}", tasks.size());
-      CompletionService<BackgroundTaskResult> taskCompletionService =
-          new ExecutorCompletionService<>(exec);
-
-      List<Future<BackgroundTaskResult>> results = Lists.newArrayList();
-      while (tasks.size() > 0) {
-        BackgroundTask task = tasks.poll();
-        Future<BackgroundTaskResult> result =
-            taskCompletionService.submit(task);
-        results.add(result);
-      }
-
-      results.parallelStream().forEach(taskResultFuture -> {
-        try {
-          // Collect task results
-          BackgroundTaskResult result = serviceTimeout > 0
-              ? taskResultFuture.get(serviceTimeout, TimeUnit.MILLISECONDS)
-              : taskResultFuture.get();
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("task execution result size {}", result.getSize());
-          }
-        } catch (InterruptedException | ExecutionException e) {
-          LOG.warn(
-              "Background task fails to execute, "
-                  + "retrying in next interval", e);
-        } catch (TimeoutException e) {
-          LOG.warn("Background task executes timed out, "
-              + "retrying in next interval", e);
-        }
-      });
-    }
-  }
-
-  // shutdown and make sure all threads are properly released.
-  public void shutdown() {
-    LOG.info("Shutting down service {}", this.serviceName);
-    exec.shutdown();
-    try {
-      if (!exec.awaitTermination(60, TimeUnit.SECONDS)) {
-        exec.shutdownNow();
-      }
-    } catch (InterruptedException e) {
-      exec.shutdownNow();
-    }
-    if (threadGroup.activeCount() == 0 && !threadGroup.isDestroyed()) {
-      threadGroup.destroy();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTask.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTask.java
deleted file mode 100644
index 47e8ebc..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTask.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.utils;
-
-import java.util.concurrent.Callable;
-
-/**
- * A task thread to run by {@link BackgroundService}.
- */
-public interface BackgroundTask<T> extends Callable<T> {
-
-  int getPriority();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTaskQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTaskQueue.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTaskQueue.java
deleted file mode 100644
index b56ef0c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTaskQueue.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.utils;
-
-import java.util.PriorityQueue;
-
-/**
- * A priority queue that stores a number of {@link BackgroundTask}.
- */
-public class BackgroundTaskQueue {
-
-  private final PriorityQueue<BackgroundTask> tasks;
-
-  public BackgroundTaskQueue() {
-    tasks = new PriorityQueue<>((task1, task2)
-        -> task1.getPriority() - task2.getPriority());
-  }
-
-  /**
-   * @return the head task in this queue.
-   */
-  public synchronized BackgroundTask poll() {
-    return tasks.poll();
-  }
-
-  /**
-   * Add a {@link BackgroundTask} to the queue,
-   * the task will be sorted by its priority.
-   *
-   * @param task
-   */
-  public synchronized void add(BackgroundTask task) {
-    tasks.add(task);
-  }
-
-  /**
-   * @return true if the queue contains no task, false otherwise.
-   */
-  public synchronized boolean isEmpty() {
-    return tasks.isEmpty();
-  }
-
-  /**
-   * @return the size of the queue.
-   */
-  public synchronized int size() {
-    return tasks.size();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTaskResult.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTaskResult.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTaskResult.java
deleted file mode 100644
index 198300f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTaskResult.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.utils;
-
-/**
- * Result of a {@link BackgroundTask}.
- */
-public interface BackgroundTaskResult {
-
-  /**
-   * Returns the size of entries included in this result.
-   */
-  int getSize();
-
-  /**
-   * An empty task result implementation.
-   */
-  class EmptyTaskResult implements BackgroundTaskResult {
-
-    public static EmptyTaskResult newResult() {
-      return new EmptyTaskResult();
-    }
-
-    @Override
-    public int getSize() {
-      return 0;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BatchOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BatchOperation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BatchOperation.java
deleted file mode 100644
index 47699eb..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BatchOperation.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.utils;
-
-import com.google.common.collect.Lists;
-
-import java.util.List;
-
-/**
- * An utility class to store a batch of DB write operations.
- */
-public class BatchOperation {
-
-  /**
-   * Enum for write operations.
-   */
-  public enum Operation {
-    DELETE, PUT
-  }
-
-  private List<SingleOperation> operations =
-      Lists.newArrayList();
-
-  /**
-   * Add a PUT operation into the batch.
-   */
-  public void put(byte[] key, byte[] value) {
-    operations.add(new SingleOperation(Operation.PUT, key, value));
-  }
-
-  /**
-   * Add a DELETE operation into the batch.
-   */
-  public void delete(byte[] key) {
-    operations.add(new SingleOperation(Operation.DELETE, key, null));
-
-  }
-
-  public List<SingleOperation> getOperations() {
-    return operations;
-  }
-
-  /**
-   * A SingleOperation represents a PUT or DELETE operation
-   * and the data the operation needs to manipulates.
-   */
-  public static class SingleOperation {
-
-    private Operation opt;
-    private byte[] key;
-    private byte[] value;
-
-    public SingleOperation(Operation opt, byte[] key, byte[] value) {
-      this.opt = opt;
-      if (key == null) {
-        throw new IllegalArgumentException("key cannot be null");
-      }
-      this.key = key.clone();
-      this.value = value == null ? null : value.clone();
-    }
-
-    public Operation getOpt() {
-      return opt;
-    }
-
-    public byte[] getKey() {
-      return key.clone();
-    }
-
-    public byte[] getValue() {
-      return value == null ? null : value.clone();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/EntryConsumer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/EntryConsumer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/EntryConsumer.java
deleted file mode 100644
index c407398..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/EntryConsumer.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.utils;
-
-import java.io.IOException;
-
-/**
- * A consumer for metadata store key-value entries.
- * Used by {@link MetadataStore} class.
- */
-@FunctionalInterface
-public interface EntryConsumer {
-
-  /**
-   * Consumes a key and value and produces a boolean result.
-   * @param key key
-   * @param value value
-   * @return a boolean value produced by the consumer
-   * @throws IOException
-   */
-  boolean consume(byte[] key, byte[] value) throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
deleted file mode 100644
index 38cee76..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.utils;
-
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
-import org.fusesource.leveldbjni.JniDBFactory;
-import org.iq80.leveldb.WriteBatch;
-import org.iq80.leveldb.DB;
-import org.iq80.leveldb.Options;
-import org.iq80.leveldb.WriteOptions;
-import org.iq80.leveldb.DBIterator;
-import org.iq80.leveldb.Snapshot;
-import org.iq80.leveldb.ReadOptions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Map.Entry;
-
-/**
- * LevelDB interface.
- */
-public class LevelDBStore implements MetadataStore {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(LevelDBStore.class);
-
-  private DB db;
-  private final File dbFile;
-  private final Options dbOptions;
-  private final WriteOptions writeOptions;
-
-  public LevelDBStore(File dbPath, boolean createIfMissing)
-      throws IOException {
-    dbOptions = new Options();
-    dbOptions.createIfMissing(createIfMissing);
-    this.dbFile = dbPath;
-    this.writeOptions = new WriteOptions().sync(true);
-    openDB(dbPath, dbOptions);
-  }
-
-  /**
-   * Opens a DB file.
-   *
-   * @param dbPath          - DB File path
-   * @throws IOException
-   */
-  public LevelDBStore(File dbPath, Options options)
-      throws IOException {
-    dbOptions = options;
-    this.dbFile = dbPath;
-    this.writeOptions = new WriteOptions().sync(true);
-    openDB(dbPath, dbOptions);
-  }
-
-  private void openDB(File dbPath, Options options) throws IOException {
-    db = JniDBFactory.factory.open(dbPath, options);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("LevelDB successfully opened");
-      LOG.debug("[Option] cacheSize = " + options.cacheSize());
-      LOG.debug("[Option] createIfMissing = " + options.createIfMissing());
-      LOG.debug("[Option] blockSize = " + options.blockSize());
-      LOG.debug("[Option] compressionType= " + options.compressionType());
-      LOG.debug("[Option] maxOpenFiles= " + options.maxOpenFiles());
-      LOG.debug("[Option] writeBufferSize= "+ options.writeBufferSize());
-    }
-  }
-
-  /**
-   * Puts a Key into file.
-   *
-   * @param key   - key
-   * @param value - value
-   */
-  @Override
-  public void put(byte[] key, byte[] value) {
-    db.put(key, value, writeOptions);
-  }
-
-  /**
-   * Get Key.
-   *
-   * @param key key
-   * @return value
-   */
-  @Override
-  public byte[] get(byte[] key) {
-    return db.get(key);
-  }
-
-  /**
-   * Delete Key.
-   *
-   * @param key - Key
-   */
-  @Override
-  public void delete(byte[] key) {
-    db.delete(key);
-  }
-
-  /**
-   * Closes the DB.
-   *
-   * @throws IOException
-   */
-  @Override
-  public void close() throws IOException {
-    if (db != null){
-      db.close();
-    }
-  }
-
-  /**
-   * Returns true if the DB is empty.
-   *
-   * @return boolean
-   * @throws IOException
-   */
-  @Override
-  public boolean isEmpty() throws IOException {
-    try (DBIterator iter = db.iterator()) {
-      iter.seekToFirst();
-      boolean hasNext = !iter.hasNext();
-      return hasNext;
-    }
-  }
-
-  /**
-   * Returns the actual levelDB object.
-   * @return DB handle.
-   */
-  public DB getDB() {
-    return db;
-  }
-
-  /**
-   * Returns an iterator on all the key-value pairs in the DB.
-   * @return an iterator on DB entries.
-   */
-  public DBIterator getIterator() {
-    return db.iterator();
-  }
-
-
-  @Override
-  public void destroy() throws IOException {
-    close();
-    JniDBFactory.factory.destroy(dbFile, dbOptions);
-  }
-
-  @Override
-  public ImmutablePair<byte[], byte[]> peekAround(int offset,
-      byte[] from) throws IOException, IllegalArgumentException {
-    try (DBIterator it = db.iterator()) {
-      if (from == null) {
-        it.seekToFirst();
-      } else {
-        it.seek(from);
-      }
-      if (!it.hasNext()) {
-        return null;
-      }
-      switch (offset) {
-      case 0:
-        Entry<byte[], byte[]> current = it.next();
-        return new ImmutablePair<>(current.getKey(), current.getValue());
-      case 1:
-        if (it.next() != null && it.hasNext()) {
-          Entry<byte[], byte[]> next = it.peekNext();
-          return new ImmutablePair<>(next.getKey(), next.getValue());
-        }
-        break;
-      case -1:
-        if (it.hasPrev()) {
-          Entry<byte[], byte[]> prev = it.peekPrev();
-          return new ImmutablePair<>(prev.getKey(), prev.getValue());
-        }
-        break;
-      default:
-        throw new IllegalArgumentException(
-            "Position can only be -1, 0 " + "or 1, but found " + offset);
-      }
-    }
-    return null;
-  }
-
-  @Override
-  public void iterate(byte[] from, EntryConsumer consumer)
-      throws IOException {
-    try (DBIterator iter = db.iterator()) {
-      if (from != null) {
-        iter.seek(from);
-      } else {
-        iter.seekToFirst();
-      }
-      while (iter.hasNext()) {
-        Entry<byte[], byte[]> current = iter.next();
-        if (!consumer.consume(current.getKey(),
-            current.getValue())) {
-          break;
-        }
-      }
-    }
-  }
-
-  /**
-   * Compacts the DB by removing deleted keys etc.
-   * @throws IOException if there is an error.
-   */
-  @Override
-  public void compactDB() throws IOException {
-    if(db != null) {
-      // From LevelDB docs : begin == null and end == null means the whole DB.
-      db.compactRange(null, null);
-    }
-  }
-
-  @Override
-  public void writeBatch(BatchOperation operation) throws IOException {
-    List<BatchOperation.SingleOperation> operations =
-        operation.getOperations();
-    if (!operations.isEmpty()) {
-      try (WriteBatch writeBatch = db.createWriteBatch()) {
-        for (BatchOperation.SingleOperation opt : operations) {
-          switch (opt.getOpt()) {
-          case DELETE:
-            writeBatch.delete(opt.getKey());
-            break;
-          case PUT:
-            writeBatch.put(opt.getKey(), opt.getValue());
-            break;
-          default:
-            throw new IllegalArgumentException("Invalid operation "
-                + opt.getOpt());
-          }
-        }
-        db.write(writeBatch);
-      }
-    }
-  }
-
-  @Override
-  public List<Map.Entry<byte[], byte[]>> getRangeKVs(byte[] startKey,
-      int count, MetadataKeyFilters.MetadataKeyFilter... filters)
-      throws IOException, IllegalArgumentException {
-    return getRangeKVs(startKey, count, false, filters);
-  }
-
-  @Override
-  public List<Map.Entry<byte[], byte[]>> getSequentialRangeKVs(byte[] startKey,
-      int count, MetadataKeyFilters.MetadataKeyFilter... filters)
-      throws IOException, IllegalArgumentException {
-    return getRangeKVs(startKey, count, true, filters);
-  }
-
-  /**
-   * Returns a certain range of key value pairs as a list based on a
-   * startKey or count. Further a {@link MetadataKeyFilter} can be added to
-   * filter keys if necessary. To prevent race conditions while listing
-   * entries, this implementation takes a snapshot and lists the entries from
-   * the snapshot. This may, on the other hand, cause the range result slight
-   * different with actual data if data is updating concurrently.
-   * <p>
-   * If the startKey is specified and found in levelDB, this key and the keys
-   * after this key will be included in the result. If the startKey is null
-   * all entries will be included as long as other conditions are satisfied.
-   * If the given startKey doesn't exist, an empty list will be returned.
-   * <p>
-   * The count argument is to limit number of total entries to return,
-   * the value for count must be an integer greater than 0.
-   * <p>
-   * This method allows to specify one or more {@link MetadataKeyFilter}
-   * to filter keys by certain condition. Once given, only the entries
-   * whose key passes all the filters will be included in the result.
-   *
-   * @param startKey a start key.
-   * @param count max number of entries to return.
-   * @param filters customized one or more {@link MetadataKeyFilter}.
-   * @return a list of entries found in the database or an empty list if the
-   * startKey is invalid.
-   * @throws IOException if there are I/O errors.
-   * @throws IllegalArgumentException if count is less than 0.
-   */
-  private List<Entry<byte[], byte[]>> getRangeKVs(byte[] startKey,
-      int count, boolean sequential, MetadataKeyFilter... filters)
-      throws IOException {
-    List<Entry<byte[], byte[]>> result = new ArrayList<>();
-    long start = System.currentTimeMillis();
-    if (count < 0) {
-      throw new IllegalArgumentException(
-          "Invalid count given " + count + ", count must be greater than 0");
-    }
-    Snapshot snapShot = null;
-    DBIterator dbIter = null;
-    try {
-      snapShot = db.getSnapshot();
-      ReadOptions readOptions = new ReadOptions().snapshot(snapShot);
-      dbIter = db.iterator(readOptions);
-      if (startKey == null) {
-        dbIter.seekToFirst();
-      } else {
-        if (db.get(startKey) == null) {
-          // Key not found, return empty list
-          return result;
-        }
-        dbIter.seek(startKey);
-      }
-      while (dbIter.hasNext() && result.size() < count) {
-        byte[] preKey = dbIter.hasPrev() ? dbIter.peekPrev().getKey() : null;
-        byte[] nextKey = dbIter.hasNext() ? dbIter.peekNext().getKey() : null;
-        Entry<byte[], byte[]> current = dbIter.next();
-
-        if (filters == null) {
-          result.add(current);
-        } else {
-          if (Arrays.asList(filters).stream().allMatch(
-              entry -> entry.filterKey(preKey, current.getKey(), nextKey))) {
-            result.add(current);
-          } else {
-            if (result.size() > 0 && sequential) {
-              // if the caller asks for a sequential range of results,
-              // and we met a dis-match, abort iteration from here.
-              // if result is empty, we continue to look for the first match.
-              break;
-            }
-          }
-        }
-      }
-    } finally {
-      if (snapShot != null) {
-        snapShot.close();
-      }
-      if (dbIter != null) {
-        dbIter.close();
-      }
-      if (LOG.isDebugEnabled()) {
-        if (filters != null) {
-          for (MetadataKeyFilters.MetadataKeyFilter filter : filters) {
-            int scanned = filter.getKeysScannedNum();
-            int hinted = filter.getKeysHintedNum();
-            if (scanned > 0 || hinted > 0) {
-              LOG.debug(
-                  "getRangeKVs ({}) numOfKeysScanned={}, numOfKeysHinted={}",
-                  filter.getClass().getSimpleName(), filter.getKeysScannedNum(),
-                  filter.getKeysHintedNum());
-            }
-          }
-        }
-        long end = System.currentTimeMillis();
-        long timeConsumed = end - start;
-        LOG.debug("Time consumed for getRangeKVs() is {}ms,"
-            + " result length is {}.", timeConsumed, result.size());
-      }
-    }
-    return result;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
deleted file mode 100644
index 3ff0a94..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.utils;
-
-import com.google.common.base.Strings;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.OzoneConsts;
-
-/**
- * An utility class to filter levelDB keys.
- */
-public final class MetadataKeyFilters {
-
-  private static KeyPrefixFilter deletingKeyFilter =
-      new MetadataKeyFilters.KeyPrefixFilter(OzoneConsts.DELETING_KEY_PREFIX);
-
-  private static KeyPrefixFilter normalKeyFilter =
-      new MetadataKeyFilters.KeyPrefixFilter(OzoneConsts.DELETING_KEY_PREFIX,
-          true);
-
-  private MetadataKeyFilters() {
-  }
-
-  public static KeyPrefixFilter getDeletingKeyFilter() {
-    return deletingKeyFilter;
-  }
-
-  public static KeyPrefixFilter getNormalKeyFilter() {
-    return normalKeyFilter;
-  }
-  /**
-   * Interface for levelDB key filters.
-   */
-  public interface MetadataKeyFilter {
-    /**
-     * Filter levelDB key with a certain condition.
-     *
-     * @param preKey     previous key.
-     * @param currentKey current key.
-     * @param nextKey    next key.
-     * @return true if a certain condition satisfied, return false otherwise.
-     */
-    boolean filterKey(byte[] preKey, byte[] currentKey, byte[] nextKey);
-
-    default int getKeysScannedNum() {
-      return 0;
-    }
-
-    default int getKeysHintedNum() {
-      return 0;
-    }
-  }
-
-  /**
-   * Utility class to filter key by a string prefix. This filter
-   * assumes keys can be parsed to a string.
-   */
-  public static class KeyPrefixFilter implements MetadataKeyFilter {
-
-    private String keyPrefix = null;
-    private int keysScanned = 0;
-    private int keysHinted = 0;
-    private Boolean negative;
-
-    public KeyPrefixFilter(String keyPrefix) {
-      this(keyPrefix, false);
-    }
-
-    public KeyPrefixFilter(String keyPrefix, boolean negative) {
-      this.keyPrefix = keyPrefix;
-      this.negative = negative;
-    }
-
-    @Override
-    public boolean filterKey(byte[] preKey, byte[] currentKey,
-        byte[] nextKey) {
-      keysScanned++;
-      boolean accept = false;
-      if (Strings.isNullOrEmpty(keyPrefix)) {
-        accept = true;
-      } else {
-        if (currentKey != null &&
-            DFSUtil.bytes2String(currentKey).startsWith(keyPrefix)) {
-          keysHinted++;
-          accept = true;
-        } else {
-          accept = false;
-        }
-      }
-      return (negative) ? !accept : accept;
-    }
-
-    @Override
-    public int getKeysScannedNum() {
-      return keysScanned;
-    }
-
-    @Override
-    public int getKeysHintedNum() {
-      return keysHinted;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataStore.java
deleted file mode 100644
index b90b08f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataStore.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.utils;
-
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Interface for key-value store that stores ozone metadata.
- * Ozone metadata is stored as key value pairs, both key and value
- * are arbitrary byte arrays.
- */
-@InterfaceStability.Evolving
-public interface MetadataStore extends Closeable{
-
-  /**
-   * Puts a key-value pair into the store.
-   *
-   * @param key metadata key
-   * @param value metadata value
-   */
-  void put(byte[] key, byte[] value) throws IOException;
-
-  /**
-   * @return true if the metadata store is empty.
-   *
-   * @throws IOException
-   */
-  boolean isEmpty() throws IOException;
-
-  /**
-   * Returns the value mapped to the given key in byte array.
-   *
-   * @param key metadata key
-   * @return value in byte array
-   * @throws IOException
-   */
-  byte[] get(byte[] key) throws IOException;
-
-  /**
-   * Deletes a key from the metadata store.
-   *
-   * @param key metadata key
-   * @throws IOException
-   */
-  void delete(byte[] key) throws IOException;
-
-  /**
-   * Returns a certain range of key value pairs as a list based on a
-   * startKey or count. Further a {@link MetadataKeyFilter} can be added to
-   * filter keys if necessary. To prevent race conditions while listing
-   * entries, this implementation takes a snapshot and lists the entries from
-   * the snapshot. This may, on the other hand, cause the range result slight
-   * different with actual data if data is updating concurrently.
-   * <p>
-   * If the startKey is specified and found in levelDB, this key and the keys
-   * after this key will be included in the result. If the startKey is null
-   * all entries will be included as long as other conditions are satisfied.
-   * If the given startKey doesn't exist and empty list will be returned.
-   * <p>
-   * The count argument is to limit number of total entries to return,
-   * the value for count must be an integer greater than 0.
-   * <p>
-   * This method allows to specify one or more {@link MetadataKeyFilter}
-   * to filter keys by certain condition. Once given, only the entries
-   * whose key passes all the filters will be included in the result.
-   *
-   * @param startKey a start key.
-   * @param count max number of entries to return.
-   * @param filters customized one or more {@link MetadataKeyFilter}.
-   * @return a list of entries found in the database or an empty list if the
-   * startKey is invalid.
-   * @throws IOException if there are I/O errors.
-   * @throws IllegalArgumentException if count is less than 0.
-   */
-  List<Map.Entry<byte[], byte[]>> getRangeKVs(byte[] startKey,
-      int count, MetadataKeyFilter... filters)
-      throws IOException, IllegalArgumentException;
-
-  /**
-   * This method is very similar to {@link #getRangeKVs}, the only
-   * different is this method is supposed to return a sequential range
-   * of elements based on the filters. While iterating the elements,
-   * if it met any entry that cannot pass the filter, the iterator will stop
-   * from this point without looking for next match. If no filter is given,
-   * this method behaves just like {@link #getRangeKVs}.
-   *
-   * @param startKey a start key.
-   * @param count max number of entries to return.
-   * @param filters customized one or more {@link MetadataKeyFilter}.
-   * @return a list of entries found in the database.
-   * @throws IOException
-   * @throws IllegalArgumentException
-   */
-  List<Map.Entry<byte[], byte[]>> getSequentialRangeKVs(byte[] startKey,
-      int count, MetadataKeyFilter... filters)
-      throws IOException, IllegalArgumentException;
-
-  /**
-   * A batch of PUT, DELETE operations handled as a single atomic write.
-   *
-   * @throws IOException write fails
-   */
-  void writeBatch(BatchOperation operation) throws IOException;
-
-  /**
-   * Compact the entire database.
-   * @throws IOException
-   */
-  void compactDB() throws IOException;
-
-  /**
-   * Destroy the content of the specified database,
-   * a destroyed database will not be able to load again.
-   * Be very careful with this method.
-   *
-   * @throws IOException if I/O error happens
-   */
-  void destroy() throws IOException;
-
-  /**
-   * Seek the database to a certain key, returns the key-value
-   * pairs around this key based on the given offset. Note, this method
-   * can only support offset -1 (left), 0 (current) and 1 (right),
-   * any other offset given will cause a {@link IllegalArgumentException}.
-   *
-   * @param offset offset to the key
-   * @param from from which key
-   * @return a key-value pair
-   * @throws IOException
-   */
-  ImmutablePair<byte[], byte[]> peekAround(int offset, byte[] from)
-      throws IOException, IllegalArgumentException;
-
-  /**
-   * Iterates entries in the database from a certain key.
-   * Applies the given {@link EntryConsumer} to the key and value of
-   * each entry, the function produces a boolean result which is used
-   * as the criteria to exit from iteration.
-   *
-   * @param from the start key
-   * @param consumer
-   *   a {@link EntryConsumer} applied to each key and value. If the consumer
-   *   returns true, continues the iteration to next entry; otherwise exits
-   *   the iteration.
-   * @throws IOException
-   */
-  void iterate(byte[] from, EntryConsumer consumer)
-      throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java
deleted file mode 100644
index 095e718..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.utils;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_METADATA_STORE_ROCKSDB_STATISTICS;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF;
-import org.iq80.leveldb.Options;
-import org.rocksdb.BlockBasedTableConfig;
-
-import java.io.File;
-import java.io.IOException;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB;
-
-import org.rocksdb.Statistics;
-import org.rocksdb.StatsLevel;
-
-/**
- * Builder for metadata store.
- */
-public class MetadataStoreBuilder {
-
-  private File dbFile;
-  private long cacheSize;
-  private boolean createIfMissing = true;
-  private Configuration conf;
-
-  public static MetadataStoreBuilder newBuilder() {
-    return new MetadataStoreBuilder();
-  }
-
-  public MetadataStoreBuilder setDbFile(File dbPath) {
-    this.dbFile = dbPath;
-    return this;
-  }
-
-  public MetadataStoreBuilder setCacheSize(long cache) {
-    this.cacheSize = cache;
-    return this;
-  }
-
-  public MetadataStoreBuilder setCreateIfMissing(boolean doCreate) {
-    this.createIfMissing = doCreate;
-    return this;
-  }
-
-  public MetadataStoreBuilder setConf(Configuration configuration) {
-    this.conf = configuration;
-    return this;
-  }
-
-  public MetadataStore build() throws IOException {
-    if (dbFile == null) {
-      throw new IllegalArgumentException("Failed to build metadata store, "
-          + "dbFile is required but not found");
-    }
-
-    // Build db store based on configuration
-    MetadataStore store = null;
-    String impl = conf == null ?
-        OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_DEFAULT :
-        conf.getTrimmed(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
-            OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_DEFAULT);
-    if (OZONE_METADATA_STORE_IMPL_LEVELDB.equals(impl)) {
-      Options options = new Options();
-      options.createIfMissing(createIfMissing);
-      if (cacheSize > 0) {
-        options.cacheSize(cacheSize);
-      }
-      store = new LevelDBStore(dbFile, options);
-    } else if (OZONE_METADATA_STORE_IMPL_ROCKSDB.equals(impl)) {
-      org.rocksdb.Options opts = new org.rocksdb.Options();
-      opts.setCreateIfMissing(createIfMissing);
-
-      if (cacheSize > 0) {
-        BlockBasedTableConfig tableConfig = new BlockBasedTableConfig();
-        tableConfig.setBlockCacheSize(cacheSize);
-        opts.setTableFormatConfig(tableConfig);
-      }
-
-      String rocksDbStat = conf == null ?
-          OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT :
-          conf.getTrimmed(OZONE_METADATA_STORE_ROCKSDB_STATISTICS,
-              OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT);
-
-      if (!rocksDbStat.equals(OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF)) {
-        Statistics statistics = new Statistics();
-        statistics.setStatsLevel(StatsLevel.valueOf(rocksDbStat));
-        opts = opts.setStatistics(statistics);
-
-      }
-      store = new RocksDBStore(dbFile, opts);
-    } else {
-      throw new IllegalArgumentException("Invalid argument for "
-          + OzoneConfigKeys.OZONE_METADATA_STORE_IMPL
-          + ". Expecting " + OZONE_METADATA_STORE_IMPL_LEVELDB
-          + " or " + OZONE_METADATA_STORE_IMPL_ROCKSDB
-          + ", but met " + impl);
-    }
-    return store;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
deleted file mode 100644
index 2f340a5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
+++ /dev/null
@@ -1,382 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.ratis.shaded.com.google.common.annotations.VisibleForTesting;
-import org.rocksdb.RocksIterator;
-import org.rocksdb.Options;
-import org.rocksdb.WriteOptions;
-import org.rocksdb.RocksDB;
-import org.rocksdb.RocksDBException;
-import org.rocksdb.WriteBatch;
-import org.rocksdb.DbPath;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.ObjectName;
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.AbstractMap;
-
-/**
- * RocksDB implementation of ozone metadata store.
- */
-public class RocksDBStore implements MetadataStore {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(RocksDBStore.class);
-
-  private RocksDB db = null;
-  private File dbLocation;
-  private WriteOptions writeOptions;
-  private Options dbOptions;
-  private ObjectName statMBeanName;
-
-  public RocksDBStore(File dbFile, Options options)
-      throws IOException {
-    Preconditions.checkNotNull(dbFile, "DB file location cannot be null");
-    RocksDB.loadLibrary();
-    dbOptions = options;
-    dbLocation = dbFile;
-    writeOptions = new WriteOptions();
-    try {
-
-      db = RocksDB.open(dbOptions, dbLocation.getAbsolutePath());
-      if (dbOptions.statistics() != null) {
-
-        Map<String, String> jmxProperties = new HashMap<String, String>();
-        jmxProperties.put("dbName", dbFile.getName());
-        statMBeanName = MBeans.register("Ozone", "RocksDbStore", jmxProperties,
-            new RocksDBStoreMBean(dbOptions.statistics()));
-        if (statMBeanName == null) {
-          LOG.warn("jmx registration failed during RocksDB init, db path :{}",
-              dbFile.getAbsolutePath());
-        }
-      }
-    } catch (RocksDBException e) {
-      throw new IOException(
-          "Failed init RocksDB, db path : " + dbFile.getAbsolutePath(), e);
-    }
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("RocksDB successfully opened.");
-      LOG.debug("[Option] dbLocation= {}", dbLocation.getAbsolutePath());
-      LOG.debug("[Option] createIfMissing = {}", options.createIfMissing());
-      LOG.debug("[Option] compactionPriority= {}", options.compactionStyle());
-      LOG.debug("[Option] compressionType= {}", options.compressionType());
-      LOG.debug("[Option] maxOpenFiles= {}", options.maxOpenFiles());
-      LOG.debug("[Option] writeBufferSize= {}", options.writeBufferSize());
-    }
-  }
-
-  private IOException toIOException(String msg, RocksDBException e) {
-    String statusCode = e.getStatus() == null ? "N/A" :
-        e.getStatus().getCodeString();
-    String errMessage = e.getMessage() == null ? "Unknown error" :
-        e.getMessage();
-    String output = msg + "; status : " + statusCode
-        + "; message : " + errMessage;
-    return new IOException(output, e);
-  }
-
-  @Override
-  public void put(byte[] key, byte[] value) throws IOException {
-    try {
-      db.put(writeOptions, key, value);
-    } catch (RocksDBException e) {
-      throw toIOException("Failed to put key-value to metadata store", e);
-    }
-  }
-
-  @Override
-  public boolean isEmpty() throws IOException {
-    RocksIterator it = null;
-    try {
-      it = db.newIterator();
-      it.seekToFirst();
-      return !it.isValid();
-    } finally {
-      if (it != null) {
-        it.close();
-      }
-    }
-  }
-
-  @Override
-  public byte[] get(byte[] key) throws IOException {
-    try {
-      return db.get(key);
-    } catch (RocksDBException e) {
-      throw toIOException("Failed to get the value for the given key", e);
-    }
-  }
-
-  @Override
-  public void delete(byte[] key) throws IOException {
-    try {
-      db.delete(key);
-    } catch (RocksDBException e) {
-      throw toIOException("Failed to delete the given key", e);
-    }
-  }
-
-  @Override
-  public List<Map.Entry<byte[], byte[]>> getRangeKVs(byte[] startKey,
-      int count, MetadataKeyFilters.MetadataKeyFilter... filters)
-      throws IOException, IllegalArgumentException {
-    return getRangeKVs(startKey, count, false, filters);
-  }
-
-  @Override
-  public List<Map.Entry<byte[], byte[]>> getSequentialRangeKVs(byte[] startKey,
-      int count, MetadataKeyFilters.MetadataKeyFilter... filters)
-      throws IOException, IllegalArgumentException {
-    return getRangeKVs(startKey, count, true, filters);
-  }
-
-  private List<Map.Entry<byte[], byte[]>> getRangeKVs(byte[] startKey,
-      int count, boolean sequential,
-      MetadataKeyFilters.MetadataKeyFilter... filters)
-      throws IOException, IllegalArgumentException {
-    List<Map.Entry<byte[], byte[]>> result = new ArrayList<>();
-    long start = System.currentTimeMillis();
-    if (count < 0) {
-      throw new IllegalArgumentException(
-          "Invalid count given " + count + ", count must be greater than 0");
-    }
-    RocksIterator it = null;
-    try {
-      it = db.newIterator();
-      if (startKey == null) {
-        it.seekToFirst();
-      } else {
-        if(get(startKey) == null) {
-          // Key not found, return empty list
-          return result;
-        }
-        it.seek(startKey);
-      }
-      while(it.isValid() && result.size() < count) {
-        byte[] currentKey = it.key();
-        byte[] currentValue = it.value();
-
-        it.prev();
-        final byte[] prevKey = it.isValid() ? it.key() : null;
-
-        it.seek(currentKey);
-        it.next();
-        final byte[] nextKey = it.isValid() ? it.key() : null;
-
-        if (filters == null) {
-          result.add(new AbstractMap.SimpleImmutableEntry<>(currentKey,
-              currentValue));
-        } else {
-          if (Arrays.asList(filters).stream()
-              .allMatch(entry -> entry.filterKey(prevKey,
-                  currentKey, nextKey))) {
-            result.add(new AbstractMap.SimpleImmutableEntry<>(currentKey,
-                currentValue));
-          } else {
-            if (result.size() > 0 && sequential) {
-              // if the caller asks for a sequential range of results,
-              // and we met a dis-match, abort iteration from here.
-              // if result is empty, we continue to look for the first match.
-              break;
-            }
-          }
-        }
-      }
-    } finally {
-      if (it != null) {
-        it.close();
-      }
-      long end = System.currentTimeMillis();
-      long timeConsumed = end - start;
-      if (LOG.isDebugEnabled()) {
-        if (filters != null) {
-          for (MetadataKeyFilters.MetadataKeyFilter filter : filters) {
-            int scanned = filter.getKeysScannedNum();
-            int hinted = filter.getKeysHintedNum();
-            if (scanned > 0 || hinted > 0) {
-              LOG.debug(
-                  "getRangeKVs ({}) numOfKeysScanned={}, numOfKeysHinted={}",
-                  filter.getClass().getSimpleName(), filter.getKeysScannedNum(),
-                  filter.getKeysHintedNum());
-            }
-          }
-        }
-        LOG.debug("Time consumed for getRangeKVs() is {}ms,"
-            + " result length is {}.", timeConsumed, result.size());
-      }
-    }
-    return result;
-  }
-
-  @Override
-  public void writeBatch(BatchOperation operation)
-      throws IOException {
-    List<BatchOperation.SingleOperation> operations =
-        operation.getOperations();
-    if (!operations.isEmpty()) {
-      try (WriteBatch writeBatch = new WriteBatch()) {
-        for (BatchOperation.SingleOperation opt : operations) {
-          switch (opt.getOpt()) {
-          case DELETE:
-            writeBatch.remove(opt.getKey());
-            break;
-          case PUT:
-            writeBatch.put(opt.getKey(), opt.getValue());
-            break;
-          default:
-            throw new IllegalArgumentException("Invalid operation "
-                + opt.getOpt());
-          }
-        }
-        db.write(writeOptions, writeBatch);
-      } catch (RocksDBException e) {
-        throw toIOException("Batch write operation failed", e);
-      }
-    }
-  }
-
-  @Override
-  public void compactDB() throws IOException {
-    if (db != null) {
-      try {
-        db.compactRange();
-      } catch (RocksDBException e) {
-        throw toIOException("Failed to compact db", e);
-      }
-    }
-  }
-
-  private void deleteQuietly(File fileOrDir) {
-    if (fileOrDir != null && fileOrDir.exists()) {
-      try {
-        FileUtils.forceDelete(fileOrDir);
-      } catch (IOException e) {
-        LOG.warn("Failed to delete dir {}", fileOrDir.getAbsolutePath(), e);
-      }
-    }
-  }
-
-  @Override
-  public void destroy() throws IOException {
-    // Make sure db is closed.
-    close();
-
-    // There is no destroydb java API available,
-    // equivalently we can delete all db directories.
-    deleteQuietly(dbLocation);
-    deleteQuietly(new File(dbOptions.dbLogDir()));
-    deleteQuietly(new File(dbOptions.walDir()));
-    List<DbPath> dbPaths = dbOptions.dbPaths();
-    if (dbPaths != null) {
-      dbPaths.forEach(dbPath -> {
-        deleteQuietly(new File(dbPath.toString()));
-      });
-    }
-  }
-
-  @Override
-  public ImmutablePair<byte[], byte[]> peekAround(int offset,
-      byte[] from) throws IOException, IllegalArgumentException {
-    RocksIterator it = null;
-    try {
-      it = db.newIterator();
-      if (from == null) {
-        it.seekToFirst();
-      } else {
-        it.seek(from);
-      }
-      if (!it.isValid()) {
-        return null;
-      }
-
-      switch (offset) {
-      case 0:
-        break;
-      case 1:
-        it.next();
-        break;
-      case -1:
-        it.prev();
-        break;
-      default:
-        throw new IllegalArgumentException(
-            "Position can only be -1, 0 " + "or 1, but found " + offset);
-      }
-      return it.isValid() ? new ImmutablePair<>(it.key(), it.value()) : null;
-    } finally {
-      if (it != null) {
-        it.close();
-      }
-    }
-  }
-
-  @Override
-  public void iterate(byte[] from, EntryConsumer consumer)
-      throws IOException {
-    RocksIterator it = null;
-    try {
-      it = db.newIterator();
-      if (from != null) {
-        it.seek(from);
-      } else {
-        it.seekToFirst();
-      }
-      while (it.isValid()) {
-        if (!consumer.consume(it.key(), it.value())) {
-          break;
-        }
-        it.next();
-      }
-    } finally {
-      if (it != null) {
-        it.close();
-      }
-    }
-  }
-
-  @Override
-  public void close() throws IOException {
-    if (statMBeanName != null) {
-      MBeans.unregister(statMBeanName);
-    }
-    if (db != null) {
-      db.close();
-    }
-
-  }
-
-  @VisibleForTesting
-  protected ObjectName getStatMBeanName() {
-    return statMBeanName;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStoreMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStoreMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStoreMBean.java
deleted file mode 100644
index df0f41c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStoreMBean.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.utils;
-
-import org.rocksdb.HistogramData;
-import org.rocksdb.HistogramType;
-import org.rocksdb.Statistics;
-import org.rocksdb.TickerType;
-
-import javax.management.Attribute;
-import javax.management.AttributeList;
-import javax.management.AttributeNotFoundException;
-import javax.management.DynamicMBean;
-import javax.management.InvalidAttributeValueException;
-import javax.management.MBeanAttributeInfo;
-import javax.management.MBeanException;
-import javax.management.MBeanInfo;
-import javax.management.ReflectionException;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Stream;
-
-/**
- * Adapter JMX bean to publish all the Rocksdb metrics.
- */
-public class RocksDBStoreMBean implements DynamicMBean {
-
-  private Statistics statistics;
-
-  private Set<String> histogramAttributes = new HashSet<>();
-
-  public RocksDBStoreMBean(Statistics statistics) {
-    this.statistics = statistics;
-    histogramAttributes.add("Average");
-    histogramAttributes.add("Median");
-    histogramAttributes.add("Percentile95");
-    histogramAttributes.add("Percentile99");
-    histogramAttributes.add("StandardDeviation");
-  }
-
-  @Override
-  public Object getAttribute(String attribute)
-      throws AttributeNotFoundException, MBeanException, ReflectionException {
-    for (String histogramAttribute : histogramAttributes) {
-      if (attribute.endsWith("_" + histogramAttribute.toUpperCase())) {
-        String keyName = attribute
-            .substring(0, attribute.length() - histogramAttribute.length() -1);
-        try {
-          HistogramData histogram =
-              statistics.getHistogramData(HistogramType.valueOf(keyName));
-          try {
-            Method method =
-                HistogramData.class.getMethod("get" + histogramAttribute);
-            return method.invoke(histogram);
-          } catch (Exception e) {
-            throw new ReflectionException(e,
-                "Can't read attribute " + attribute);
-          }
-        } catch (IllegalArgumentException exception) {
-          throw new AttributeNotFoundException(
-              "No such attribute in RocksDB stats: " + attribute);
-        }
-      }
-    }
-    try {
-      return statistics.getTickerCount(TickerType.valueOf(attribute));
-    } catch (IllegalArgumentException ex) {
-      throw new AttributeNotFoundException(
-          "No such attribute in RocksDB stats: " + attribute);
-    }
-  }
-
-  @Override
-  public void setAttribute(Attribute attribute)
-      throws AttributeNotFoundException, InvalidAttributeValueException,
-      MBeanException, ReflectionException {
-
-  }
-
-  @Override
-  public AttributeList getAttributes(String[] attributes) {
-    AttributeList result = new AttributeList();
-    for (String attributeName : attributes) {
-      try {
-        Object value = getAttribute(attributeName);
-        result.add(value);
-      } catch (Exception e) {
-        //TODO
-      }
-    }
-    return result;
-  }
-
-  @Override
-  public AttributeList setAttributes(AttributeList attributes) {
-    return null;
-  }
-
-  @Override
-  public Object invoke(String actionName, Object[] params, String[] signature)
-      throws MBeanException, ReflectionException {
-    return null;
-  }
-
-  @Override
-  public MBeanInfo getMBeanInfo() {
-
-    List<MBeanAttributeInfo> attributes = new ArrayList<>();
-    for (TickerType tickerType : TickerType.values()) {
-      attributes.add(new MBeanAttributeInfo(tickerType.name(), "long",
-          "RocksDBStat: " + tickerType.name(), true, false, false));
-    }
-    for (HistogramType histogramType : HistogramType.values()) {
-      for (String histogramAttribute : histogramAttributes) {
-        attributes.add(new MBeanAttributeInfo(
-            histogramType.name() + "_" + histogramAttribute.toUpperCase(),
-            "long", "RocksDBStat: " + histogramType.name(), true, false,
-            false));
-      }
-    }
-
-    return new MBeanInfo("", "RocksDBStat",
-        attributes.toArray(new MBeanAttributeInfo[0]), null, null, null);
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/package-info.java
deleted file mode 100644
index 4466337..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.utils;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto
deleted file mode 100644
index 5e3e0c7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are private and unstable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
- * for what changes are allowed for a *unstable* .proto interface.
- */
-option java_package = "org.apache.hadoop.cblock.protocol.proto";
-option java_outer_classname = "CBlockClientServerProtocolProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.cblock;
-
-import "Ozone.proto";
-import "CBlockServiceProtocol.proto";
-/**
-* This message is sent from CBlock client side to CBlock server to
-* mount a volume specified by owner name and volume name.
-*
-* Right now, this is the only communication between client and server.
-* After the volume is mounted, CBlock client will talk to containers
-* by itself, nothing to do with CBlock server.
-*/
-message MountVolumeRequestProto {
-    required string userName = 1;
-    required string volumeName = 2;
-}
-
-/**
-* This message is sent from CBlock server to CBlock client as response
-* of mount a volume. It checks the whether the volume is valid to access
-* at all.(e.g. volume exist)
-*
-* And include enough information (volume size, block size, list of
-* containers for this volume) for client side to perform read/write on
-* the volume.
-*/
-message MountVolumeResponseProto {
-    required bool isValid = 1;
-    optional string userName = 2;
-    optional string volumeName = 3;
-    optional uint64 volumeSize = 4;
-    optional uint32 blockSize = 5;
-    repeated ContainerIDProto allContainerIDs = 6;
-}
-
-/**
-* This message include ID of container which can be used to locate the
-* container. Since the order of containers needs to be maintained, also
-* includes a index field to verify the correctness of the order.
-*/
-message ContainerIDProto {
-    required string containerID = 1;
-    required uint64 index = 2;
-    // making pipeline optional to be compatible with exisiting tests
-    optional hadoop.hdfs.ozone.Pipeline pipeline = 3;
-}
-
-
-message ListVolumesRequestProto {
-
-}
-
-message ListVolumesResponseProto {
-    repeated VolumeInfoProto volumeEntry = 1;
-}
-
-
-service CBlockClientServerProtocolService {
-    /**
-    * mount the volume.
-    */
-    rpc mountVolume(MountVolumeRequestProto) returns (MountVolumeResponseProto);
-
-    rpc listVolumes(ListVolumesRequestProto) returns(ListVolumesResponseProto);
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockServiceProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockServiceProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockServiceProtocol.proto
deleted file mode 100644
index 36e4b59..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockServiceProtocol.proto
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are private and unstable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
- * for what changes are allowed for a *unstable* .proto interface.
- */
-
-option java_package = "org.apache.hadoop.cblock.protocol.proto";
-option java_outer_classname = "CBlockServiceProtocolProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.cblock;
-
-/**
-* This message is sent to CBlock server to create a volume. Creating
-* volume requries four parameters: owner of the volume, name of the volume
-* size of volume and block size of the volume.
-*/
-message CreateVolumeRequestProto {
-    required string userName = 1;
-    required string volumeName = 2;
-    required uint64 volumeSize = 3;
-    optional uint32 blockSize = 4 [default = 4096];
-}
-
-/**
-* Empty response message.
-*/
-message CreateVolumeResponseProto {
-
-}
-
-/**
-* This message is sent to CBlock server to delete a volume. The volume
-* is specified by owner name and volume name. If force is set to
-* false, volume will be deleted only if it is empty. Otherwise delete it
-* regardless.
-*/
-message DeleteVolumeRequestProto {
-    required string userName = 1;
-    required string volumeName = 2;
-    optional bool force = 3;
-}
-
-/**
-* Empty response message.
-*/
-message DeleteVolumeResponseProto {
-
-}
-
-/**
-* This message is sent to CBlock server to request info of a volume. The
-* volume is specified by owner name and volume name.
-*/
-message InfoVolumeRequestProto {
-    required string userName = 1;
-    required string volumeName = 2;
-}
-
-/**
-* This message describes the information of a volume.
-* Currently, the info includes the volume creation parameters and a number
-* as the usage of the volume, in terms of number of bytes.
-*/
-message VolumeInfoProto {
-    required string userName = 1;
-    required string volumeName = 2;
-    required uint64 volumeSize = 3;
-    required uint64 blockSize = 4;
-    optional uint64 usage = 5;
-    // TODO : potentially volume ACL
-}
-
-/**
-* This message is sent from CBlock server as response of info volume request.
-*/
-message InfoVolumeResponseProto {
-    optional VolumeInfoProto volumeInfo = 1;
-}
-
-/**
-* This message is sent to CBlock server to list all available volume.
-*/
-message ListVolumeRequestProto {
-    optional string userName = 1;
-}
-
-/**
-* This message is sent from CBlock server as response of volume listing.
-*/
-message ListVolumeResponseProto {
-    repeated VolumeInfoProto volumeEntry = 1;
-}
-
-service CBlockServiceProtocolService {
-    /**
-    * Create a volume.
-    */
-    rpc createVolume(CreateVolumeRequestProto) returns(CreateVolumeResponseProto);
-
-    /**
-    * Delete a volume.
-    */
-    rpc deleteVolume(DeleteVolumeRequestProto) returns(DeleteVolumeResponseProto);
-
-    /**
-    * Get info of a volume.
-    */
-    rpc infoVolume(InfoVolumeRequestProto) returns(InfoVolumeResponseProto);
-
-    /**
-    * List all available volumes.
-    */
-    rpc listVolume(ListVolumeRequestProto) returns(ListVolumeResponseProto);
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[33/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
index 35906c6..0af3c4f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
@@ -189,10 +189,8 @@ public interface HdfsServerConstants {
         return NamenodeRole.NAMENODE;
       }
     }
-
+    
     public void setClusterId(String cid) {
-      Preconditions.checkState(this == UPGRADE || this == UPGRADEONLY
-          || this == FORMAT);
       clusterId = cid;
     }
 
@@ -217,7 +215,6 @@ public interface HdfsServerConstants {
     }
 
     public void setForce(int force) {
-      Preconditions.checkState(this == RECOVER);
       this.force = force;
     }
     
@@ -230,7 +227,6 @@ public interface HdfsServerConstants {
     }
     
     public void setForceFormat(boolean force) {
-      Preconditions.checkState(this == FORMAT);
       isForceFormat = force;
     }
     
@@ -239,7 +235,6 @@ public interface HdfsServerConstants {
     }
     
     public void setInteractiveFormat(boolean interactive) {
-      Preconditions.checkState(this == FORMAT);
       isInteractiveFormat = interactive;
     }
     

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 1f51935..b757fd7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -48,6 +48,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_METRICS_LOGGER_P
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_METRICS_LOGGER_PERIOD_SECONDS_KEY;
 import static org.apache.hadoop.util.ExitUtil.terminate;
 
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.hdfs.protocol.proto.ReconfigurationProtocolProtos.ReconfigurationProtocolService;
 
@@ -109,10 +110,8 @@ import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.DFSUtilClient;
 import org.apache.hadoop.hdfs.HDFSPolicyProvider;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
 import org.apache.hadoop.hdfs.server.datanode.checker.DatasetVolumeChecker;
 import org.apache.hadoop.hdfs.server.datanode.checker.StorageLocationChecker;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
 import org.apache.hadoop.util.AutoCloseableLock;
 import org.apache.hadoop.hdfs.client.BlockReportOptions;
 import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@@ -191,7 +190,6 @@ import org.apache.hadoop.metrics2.util.MBeans;
 import org.apache.hadoop.net.DNS;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.net.unix.DomainSocket;
-import org.apache.hadoop.conf.OzoneConfiguration;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.SaslPropertiesResolver;
 import org.apache.hadoop.security.SecurityUtil;
@@ -373,7 +371,6 @@ public class DataNode extends ReconfigurableBase
   private final String confVersion;
   private final long maxNumberOfBlocksToLog;
   private final boolean pipelineSupportECN;
-  private final boolean ozoneEnabled;
 
   private final List<String> usersWithLocalPathAccess;
   private final boolean connectToDnViaHostname;
@@ -402,8 +399,6 @@ public class DataNode extends ReconfigurableBase
 
   private final SocketFactory socketFactory;
 
-  private DatanodeStateMachine datanodeStateMachine;
-
   private static Tracer createTracer(Configuration conf) {
     return new Tracer.Builder("DataNode").
         conf(TraceUtils.wrapHadoopConf(DATANODE_HTRACE_PREFIX , conf)).
@@ -414,8 +409,6 @@ public class DataNode extends ReconfigurableBase
 
   private ScheduledThreadPoolExecutor metricsLoggerTimer;
 
-  private ObjectStoreHandler objectStoreHandler = null;
-
   /**
    * Creates a dummy DataNode for testing purpose.
    */
@@ -434,7 +427,6 @@ public class DataNode extends ReconfigurableBase
     this.connectToDnViaHostname = false;
     this.blockScanner = new BlockScanner(this, this.getConf());
     this.pipelineSupportECN = false;
-    this.ozoneEnabled = false;
     this.socketFactory = NetUtils.getDefaultSocketFactory(conf);
     this.dnConf = new DNConf(this);
     initOOBTimeout();
@@ -474,8 +466,6 @@ public class DataNode extends ReconfigurableBase
         DFSConfigKeys.DFS_PIPELINE_ECN_ENABLED,
         DFSConfigKeys.DFS_PIPELINE_ECN_ENABLED_DEFAULT);
 
-    this.ozoneEnabled = DFSUtil.isOzoneEnabled(conf);
-
     confVersion = "core-" +
         conf.get("hadoop.common.configuration.version", "UNSPECIFIED") +
         ",hdfs-" +
@@ -531,7 +521,7 @@ public class DataNode extends ReconfigurableBase
 
   @Override  // ReconfigurableBase
   protected Configuration getNewConf() {
-    return new OzoneConfiguration();
+    return new HdfsConfiguration();
   }
 
   /**
@@ -961,8 +951,8 @@ public class DataNode extends ReconfigurableBase
     // the DN is started by JSVC, pass it along.
     ServerSocketChannel httpServerChannel = secureResources != null ?
         secureResources.getHttpServerChannel() : null;
-    this.httpServer = new DatanodeHttpServer(getConf(), this, httpServerChannel,
-        this.objectStoreHandler);
+
+    httpServer = new DatanodeHttpServer(getConf(), this, httpServerChannel);
     httpServer.start();
     if (httpServer.getHttpAddress() != null) {
       infoPort = httpServer.getHttpAddress().getPort();
@@ -1411,9 +1401,7 @@ public class DataNode extends ReconfigurableBase
     
     // global DN settings
     registerMXBean();
-
     initDataXceiver();
-    initObjectStoreHandler();
     startInfoServer();
     pauseMonitor = new JvmPauseMonitor();
     pauseMonitor.init(getConf());
@@ -1454,19 +1442,6 @@ public class DataNode extends ReconfigurableBase
   }
 
   /**
-   * Initializes the object store handler.  This must be called before
-   * initialization of the HTTP server.
-   *
-   * @throws IOException if there is an I/O error
-   */
-  private void initObjectStoreHandler() throws IOException {
-    if (this.ozoneEnabled) {
-      this.objectStoreHandler = new ObjectStoreHandler(getConf());
-      LOG.info("ozone is enabled.");
-    }
-  }
-
-  /**
    * Checks if the DataNode has a secure configuration if security is enabled.
    * There are 2 possible configurations that are considered secure:
    * 1. The server has bound to privileged ports for RPC and HTTP via
@@ -1578,6 +1553,11 @@ public class DataNode extends ReconfigurableBase
         streamingAddr.getAddress().getHostAddress(), hostName, 
         storage.getDatanodeUuid(), getXferPort(), getInfoPort(),
             infoSecurePort, getIpcPort());
+    for (ServicePlugin plugin : plugins) {
+      if (plugin instanceof DataNodeServicePlugin) {
+        ((DataNodeServicePlugin) plugin).onDatanodeIdCreation(dnId);
+      }
+    }
     return new DatanodeRegistration(dnId, storageInfo, 
         new ExportedBlockKeys(), VersionInfo.getVersion());
   }
@@ -1598,31 +1578,15 @@ public class DataNode extends ReconfigurableBase
           + ". Expecting " + storage.getDatanodeUuid());
     }
 
-    if (isOzoneEnabled()) {
-      if (datanodeStateMachine == null) {
-        datanodeStateMachine = new DatanodeStateMachine(
-            getDatanodeId(),
-            getConf());
-        datanodeStateMachine.startDaemon();
+    for (ServicePlugin plugin : plugins) {
+      if (plugin instanceof DataNodeServicePlugin) {
+        ((DataNodeServicePlugin) plugin)
+            .onDatanodeSuccessfulNamenodeRegisration(bpRegistration);
       }
     }
     registerBlockPoolWithSecretManager(bpRegistration, blockPoolId);
   }
-
-  @VisibleForTesting
-  public OzoneContainer getOzoneContainerManager() {
-    return this.datanodeStateMachine.getContainer();
-  }
-
-  @VisibleForTesting
-  public DatanodeStateMachine.DatanodeStates getOzoneStateMachineState() {
-    if (this.datanodeStateMachine != null) {
-      return this.datanodeStateMachine.getContext().getState();
-    }
-    // if the state machine doesn't exist then DN initialization is in progress
-    return DatanodeStateMachine.DatanodeStates.INIT;
-  }
-
+  
   /**
    * After the block pool has contacted the NN, registers that block pool
    * with the secret manager, updating it with the secrets provided by the NN.
@@ -1729,11 +1693,11 @@ public class DataNode extends ReconfigurableBase
   BPOfferService getBPOfferService(String bpid){
     return blockPoolManager.get(bpid);
   }
-
+  
   int getBpOsCount() {
     return blockPoolManager.getAllNamenodeThreads().size();
   }
-
+  
   /**
    * Initializes the {@link #data}. The initialization is done only once, when
    * handshake with the the first namenode is completed.
@@ -2021,7 +1985,7 @@ public class DataNode extends ReconfigurableBase
         }
       }
     }
-
+    
     List<BPOfferService> bposArray = (this.blockPoolManager == null)
         ? new ArrayList<BPOfferService>()
         : this.blockPoolManager.getAllNamenodeThreads();
@@ -2054,6 +2018,7 @@ public class DataNode extends ReconfigurableBase
 
     // Terminate directory scanner and block scanner
     shutdownPeriodicScanners();
+    shutdownDiskBalancer();
 
     // Stop the web server
     if (httpServer != null) {
@@ -2064,17 +2029,6 @@ public class DataNode extends ReconfigurableBase
       }
     }
 
-    // Stop the object store handler
-    if (isOzoneEnabled()) {
-      if (this.objectStoreHandler != null) {
-        this.objectStoreHandler.close();
-        if(datanodeStateMachine != null &&
-            !datanodeStateMachine.isDaemonStopped()) {
-          datanodeStateMachine.stopDaemon();
-        }
-      }
-    }
-
     volumeChecker.shutdownAndWait(1, TimeUnit.SECONDS);
 
     if (storageLocationChecker != null) {
@@ -2088,7 +2042,7 @@ public class DataNode extends ReconfigurableBase
     // shouldRun is set to false here to prevent certain threads from exiting
     // before the restart prep is done.
     this.shouldRun = false;
-
+    
     // wait reconfiguration thread, if any, to exit
     shutdownReconfigurationTask();
 
@@ -2098,8 +2052,9 @@ public class DataNode extends ReconfigurableBase
       while (true) {
         // When shutting down for restart, wait 2.5 seconds before forcing
         // termination of receiver threads.
-        if (!this.shutdownForUpgrade || (this.shutdownForUpgrade && (
-            Time.monotonicNow() - timeNotified > 1000))) {
+        if (!this.shutdownForUpgrade ||
+            (this.shutdownForUpgrade && (Time.monotonicNow() - timeNotified
+                > 1000))) {
           this.threadGroup.interrupt();
           break;
         }
@@ -2110,8 +2065,7 @@ public class DataNode extends ReconfigurableBase
         }
         try {
           Thread.sleep(sleepMs);
-        } catch (InterruptedException e) {
-        }
+        } catch (InterruptedException e) {}
         sleepMs = sleepMs * 3 / 2; // exponential backoff
         if (sleepMs > 200) {
           sleepMs = 200;
@@ -2137,9 +2091,9 @@ public class DataNode extends ReconfigurableBase
       metrics.setDataNodeActiveXceiversCount(0);
     }
 
-    // IPC server needs to be shutdown late in the process, otherwise
-    // shutdown command response won't get sent.
-    if (ipcServer != null) {
+   // IPC server needs to be shutdown late in the process, otherwise
+   // shutdown command response won't get sent.
+   if (ipcServer != null) {
       ipcServer.stop();
     }
 
@@ -2154,7 +2108,7 @@ public class DataNode extends ReconfigurableBase
         LOG.warn("Received exception in BlockPoolManager#shutDownAll", ie);
       }
     }
-
+    
     if (storage != null) {
       try {
         this.storage.unlockAll();
@@ -2175,11 +2129,9 @@ public class DataNode extends ReconfigurableBase
       MBeans.unregister(dataNodeInfoBeanName);
       dataNodeInfoBeanName = null;
     }
-    if (shortCircuitRegistry != null) {
-      shortCircuitRegistry.shutdown();
-    }
+    if (shortCircuitRegistry != null) shortCircuitRegistry.shutdown();
     LOG.info("Shutdown complete.");
-    synchronized (this) {
+    synchronized(this) {
       // it is already false, but setting it again to avoid a findbug warning.
       this.shouldRun = false;
       // Notify the main thread.
@@ -2717,9 +2669,8 @@ public class DataNode extends ReconfigurableBase
    */
   public static DataNode instantiateDataNode(String args [], Configuration conf,
       SecureResources resources) throws IOException {
-    if (conf == null) {
-      conf = new OzoneConfiguration();
-    }
+    if (conf == null)
+      conf = new HdfsConfiguration();
     
     if (args != null) {
       // parse generic hadoop options
@@ -3252,12 +3203,6 @@ public class DataNode extends ReconfigurableBase
           } catch (InterruptedException ie) { }
         }
         shutdown();
-
-        if (isOzoneEnabled()) {
-          if(datanodeStateMachine != null) {
-            datanodeStateMachine.stopDaemon();
-          }
-        }
       }
     };
 
@@ -3552,13 +3497,10 @@ public class DataNode extends ReconfigurableBase
     return metricsLoggerTimer;
   }
 
-  public boolean isOzoneEnabled() {
-    return ozoneEnabled;
-  }
-
   public Tracer getTracer() {
     return tracer;
   }
+
   /**
    * Allows submission of a disk balancer Job.
    * @param planID  - Hash value of the plan.
@@ -3676,4 +3618,14 @@ public class DataNode extends ReconfigurableBase
     }
     return volumeInfoList;
   }
+
+  @Private
+  public SecureResources getSecureResources() {
+    return secureResources;
+  }
+
+  @Private
+  public Collection<ServicePlugin> getPlugins() {
+    return Collections.unmodifiableList(plugins);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeServicePlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeServicePlugin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeServicePlugin.java
new file mode 100644
index 0000000..08aae8b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeServicePlugin.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode;
+
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.util.ServicePlugin;
+
+/**
+ * Datanode specific service plugin with additional hooks.
+ */
+public interface DataNodeServicePlugin extends ServicePlugin{
+
+  /**
+   * Extension point to modify the datanode id.
+   *
+   * @param dataNodeId
+   */
+  default void onDatanodeIdCreation(DatanodeID dataNodeId) {
+    //NOOP
+  }
+
+  /**
+   * Extension point to modify the datanode id.
+   *
+   * @param dataNodeId
+   */
+  default void onDatanodeSuccessfulNamenodeRegisration(
+      DatanodeRegistration dataNodeId) {
+    //NOOP
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java
deleted file mode 100644
index a302462..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.datanode;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.*;
-import static com.sun.jersey.api.core.ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS;
-import static com.sun.jersey.api.core.ResourceConfig.FEATURE_TRACE;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.HashMap;
-import java.util.Map;
-
-import com.sun.jersey.api.container.ContainerFactory;
-import com.sun.jersey.api.core.ApplicationAdapter;
-
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.ksm.protocolPB
-    .KeySpaceManagerProtocolClientSideTranslatorPB;
-import org.apache.hadoop.ozone.ksm.protocolPB.KeySpaceManagerProtocolPB;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.scm.protocolPB
-    .ScmBlockLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.protocolPB.ScmBlockLocationProtocolPB;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.scm.protocolPB
-    .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
-import org.apache.hadoop.ozone.web.handlers.ServiceFilter;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.ObjectStoreApplication;
-import org.apache.hadoop.ozone.web.netty.ObjectStoreJerseyContainer;
-import org.apache.hadoop.ozone.web.storage.DistributedStorageHandler;
-import org.apache.hadoop.ozone.web.localstorage.LocalStorageHandler;
-import org.apache.hadoop.security.UserGroupInformation;
-
-/**
- * Implements object store handling within the DataNode process.  This class is
- * responsible for initializing and maintaining the RPC clients and servers and
- * the web application required for the object store implementation.
- */
-public final class ObjectStoreHandler implements Closeable {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ObjectStoreJerseyContainer.class);
-
-  private final ObjectStoreJerseyContainer objectStoreJerseyContainer;
-  private final KeySpaceManagerProtocolClientSideTranslatorPB
-      keySpaceManagerClient;
-  private final StorageContainerLocationProtocolClientSideTranslatorPB
-      storageContainerLocationClient;
-  private final ScmBlockLocationProtocolClientSideTranslatorPB
-      scmBlockLocationClient;
-  private final StorageHandler storageHandler;
-
-  /**
-   * Creates a new ObjectStoreHandler.
-   *
-   * @param conf configuration
-   * @throws IOException if there is an I/O error
-   */
-  public ObjectStoreHandler(Configuration conf) throws IOException {
-    String shType = conf.getTrimmed(OZONE_HANDLER_TYPE_KEY,
-        OZONE_HANDLER_TYPE_DEFAULT);
-    LOG.info("ObjectStoreHandler initializing with {}: {}",
-        OZONE_HANDLER_TYPE_KEY, shType);
-    boolean ozoneTrace = conf.getBoolean(OZONE_TRACE_ENABLED_KEY,
-        OZONE_TRACE_ENABLED_DEFAULT);
-
-    // Initialize Jersey container for object store web application.
-    if (OzoneConsts.OZONE_HANDLER_DISTRIBUTED.equalsIgnoreCase(shType)) {
-      RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class,
-          ProtobufRpcEngine.class);
-      long scmVersion =
-          RPC.getProtocolVersion(StorageContainerLocationProtocolPB.class);
-
-      InetSocketAddress scmAddress =
-          OzoneClientUtils.getScmAddressForClients(conf);
-      this.storageContainerLocationClient =
-          new StorageContainerLocationProtocolClientSideTranslatorPB(
-              RPC.getProxy(StorageContainerLocationProtocolPB.class, scmVersion,
-              scmAddress, UserGroupInformation.getCurrentUser(), conf,
-              NetUtils.getDefaultSocketFactory(conf),
-              Client.getRpcTimeout(conf)));
-
-      InetSocketAddress scmBlockAddress =
-          OzoneClientUtils.getScmAddressForBlockClients(conf);
-      this.scmBlockLocationClient =
-          new ScmBlockLocationProtocolClientSideTranslatorPB(
-              RPC.getProxy(ScmBlockLocationProtocolPB.class, scmVersion,
-                  scmBlockAddress, UserGroupInformation.getCurrentUser(), conf,
-                  NetUtils.getDefaultSocketFactory(conf),
-                  Client.getRpcTimeout(conf)));
-
-      RPC.setProtocolEngine(conf, KeySpaceManagerProtocolPB.class,
-          ProtobufRpcEngine.class);
-      long ksmVersion =
-          RPC.getProtocolVersion(KeySpaceManagerProtocolPB.class);
-      InetSocketAddress ksmAddress = OzoneClientUtils.getKsmAddress(conf);
-      this.keySpaceManagerClient =
-          new KeySpaceManagerProtocolClientSideTranslatorPB(
-              RPC.getProxy(KeySpaceManagerProtocolPB.class, ksmVersion,
-              ksmAddress, UserGroupInformation.getCurrentUser(), conf,
-              NetUtils.getDefaultSocketFactory(conf),
-              Client.getRpcTimeout(conf)));
-
-      storageHandler = new DistributedStorageHandler(
-          new OzoneConfiguration(conf),
-          this.storageContainerLocationClient,
-          this.keySpaceManagerClient);
-    } else {
-      if (OzoneConsts.OZONE_HANDLER_LOCAL.equalsIgnoreCase(shType)) {
-        storageHandler = new LocalStorageHandler(conf);
-        this.storageContainerLocationClient = null;
-        this.scmBlockLocationClient = null;
-        this.keySpaceManagerClient = null;
-      } else {
-        throw new IllegalArgumentException(
-            String.format("Unrecognized value for %s: %s,"
-                + " Allowed values are %s,%s",
-                OZONE_HANDLER_TYPE_KEY, shType,
-                OzoneConsts.OZONE_HANDLER_DISTRIBUTED,
-                OzoneConsts.OZONE_HANDLER_LOCAL));
-      }
-    }
-    ApplicationAdapter aa =
-        new ApplicationAdapter(new ObjectStoreApplication());
-    Map<String, Object> settingsMap = new HashMap<>();
-    settingsMap.put(PROPERTY_CONTAINER_REQUEST_FILTERS,
-        ServiceFilter.class.getCanonicalName());
-    settingsMap.put(FEATURE_TRACE, ozoneTrace);
-    aa.setPropertiesAndFeatures(settingsMap);
-    this.objectStoreJerseyContainer = ContainerFactory.createContainer(
-        ObjectStoreJerseyContainer.class, aa);
-    this.objectStoreJerseyContainer.setStorageHandler(storageHandler);
-  }
-
-  /**
-   * Returns the initialized web application container.
-   *
-   * @return initialized web application container
-   */
-  public ObjectStoreJerseyContainer getObjectStoreJerseyContainer() {
-    return this.objectStoreJerseyContainer;
-  }
-
-  /**
-   * Returns the storage handler.
-   *
-   * @return returns the storage handler
-   */
-  public StorageHandler getStorageHandler() {
-    return this.storageHandler;
-  }
-
-  @Override
-  public void close() {
-    LOG.info("Closing ObjectStoreHandler.");
-    storageHandler.close();
-    IOUtils.cleanupWithLogger(LOG, storageContainerLocationClient);
-    IOUtils.cleanupWithLogger(LOG, scmBlockLocationClient);
-    IOUtils.cleanupWithLogger(LOG, keySpaceManagerClient);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 1852192..c141293 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -1745,7 +1745,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
       }
       assert newReplicaInfo.getState() == ReplicaState.FINALIZED
           : "Replica should be finalized";
-      
+
       volumeMap.add(bpid, newReplicaInfo);
       return newReplicaInfo;
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
index dbc03c2..2e46b28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
@@ -51,11 +51,9 @@ import org.apache.hadoop.hdfs.server.common.JspHelper;
 import org.apache.hadoop.hdfs.server.datanode.BlockScanner;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.web.webhdfs.DataNodeUGIProvider;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
 import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.http.HttpServer2;
 import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.web.netty.ObjectStoreJerseyContainer;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.security.http.RestCsrfPreventionFilter;
 import org.apache.hadoop.security.ssl.SSLFactory;
@@ -93,19 +91,11 @@ public class DatanodeHttpServer implements Closeable {
 
   public DatanodeHttpServer(final Configuration conf,
       final DataNode datanode,
-      final ServerSocketChannel externalHttpChannel,
-      final ObjectStoreHandler objectStoreHandler)
+      final ServerSocketChannel externalHttpChannel)
     throws IOException {
     this.restCsrfPreventionFilter = createRestCsrfPreventionFilter(conf);
     this.conf = conf;
 
-    final ObjectStoreJerseyContainer finalContainer;
-    if (objectStoreHandler != null) {
-      finalContainer = objectStoreHandler.getObjectStoreJerseyContainer();
-    } else {
-      finalContainer = null;
-    }
-
     Configuration confForInfoServer = new Configuration(conf);
     confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS_KEY, 10);
     int proxyPort =
@@ -160,8 +150,7 @@ public class DatanodeHttpServer implements Closeable {
           }
           p.addLast(
               new ChunkedWriteHandler(),
-              new URLDispatcher(jettyAddr, conf, confForCreate,
-                  finalContainer));
+              new URLDispatcher(jettyAddr, conf, confForCreate));
         }
       });
 
@@ -218,8 +207,7 @@ public class DatanodeHttpServer implements Closeable {
             }
             p.addLast(
                 new ChunkedWriteHandler(),
-                new URLDispatcher(jettyAddr, conf, confForCreate,
-                  finalContainer));
+                new URLDispatcher(jettyAddr, conf, confForCreate));
           }
         });
     } else {
@@ -386,4 +374,3 @@ public class DatanodeHttpServer implements Closeable {
     }
   }
 }
-

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
index 4958bb5..e275fb3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.security.http.RestCsrfPreventionFilter.HttpInteraction;
  * handler drops the request and immediately sends an HTTP 400 response.
  */
 @InterfaceAudience.Private
-final class RestCsrfPreventionFilterHandler
+public final class RestCsrfPreventionFilterHandler
     extends SimpleChannelInboundHandler<HttpRequest> {
 
   private static final Log LOG = DatanodeHttpServer.LOG;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java
index dd958d1..8ec5bf6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java
@@ -17,106 +17,42 @@
  */
 package org.apache.hadoop.hdfs.server.datanode.web;
 
-import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.WEBHDFS_PREFIX;
-
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelPipeline;
 import io.netty.channel.SimpleChannelInboundHandler;
 import io.netty.handler.codec.http.HttpRequest;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.netty.ObjectStoreJerseyContainer;
-import org.apache.hadoop.ozone.web.netty.RequestDispatchObjectStoreChannelHandler;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
 import java.net.InetSocketAddress;
 
+import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.WEBHDFS_PREFIX;
+
 class URLDispatcher extends SimpleChannelInboundHandler<HttpRequest> {
-  protected static final Logger LOG =
-      LoggerFactory.getLogger(URLDispatcher.class);
   private final InetSocketAddress proxyHost;
   private final Configuration conf;
   private final Configuration confForCreate;
-  private final ObjectStoreJerseyContainer objectStoreJerseyContainer;
 
   URLDispatcher(InetSocketAddress proxyHost, Configuration conf,
-                Configuration confForCreate,
-                ObjectStoreJerseyContainer objectStoreJerseyContainer)
-      throws IOException {
+                Configuration confForCreate) {
     this.proxyHost = proxyHost;
     this.conf = conf;
     this.confForCreate = confForCreate;
-    this.objectStoreJerseyContainer = objectStoreJerseyContainer;
   }
 
   @Override
   protected void channelRead0(ChannelHandlerContext ctx, HttpRequest req)
       throws Exception {
+    String uri = req.getUri();
     ChannelPipeline p = ctx.pipeline();
-    if (isWebHdfsRequest(req)) {
+    if (uri.startsWith(WEBHDFS_PREFIX)) {
       WebHdfsHandler h = new WebHdfsHandler(conf, confForCreate);
       p.replace(this, WebHdfsHandler.class.getSimpleName(), h);
       h.channelRead0(ctx, req);
-    } else if (isObjectStoreRequest(req)) {
-      RequestDispatchObjectStoreChannelHandler h =
-          new RequestDispatchObjectStoreChannelHandler(
-              this.objectStoreJerseyContainer);
-      p.replace(this,
-          RequestDispatchObjectStoreChannelHandler.class.getSimpleName(), h);
-      h.channelRead0(ctx, req);
-    } else if (!isObjectStoreRequestHeaders(req)){
+    } else {
       SimpleHttpProxyHandler h = new SimpleHttpProxyHandler(proxyHost);
       p.replace(this, SimpleHttpProxyHandler.class.getSimpleName(), h);
       h.channelRead0(ctx, req);
     }
   }
-
-
-  /*
-   * Returns true if the request has ozone headers
-   *
-   * @param req HTTP request
-   * @return true if request has ozone header, else false
-   */
-
-  private boolean isObjectStoreRequestHeaders(HttpRequest req) {
-    for (String version : req.headers().getAll(Header.OZONE_VERSION_HEADER)) {
-      if (version != null) {
-        LOG.debug("ozone : dispatching call to Ozone, when security is not " +
-            "enabled");
-        return true;
-      }
-    }
-    return false;
-  }
-
-
-  /*
-   * Returns true if the request is to be handled by the object store.
-   *
-   * @param req HTTP request
-   * @return true if the request is to be handled by the object store
-   */
-  private boolean isObjectStoreRequest(HttpRequest req) {
-    if (this.objectStoreJerseyContainer == null) {
-      LOG.debug("ozone : ozone is disabled or when security is enabled, ozone" +
-          " is not supported");
-      return false;
-    }
-    return isObjectStoreRequestHeaders(req);
-  }
-
-  /**
-   * Returns true if the request is to be handled by WebHDFS.
-   *
-   * @param req HTTP request
-   * @return true if the request is to be handled by WebHDFS
-   */
-  private boolean isWebHdfsRequest(HttpRequest req) {
-    return req.getUri().startsWith(WEBHDFS_PREFIX);
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
index 2debf2e..10650da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
@@ -34,9 +34,9 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 
 /**
- * NamespaceInfo is returned by the name-node in reply
+ * NamespaceInfo is returned by the name-node in reply 
  * to a data-node handshake.
- *
+ * 
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
@@ -110,7 +110,7 @@ public class NamespaceInfo extends StorageInfo {
     this.capabilities = capabilities;
   }
 
-  public NamespaceInfo(int nsID, String clusterID, String bpID,
+  public NamespaceInfo(int nsID, String clusterID, String bpID, 
       long cT) {
     this(nsID, clusterID, bpID, cT, Storage.getBuildVersion(),
         VersionInfo.getVersion());
@@ -122,7 +122,7 @@ public class NamespaceInfo extends StorageInfo {
         VersionInfo.getVersion());
     this.state = st;
   }
-
+  
   public long getCapabilities() {
     return capabilities;
   }
@@ -151,7 +151,7 @@ public class NamespaceInfo extends StorageInfo {
   public String getBlockPoolID() {
     return blockPoolID;
   }
-
+  
   public String getSoftwareVersion() {
     return softwareVersion;
   }
@@ -194,4 +194,4 @@ public class NamespaceInfo extends StorageInfo {
           "BPID=" + storage.getBlockPoolID() + ".");
     }
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java
index a1ea3fb..aaa1038 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java
@@ -26,7 +26,6 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Collection;
 import java.util.Set;
 
 import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -36,8 +35,6 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.DFSUtil.ConfiguredNNAddress;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.conf.OzoneConfiguration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Tool;
@@ -81,10 +78,6 @@ public class GetConf extends Configured implements Tool {
         "gets the exclude file path that defines the datanodes " +
         "that need to decommissioned."),
     NNRPCADDRESSES("-nnRpcAddresses", "gets the namenode rpc addresses"),
-    KEYSPACEMANAGER("-keyspacemanagers",
-        "gets list of ozone key space manager nodes in the cluster"),
-    STORAGECONTAINERMANAGER("-storagecontainermanagers",
-        "gets list of ozone storage container manager nodes in the cluster"),
     CONFKEY("-confKey [key]", "gets a specific key from the configuration");
 
     private static final Map<String, CommandHandler> map;
@@ -104,10 +97,6 @@ public class GetConf extends Configured implements Tool {
           new CommandHandler(DFSConfigKeys.DFS_HOSTS_EXCLUDE));
       map.put(StringUtils.toLowerCase(NNRPCADDRESSES.getName()),
           new NNRpcAddressesCommandHandler());
-      map.put(StringUtils.toLowerCase(KEYSPACEMANAGER.getName()),
-          new KeySpaceManagersCommandHandler());
-      map.put(StringUtils.toLowerCase(STORAGECONTAINERMANAGER.getName()),
-          new StorageContainerManagersCommandHandler());
       map.put(StringUtils.toLowerCase(CONFKEY.getName()),
           new PrintConfKeyCommandHandler());
     }
@@ -235,36 +224,9 @@ public class GetConf extends Configured implements Tool {
    * Handler for {@link Command#SECONDARY}
    */
   static class SecondaryNameNodesCommandHandler extends CommandHandler {
-    @Override public int doWorkInternal(GetConf tool, String[] args)
-        throws IOException {
-      tool.printMap(DFSUtil.getSecondaryNameNodeAddresses(tool.getConf()));
-      return 0;
-    }
-  }
-
-  /**
-   * Handler for {@link Command#STORAGECONTAINERMANAGER}.
-   */
-  static class StorageContainerManagersCommandHandler extends CommandHandler {
     @Override
-    public int doWorkInternal(GetConf tool, String[] args) throws IOException {
-      Collection<InetSocketAddress> addresses =
-          OzoneClientUtils.getSCMAddresses(tool.getConf());
-      for (InetSocketAddress addr : addresses) {
-        tool.printOut(addr.getHostName());
-      }
-      return 0;
-    }
-  }
-
-  /**
-   * Handler for {@link Command#KEYSPACEMANAGER}.
-   */
-  static class KeySpaceManagersCommandHandler extends CommandHandler {
-    @Override
-    public int doWorkInternal(GetConf tool, String[] args) throws IOException {
-      tool.printOut(OzoneClientUtils.getKsmAddress(tool.getConf())
-          .getHostName());
+    public int doWorkInternal(GetConf tool, String []args) throws IOException {
+      tool.printMap(DFSUtil.getSecondaryNameNodeAddresses(tool.getConf()));
       return 0;
     }
   }
@@ -395,11 +357,8 @@ public class GetConf extends Configured implements Tool {
     if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) {
       System.exit(0);
     }
-
-    Configuration conf = new Configuration();
-    conf.addResource(new HdfsConfiguration());
-    conf.addResource(new OzoneConfiguration());
-    int res = ToolRunner.run(new GetConf(conf), args);
+    
+    int res = ToolRunner.run(new GetConf(new HdfsConfiguration()), args);
     System.exit(res);
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/InconsistentStorageStateException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/InconsistentStorageStateException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/InconsistentStorageStateException.java
deleted file mode 100644
index c3f9234..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/InconsistentStorageStateException.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
-  * Licensed to the Apache Software Foundation (ASF) under one
-  * or more contributor license agreements.  See the NOTICE file
-  * distributed with this work for additional information
-  * regarding copyright ownership.  The ASF licenses this file
-  * to you under the Apache License, Version 2.0 (the
-  * "License"); you may not use this file except in compliance
-  * with the License.  You may obtain a copy of the License at
-  *
-  *     http://www.apache.org/licenses/LICENSE-2.0
-  *
-  * Unless required by applicable law or agreed to in writing, software
-  * distributed under the License is distributed on an "AS IS" BASIS,
-  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  * See the License for the specific language governing permissions and
-  * limitations under the License.
-  */
-package org.apache.hadoop.ozone.common;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * The exception is thrown when file system state is inconsistent
- * and is not recoverable.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class InconsistentStorageStateException extends IOException {
-  private static final long serialVersionUID = 1L;
-
-  public InconsistentStorageStateException(String descr) {
-    super(descr);
-  }
-
-  public InconsistentStorageStateException(File dir, String descr) {
-    super("Directory " + getFilePath(dir) + " is in an inconsistent state: "
-        + descr);
-  }
-
-  private static String getFilePath(File dir) {
-    try {
-      return dir.getCanonicalPath();
-    } catch (IOException e) {
-    }
-    return dir.getPath();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/Storage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/Storage.java
deleted file mode 100644
index cfcddb4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/Storage.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.common;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.DirectoryStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Properties;
-
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeType;
-import org.apache.hadoop.util.Time;
-
-/**
- * Storage information file. This Class defines the methods to check
- * the consistency of the storage dir and the version file.
- * <p>
- * Local storage information is stored in a separate file VERSION.
- * It contains type of the node,
- * the storage layout version, the SCM id, and
- * the KSM/SCM state creation time.
- *
- */
-@InterfaceAudience.Private
-public abstract class Storage {
-  private static final Logger LOG = LoggerFactory.getLogger(Storage.class);
-
-  protected static final String STORAGE_DIR_CURRENT = "current";
-  protected static final String STORAGE_FILE_VERSION = "VERSION";
-
-  private final NodeType nodeType;
-  private final File root;
-  private final File storageDir;
-
-  private StorageState state;
-  private StorageInfo storageInfo;
-
-
-  /**
-   * Determines the state of the Version file.
-   */
-  public enum StorageState {
-    NON_EXISTENT, NOT_INITIALIZED, INITIALIZED
-  }
-
-  public Storage(NodeType type, File root, String sdName)
-      throws IOException {
-    this.nodeType = type;
-    this.root = root;
-    this.storageDir = new File(root, sdName);
-    this.state = getStorageState();
-    if (state == StorageState.INITIALIZED) {
-      this.storageInfo = new StorageInfo(type, getVersionFile());
-    } else {
-      this.storageInfo = new StorageInfo(
-          nodeType, StorageInfo.newClusterID(), Time.now());
-      setNodeProperties();
-    }
-  }
-
-  /**
-   * Gets the path of the Storage dir.
-   * @return Stoarge dir path
-   */
-  public String getStorageDir() {
-    return storageDir.getAbsoluteFile().toString();
-  }
-
-  /**
-   * Gets the state of the version file.
-   * @return the state of the Version file
-   */
-  public StorageState getState() {
-    return state;
-  }
-
-  public NodeType getNodeType() {
-    return storageInfo.getNodeType();
-  }
-
-  public String getClusterID() {
-    return storageInfo.getClusterID();
-  }
-
-  public long getCreationTime() {
-    return storageInfo.getCreationTime();
-  }
-
-  public void setClusterId(String clusterId) throws IOException {
-    if (state == StorageState.INITIALIZED) {
-      throw new IOException(
-          "Storage directory " + storageDir + " already initialized.");
-    } else {
-      storageInfo.setClusterId(clusterId);
-    }
-  }
-
-  /**
-   * Retreives the storageInfo instance to read/write the common
-   * version file properties.
-   * @return the instance of the storageInfo class
-   */
-  protected StorageInfo getStorageInfo() {
-    return storageInfo;
-  }
-
-  abstract protected Properties getNodeProperties();
-
-  /**
-   * Sets the Node properties spaecific to KSM/SCM.
-   */
-  private void setNodeProperties() {
-    Properties nodeProperties = getNodeProperties();
-    if (nodeProperties != null) {
-      for (String key : nodeProperties.stringPropertyNames()) {
-        storageInfo.setProperty(key, nodeProperties.getProperty(key));
-      }
-    }
-  }
-
-  /**
-   * Directory {@code current} contains latest files defining
-   * the file system meta-data.
-   *
-   * @return the directory path
-   */
-  private File getCurrentDir() {
-    return new File(storageDir, STORAGE_DIR_CURRENT);
-  }
-
-  /**
-   * File {@code VERSION} contains the following fields:
-   * <ol>
-   * <li>node type</li>
-   * <li>KSM/SCM state creation time</li>
-   * <li>other fields specific for this node type</li>
-   * </ol>
-   * The version file is always written last during storage directory updates.
-   * The existence of the version file indicates that all other files have
-   * been successfully written in the storage directory, the storage is valid
-   * and does not need to be recovered.
-   *
-   * @return the version file path
-   */
-  private File getVersionFile() {
-    return new File(getCurrentDir(), STORAGE_FILE_VERSION);
-  }
-
-
-  /**
-   * Check to see if current/ directory is empty. This method is used
-   * before determining to format the directory.
-   * @throws IOException if unable to list files under the directory.
-   */
-  private void checkEmptyCurrent() throws IOException {
-    File currentDir = getCurrentDir();
-    if (!currentDir.exists()) {
-      // if current/ does not exist, it's safe to format it.
-      return;
-    }
-    try (DirectoryStream<Path> dirStream = Files
-        .newDirectoryStream(currentDir.toPath())) {
-      if (dirStream.iterator().hasNext()) {
-        throw new InconsistentStorageStateException(getCurrentDir(),
-            "Can't initialize the storage directory because the current "
-                + "it is not empty.");
-      }
-    }
-  }
-
-  /**
-   * Check consistency of the storage directory.
-   *
-   * @return state {@link StorageState} of the storage directory
-   * @throws IOException
-   */
-  private StorageState getStorageState() throws IOException {
-    assert root != null : "root is null";
-    String rootPath = root.getCanonicalPath();
-    try { // check that storage exists
-      if (!root.exists()) {
-        // storage directory does not exist
-        LOG.warn("Storage directory " + rootPath + " does not exist");
-        return StorageState.NON_EXISTENT;
-      }
-      // or is inaccessible
-      if (!root.isDirectory()) {
-        LOG.warn(rootPath + "is not a directory");
-        return StorageState.NON_EXISTENT;
-      }
-      if (!FileUtil.canWrite(root)) {
-        LOG.warn("Cannot access storage directory " + rootPath);
-        return StorageState.NON_EXISTENT;
-      }
-    } catch (SecurityException ex) {
-      LOG.warn("Cannot access storage directory " + rootPath, ex);
-      return StorageState.NON_EXISTENT;
-    }
-
-    // check whether current directory is valid
-    File versionFile = getVersionFile();
-    boolean hasCurrent = versionFile.exists();
-
-    if (hasCurrent) {
-      return StorageState.INITIALIZED;
-    } else {
-      checkEmptyCurrent();
-      return StorageState.NOT_INITIALIZED;
-    }
-  }
-
-  /**
-   * Creates the Version file if not present,
-   * otherwise returns with IOException.
-   * @throws IOException
-   */
-  public void initialize() throws IOException {
-    if (state == StorageState.INITIALIZED) {
-      throw new IOException("Storage directory already initialized.");
-    }
-    if (!getCurrentDir().mkdirs()) {
-      throw new IOException("Cannot create directory " + getCurrentDir());
-    }
-    storageInfo.writeTo(getVersionFile());
-  }
-
-}
-

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java
deleted file mode 100644
index 82ad955..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.common;
-
-import java.io.IOException;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.File;
-import java.io.RandomAccessFile;
-
-import java.util.Properties;
-import java.util.UUID;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeType;
-
-/**
- * Common class for storage information. This class defines the common
- * properties and functions to set them , write them into the version file
- * and read them from the version file.
- *
- */
-@InterfaceAudience.Private
-public class StorageInfo {
-
-  private Properties properties = new Properties();
-
-  /**
-   * Property to hold node type.
-   */
-  private static final String NODE_TYPE = "nodeType";
-  /**
-   * Property to hold ID of the cluster.
-   */
-  private static final String CLUSTER_ID = "clusterID";
-  /**
-   * Property to hold creation time of the storage.
-   */
-  private static final String CREATION_TIME = "cTime";
-
-  /**
-   * Constructs StorageInfo instance.
-   * @param type
-   *          Type of the node using the storage
-   * @param cid
-   *          Cluster ID
-   * @param cT
-   *          Cluster creation Time
-
-   * @throws IOException
-   */
-  public StorageInfo(NodeType type, String cid, long cT)
-      throws IOException {
-    Preconditions.checkNotNull(type);
-    Preconditions.checkNotNull(cid);
-    Preconditions.checkNotNull(cT);
-    properties.setProperty(NODE_TYPE, type.name());
-    properties.setProperty(CLUSTER_ID, cid);
-    properties.setProperty(CREATION_TIME, String.valueOf(cT));
-  }
-
-  public StorageInfo(NodeType type, File propertiesFile)
-      throws IOException {
-    this.properties = readFrom(propertiesFile);
-    verifyNodeType(type);
-    verifyClusterId();
-    verifyCreationTime();
-  }
-
-  public NodeType getNodeType() {
-    return NodeType.valueOf(properties.getProperty(NODE_TYPE));
-  }
-
-  public String getClusterID() {
-    return properties.getProperty(CLUSTER_ID);
-  }
-
-  public Long  getCreationTime() {
-    String creationTime = properties.getProperty(CREATION_TIME);
-    if(creationTime != null) {
-      return Long.parseLong(creationTime);
-    }
-    return null;
-  }
-
-  public String getProperty(String key) {
-    return properties.getProperty(key);
-  }
-
-  public void setProperty(String key, String value) {
-    properties.setProperty(key, value);
-  }
-
-  public void setClusterId(String clusterId) {
-    properties.setProperty(CLUSTER_ID, clusterId);
-  }
-
-  private void verifyNodeType(NodeType type)
-      throws InconsistentStorageStateException {
-    NodeType nodeType = getNodeType();
-    Preconditions.checkNotNull(nodeType);
-    if(type != nodeType) {
-      throw new InconsistentStorageStateException("Expected NodeType: " + type +
-          ", but found: " + nodeType);
-    }
-  }
-
-  private void verifyClusterId()
-      throws InconsistentStorageStateException {
-    String clusterId = getClusterID();
-    Preconditions.checkNotNull(clusterId);
-    if(clusterId.isEmpty()) {
-      throw new InconsistentStorageStateException("Cluster ID not found");
-    }
-  }
-
-  private void verifyCreationTime() {
-    Long creationTime = getCreationTime();
-    Preconditions.checkNotNull(creationTime);
-  }
-
-
-  public void writeTo(File to)
-      throws IOException {
-    try (RandomAccessFile file = new RandomAccessFile(to, "rws");
-         FileOutputStream out = new FileOutputStream(file.getFD())) {
-      file.seek(0);
-    /*
-     * If server is interrupted before this line,
-     * the version file will remain unchanged.
-     */
-      properties.store(out, null);
-    /*
-     * Now the new fields are flushed to the head of the file, but file
-     * length can still be larger then required and therefore the file can
-     * contain whole or corrupted fields from its old contents in the end.
-     * If server is interrupted here and restarted later these extra fields
-     * either should not effect server behavior or should be handled
-     * by the server correctly.
-     */
-      file.setLength(out.getChannel().position());
-    }
-  }
-
-  private Properties readFrom(File from) throws IOException {
-    try (RandomAccessFile file = new RandomAccessFile(from, "rws");
-        FileInputStream in = new FileInputStream(file.getFD())) {
-      Properties props = new Properties();
-      file.seek(0);
-      props.load(in);
-      return props;
-    }
-  }
-
-  /**
-   * Generate new clusterID.
-   *
-   * clusterID is a persistent attribute of the cluster.
-   * It is generated when the cluster is created and remains the same
-   * during the life cycle of the cluster.  When a new SCM node is initialized,
-   * if this is a new cluster, a new clusterID is generated and stored.
-   * @return new clusterID
-   */
-  public static String newClusterID() {
-    return "CID-" + UUID.randomUUID().toString();
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/package-info.java
deleted file mode 100644
index 6517e58..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.common;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java
deleted file mode 100644
index 690e3b5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.TreeMap;
-
-/**
- * Java class that represents ChunkInfo ProtoBuf class. This helper class allows
- * us to convert to and from protobuf to normal java.
- */
-public class ChunkInfo {
-  private final String chunkName;
-  private final long offset;
-  private final long len;
-  private String checksum;
-  private final Map<String, String> metadata;
-
-
-  /**
-   * Constructs a ChunkInfo.
-   *
-   * @param chunkName - File Name where chunk lives.
-   * @param offset    - offset where Chunk Starts.
-   * @param len       - Length of the Chunk.
-   */
-  public ChunkInfo(String chunkName, long offset, long len) {
-    this.chunkName = chunkName;
-    this.offset = offset;
-    this.len = len;
-    this.metadata = new TreeMap<>();
-  }
-
-  /**
-   * Adds metadata.
-   *
-   * @param key   - Key Name.
-   * @param value - Value.
-   * @throws IOException
-   */
-  public void addMetadata(String key, String value) throws IOException {
-    synchronized (this.metadata) {
-      if (this.metadata.containsKey(key)) {
-        throw new IOException("This key already exists. Key " + key);
-      }
-      metadata.put(key, value);
-    }
-  }
-
-  /**
-   * Gets a Chunkinfo class from the protobuf definitions.
-   *
-   * @param info - Protobuf class
-   * @return ChunkInfo
-   * @throws IOException
-   */
-  public static ChunkInfo getFromProtoBuf(ContainerProtos.ChunkInfo info)
-      throws IOException {
-    Preconditions.checkNotNull(info);
-
-    ChunkInfo chunkInfo = new ChunkInfo(info.getChunkName(), info.getOffset(),
-        info.getLen());
-
-    for (int x = 0; x < info.getMetadataCount(); x++) {
-      chunkInfo.addMetadata(info.getMetadata(x).getKey(),
-          info.getMetadata(x).getValue());
-    }
-
-
-    if (info.hasChecksum()) {
-      chunkInfo.setChecksum(info.getChecksum());
-    }
-    return chunkInfo;
-  }
-
-  /**
-   * Returns a ProtoBuf Message from ChunkInfo.
-   *
-   * @return Protocol Buffer Message
-   */
-  public ContainerProtos.ChunkInfo getProtoBufMessage() {
-    ContainerProtos.ChunkInfo.Builder builder = ContainerProtos
-        .ChunkInfo.newBuilder();
-
-    builder.setChunkName(this.getChunkName());
-    builder.setOffset(this.getOffset());
-    builder.setLen(this.getLen());
-    if (this.getChecksum() != null && !this.getChecksum().isEmpty()) {
-      builder.setChecksum(this.getChecksum());
-    }
-
-    for (Map.Entry<String, String> entry : metadata.entrySet()) {
-      OzoneProtos.KeyValue.Builder keyValBuilder =
-          OzoneProtos.KeyValue.newBuilder();
-      builder.addMetadata(keyValBuilder.setKey(entry.getKey())
-          .setValue(entry.getValue()).build());
-    }
-
-    return builder.build();
-  }
-
-  /**
-   * Returns the chunkName.
-   *
-   * @return - String
-   */
-  public String getChunkName() {
-    return chunkName;
-  }
-
-  /**
-   * Gets the start offset of the given chunk in physical file.
-   *
-   * @return - long
-   */
-  public long getOffset() {
-    return offset;
-  }
-
-  /**
-   * Returns the length of the Chunk.
-   *
-   * @return long
-   */
-  public long getLen() {
-    return len;
-  }
-
-  /**
-   * Returns the SHA256 value of this chunk.
-   *
-   * @return - Hash String
-   */
-  public String getChecksum() {
-    return checksum;
-  }
-
-  /**
-   * Sets the Hash value of this chunk.
-   *
-   * @param checksum - Hash String.
-   */
-  public void setChecksum(String checksum) {
-    this.checksum = checksum;
-  }
-
-  /**
-   * Returns Metadata associated with this Chunk.
-   *
-   * @return - Map of Key,values.
-   */
-  public Map<String, String> getMetadata() {
-    return metadata;
-  }
-
-  @Override
-  public String toString() {
-    return "ChunkInfo{" +
-        "chunkName='" + chunkName +
-        ", offset=" + offset +
-        ", len=" + len +
-        '}';
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java
deleted file mode 100644
index 4eb211b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java
+++ /dev/null
@@ -1,345 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import com.google.common.base.Preconditions;
-import com.google.protobuf.ByteString;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.impl.ChunkManagerImpl;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.AsynchronousFileChannel;
-import java.nio.channels.FileLock;
-import java.nio.file.StandardOpenOption;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.concurrent.ExecutionException;
-
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.CHECKSUM_MISMATCH;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.CONTAINER_INTERNAL_ERROR;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.CONTAINER_NOT_FOUND;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.INVALID_WRITE_SIZE;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.IO_EXCEPTION;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.OVERWRITE_FLAG_REQUIRED;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.UNABLE_TO_FIND_CHUNK;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.UNABLE_TO_FIND_DATA_DIR;
-
-/**
- * Set of utility functions used by the chunk Manager.
- */
-public final class ChunkUtils {
-
-  /* Never constructed. */
-  private ChunkUtils() {
-  }
-
-  /**
-   * Checks if we are getting a request to overwrite an existing range of
-   * chunk.
-   *
-   * @param chunkFile - File
-   * @param chunkInfo - Buffer to write
-   * @return bool
-   */
-  public static boolean isOverWriteRequested(File chunkFile, ChunkInfo
-      chunkInfo) {
-
-    if (!chunkFile.exists()) {
-      return false;
-    }
-
-    long offset = chunkInfo.getOffset();
-    return offset < chunkFile.length();
-  }
-
-  /**
-   * Overwrite is permitted if an only if the user explicitly asks for it. We
-   * permit this iff the key/value pair contains a flag called
-   * [OverWriteRequested, true].
-   *
-   * @param chunkInfo - Chunk info
-   * @return true if the user asks for it.
-   */
-  public static boolean isOverWritePermitted(ChunkInfo chunkInfo) {
-    String overWrite = chunkInfo.getMetadata().get(OzoneConsts.CHUNK_OVERWRITE);
-    return (overWrite != null) &&
-        (!overWrite.isEmpty()) &&
-        (Boolean.valueOf(overWrite));
-  }
-
-  /**
-   * Validates chunk data and returns a file object to Chunk File that we are
-   * expected to write data to.
-   *
-   * @param pipeline - pipeline.
-   * @param data - container data.
-   * @param info - chunk info.
-   * @return File
-   * @throws StorageContainerException
-   */
-  public static File validateChunk(Pipeline pipeline, ContainerData data,
-      ChunkInfo info) throws StorageContainerException {
-
-    Logger log = LoggerFactory.getLogger(ChunkManagerImpl.class);
-
-    File chunkFile = getChunkFile(pipeline, data, info);
-    if (ChunkUtils.isOverWriteRequested(chunkFile, info)) {
-      if (!ChunkUtils.isOverWritePermitted(info)) {
-        log.error("Rejecting write chunk request. Chunk overwrite " +
-            "without explicit request. {}", info.toString());
-        throw new StorageContainerException("Rejecting write chunk request. " +
-            "OverWrite flag required." + info.toString(),
-            OVERWRITE_FLAG_REQUIRED);
-      }
-    }
-    return chunkFile;
-  }
-
-  /**
-   * Validates that Path to chunk file exists.
-   *
-   * @param pipeline - Container Info.
-   * @param data - Container Data
-   * @param info - Chunk info
-   * @return - File.
-   * @throws StorageContainerException
-   */
-  public static File getChunkFile(Pipeline pipeline, ContainerData data,
-      ChunkInfo info) throws StorageContainerException {
-
-    Logger log = LoggerFactory.getLogger(ChunkManagerImpl.class);
-    if (data == null) {
-      log.error("Invalid container Name: {}", pipeline.getContainerName());
-      throw new StorageContainerException("Unable to find the container Name:" +
-          " " +
-          pipeline.getContainerName(), CONTAINER_NOT_FOUND);
-    }
-
-    File dataDir = ContainerUtils.getDataDirectory(data).toFile();
-    if (!dataDir.exists()) {
-      log.error("Unable to find the data directory: {}", dataDir);
-      throw new StorageContainerException("Unable to find the data directory:" +
-          " " + dataDir, UNABLE_TO_FIND_DATA_DIR);
-    }
-
-    return dataDir.toPath().resolve(info.getChunkName()).toFile();
-
-  }
-
-  /**
-   * Writes the data in chunk Info to the specified location in the chunkfile.
-   *
-   * @param chunkFile - File to write data to.
-   * @param chunkInfo - Data stream to write.
-   * @param data - The data buffer.
-   * @throws StorageContainerException
-   */
-  public static void writeData(File chunkFile, ChunkInfo chunkInfo,
-      byte[] data) throws
-      StorageContainerException, ExecutionException, InterruptedException,
-      NoSuchAlgorithmException {
-
-    Logger log = LoggerFactory.getLogger(ChunkManagerImpl.class);
-    if (data.length != chunkInfo.getLen()) {
-      String err = String.format("data array does not match the length " +
-              "specified. DataLen: %d Byte Array: %d",
-          chunkInfo.getLen(), data.length);
-      log.error(err);
-      throw new StorageContainerException(err, INVALID_WRITE_SIZE);
-    }
-
-    AsynchronousFileChannel file = null;
-    FileLock lock = null;
-
-    try {
-      file =
-          AsynchronousFileChannel.open(chunkFile.toPath(),
-              StandardOpenOption.CREATE,
-              StandardOpenOption.WRITE,
-              StandardOpenOption.SPARSE,
-              StandardOpenOption.SYNC);
-      lock = file.lock().get();
-      if (chunkInfo.getChecksum() != null &&
-          !chunkInfo.getChecksum().isEmpty()) {
-        verifyChecksum(chunkInfo, data, log);
-      }
-      int size = file.write(ByteBuffer.wrap(data), chunkInfo.getOffset()).get();
-      if (size != data.length) {
-        log.error("Invalid write size found. Size:{}  Expected: {} ", size,
-            data.length);
-        throw new StorageContainerException("Invalid write size found. " +
-            "Size: " + size + " Expected: " + data.length, INVALID_WRITE_SIZE);
-      }
-    } catch (IOException e) {
-      throw new StorageContainerException(e, IO_EXCEPTION);
-
-    } finally {
-      if (lock != null) {
-        try {
-          lock.release();
-        } catch (IOException e) {
-          log.error("Unable to release lock ??, Fatal Error.");
-          throw new StorageContainerException(e, CONTAINER_INTERNAL_ERROR);
-
-        }
-      }
-      if (file != null) {
-        try {
-          file.close();
-        } catch (IOException e) {
-          throw new StorageContainerException("Error closing chunk file",
-              e, CONTAINER_INTERNAL_ERROR);
-        }
-      }
-    }
-  }
-
-  /**
-   * Verifies the checksum of a chunk against the data buffer.
-   *
-   * @param chunkInfo - Chunk Info.
-   * @param data - data buffer
-   * @param log - log
-   * @throws NoSuchAlgorithmException
-   * @throws StorageContainerException
-   */
-  private static void verifyChecksum(ChunkInfo chunkInfo, byte[] data, Logger
-      log) throws NoSuchAlgorithmException, StorageContainerException {
-    MessageDigest sha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
-    sha.update(data);
-    if (!Hex.encodeHexString(sha.digest()).equals(
-        chunkInfo.getChecksum())) {
-      log.error("Checksum mismatch. Provided: {} , computed: {}",
-          chunkInfo.getChecksum(), DigestUtils.sha256Hex(sha.digest()));
-      throw new StorageContainerException("Checksum mismatch. Provided: " +
-          chunkInfo.getChecksum() + " , computed: " +
-          DigestUtils.sha256Hex(sha.digest()), CHECKSUM_MISMATCH);
-    }
-  }
-
-  /**
-   * Reads data from an existing chunk file.
-   *
-   * @param chunkFile - file where data lives.
-   * @param data - chunk definition.
-   * @return ByteBuffer
-   * @throws StorageContainerException
-   * @throws ExecutionException
-   * @throws InterruptedException
-   */
-  public static ByteBuffer readData(File chunkFile, ChunkInfo data) throws
-      StorageContainerException, ExecutionException, InterruptedException,
-      NoSuchAlgorithmException {
-    Logger log = LoggerFactory.getLogger(ChunkManagerImpl.class);
-
-    if (!chunkFile.exists()) {
-      log.error("Unable to find the chunk file. chunk info : {}",
-          data.toString());
-      throw new StorageContainerException("Unable to find the chunk file. " +
-          "chunk info " +
-          data.toString(), UNABLE_TO_FIND_CHUNK);
-    }
-
-    AsynchronousFileChannel file = null;
-    FileLock lock = null;
-    try {
-      file =
-          AsynchronousFileChannel.open(chunkFile.toPath(),
-              StandardOpenOption.READ);
-      lock = file.lock(data.getOffset(), data.getLen(), true).get();
-
-      ByteBuffer buf = ByteBuffer.allocate((int) data.getLen());
-      file.read(buf, data.getOffset()).get();
-
-      if (data.getChecksum() != null && !data.getChecksum().isEmpty()) {
-        verifyChecksum(data, buf.array(), log);
-      }
-
-      return buf;
-    } catch (IOException e) {
-      throw new StorageContainerException(e, IO_EXCEPTION);
-    } finally {
-      if (lock != null) {
-        try {
-          lock.release();
-        } catch (IOException e) {
-          log.error("I/O error is lock release.");
-        }
-      }
-      if (file != null) {
-        IOUtils.closeStream(file);
-      }
-    }
-  }
-
-  /**
-   * Returns a CreateContainer Response. This call is used by create and delete
-   * containers which have null success responses.
-   *
-   * @param msg Request
-   * @return Response.
-   */
-  public static ContainerProtos.ContainerCommandResponseProto
-      getChunkResponse(ContainerProtos.ContainerCommandRequestProto msg) {
-    return ContainerUtils.getContainerResponse(msg);
-  }
-
-  /**
-   * Gets a response to the read chunk calls.
-   *
-   * @param msg - Msg
-   * @param data - Data
-   * @param info - Info
-   * @return Response.
-   */
-  public static ContainerProtos.ContainerCommandResponseProto
-      getReadChunkResponse(ContainerProtos.ContainerCommandRequestProto msg,
-      byte[] data, ChunkInfo info) {
-    Preconditions.checkNotNull(msg);
-
-    ContainerProtos.ReadChunkResponseProto.Builder response =
-        ContainerProtos.ReadChunkResponseProto.newBuilder();
-    response.setChunkData(info.getProtoBufMessage());
-    response.setData(ByteString.copyFrom(data));
-    response.setPipeline(msg.getReadChunk().getPipeline());
-
-    ContainerProtos.ContainerCommandResponseProto.Builder builder =
-        ContainerUtils.getContainerResponse(msg, ContainerProtos.Result
-            .SUCCESS, "");
-    builder.setReadChunk(response);
-    return builder.build();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
deleted file mode 100644
index d0886e1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
+++ /dev/null
@@ -1,326 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.util.Time;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * This class maintains the information about a container in the ozone world.
- * <p>
- * A container is a name, along with metadata- which is a set of key value
- * pair.
- */
-public class ContainerData {
-
-  private final String containerName;
-  private final Map<String, String> metadata;
-  private String dbPath;  // Path to Level DB Store.
-  // Path to Physical file system where container and checksum are stored.
-  private String containerFilePath;
-  private String hash;
-  private AtomicLong bytesUsed;
-  private long maxSize;
-  private Long containerID;
-  private OzoneProtos.LifeCycleState state;
-
-  /**
-   * Constructs a  ContainerData Object.
-   *
-   * @param containerName - Name
-   */
-  public ContainerData(String containerName, Long containerID,
-      Configuration conf) {
-    this.metadata = new TreeMap<>();
-    this.containerName = containerName;
-    this.maxSize = conf.getLong(ScmConfigKeys.SCM_CONTAINER_CLIENT_MAX_SIZE_KEY,
-        ScmConfigKeys.SCM_CONTAINER_CLIENT_MAX_SIZE_DEFAULT) * OzoneConsts.GB;
-    this.bytesUsed =  new AtomicLong(0L);
-    this.containerID = containerID;
-    this.state = OzoneProtos.LifeCycleState.OPEN;
-  }
-
-  /**
-   * Constructs a ContainerData object from ProtoBuf classes.
-   *
-   * @param protoData - ProtoBuf Message
-   * @throws IOException
-   */
-  public static ContainerData getFromProtBuf(
-      ContainerProtos.ContainerData protoData, Configuration conf)
-      throws IOException {
-    ContainerData data = new ContainerData(protoData.getName(),
-        protoData.getContainerID(), conf);
-    for (int x = 0; x < protoData.getMetadataCount(); x++) {
-      data.addMetadata(protoData.getMetadata(x).getKey(),
-          protoData.getMetadata(x).getValue());
-    }
-
-    if (protoData.hasContainerPath()) {
-      data.setContainerPath(protoData.getContainerPath());
-    }
-
-    if (protoData.hasDbPath()) {
-      data.setDBPath(protoData.getDbPath());
-    }
-
-    if (protoData.hasState()) {
-      data.setState(protoData.getState());
-    }
-
-    if(protoData.hasHash()) {
-      data.setHash(protoData.getHash());
-    }
-
-    if (protoData.hasBytesUsed()) {
-      data.setBytesUsed(protoData.getBytesUsed());
-    }
-
-    if (protoData.hasSize()) {
-      data.setMaxSize(protoData.getSize());
-    }
-    return data;
-  }
-
-  /**
-   * Returns a ProtoBuf Message from ContainerData.
-   *
-   * @return Protocol Buffer Message
-   */
-  public ContainerProtos.ContainerData getProtoBufMessage() {
-    ContainerProtos.ContainerData.Builder builder = ContainerProtos
-        .ContainerData.newBuilder();
-    builder.setName(this.getContainerName());
-    builder.setContainerID(this.getContainerID());
-
-    if (this.getDBPath() != null) {
-      builder.setDbPath(this.getDBPath());
-    }
-
-    if (this.getHash() != null) {
-      builder.setHash(this.getHash());
-    }
-
-    if (this.getContainerPath() != null) {
-      builder.setContainerPath(this.getContainerPath());
-    }
-
-    builder.setState(this.getState());
-
-    for (Map.Entry<String, String> entry : metadata.entrySet()) {
-      OzoneProtos.KeyValue.Builder keyValBuilder =
-          OzoneProtos.KeyValue.newBuilder();
-      builder.addMetadata(keyValBuilder.setKey(entry.getKey())
-          .setValue(entry.getValue()).build());
-    }
-
-    if (this.getBytesUsed() >= 0) {
-      builder.setBytesUsed(this.getBytesUsed());
-    }
-
-    if (this.getKeyCount() >= 0) {
-      builder.setKeyCount(this.getKeyCount());
-    }
-
-    if (this.getMaxSize() >= 0) {
-      builder.setSize(this.getMaxSize());
-    }
-
-    return builder.build();
-  }
-
-  /**
-   * Returns the name of the container.
-   *
-   * @return - name
-   */
-  public String getContainerName() {
-    return containerName;
-  }
-
-  /**
-   * Adds metadata.
-   */
-  public void addMetadata(String key, String value) throws IOException {
-    synchronized (this.metadata) {
-      if (this.metadata.containsKey(key)) {
-        throw new IOException("This key already exists. Key " + key);
-      }
-      metadata.put(key, value);
-    }
-  }
-
-  /**
-   * Returns all metadata.
-   */
-  public Map<String, String> getAllMetadata() {
-    synchronized (this.metadata) {
-      return Collections.unmodifiableMap(this.metadata);
-    }
-  }
-
-  /**
-   * Returns value of a key.
-   */
-  public String getValue(String key) {
-    synchronized (this.metadata) {
-      return metadata.get(key);
-    }
-  }
-
-  /**
-   * Deletes a metadata entry from the map.
-   *
-   * @param key - Key
-   */
-  public void deleteKey(String key) {
-    synchronized (this.metadata) {
-      metadata.remove(key);
-    }
-  }
-
-  /**
-   * Returns path.
-   *
-   * @return - path
-   */
-  public String getDBPath() {
-    return dbPath;
-  }
-
-  /**
-   * Sets path.
-   *
-   * @param path - String.
-   */
-  public void setDBPath(String path) {
-    this.dbPath = path;
-  }
-
-  /**
-   * This function serves as the generic key for ContainerCache class. Both
-   * ContainerData and ContainerKeyData overrides this function to appropriately
-   * return the right name that can  be used in ContainerCache.
-   *
-   * @return String Name.
-   */
-  public String getName() {
-    return getContainerName();
-  }
-
-  /**
-   * Get container file path.
-   * @return - Physical path where container file and checksum is stored.
-   */
-  public String getContainerPath() {
-    return containerFilePath;
-  }
-
-  /**
-   * Set container Path.
-   * @param containerPath - File path.
-   */
-  public void setContainerPath(String containerPath) {
-    this.containerFilePath = containerPath;
-  }
-
-  /**
-   * Get container ID.
-   * @return - container ID.
-   */
-  public synchronized Long getContainerID() {
-    return containerID;
-  }
-
-  public synchronized  void setState(OzoneProtos.LifeCycleState state) {
-    this.state = state;
-  }
-
-  public synchronized OzoneProtos.LifeCycleState getState() {
-    return this.state;
-  }
-
-  /**
-   * checks if the container is open.
-   * @return - boolean
-   */
-  public synchronized  boolean isOpen() {
-    return OzoneProtos.LifeCycleState.OPEN == state;
-  }
-
-  /**
-   * Marks this container as closed.
-   */
-  public synchronized void closeContainer() {
-    // TODO: closed or closing here
-    setState(OzoneProtos.LifeCycleState.CLOSED);
-
-    // Some thing brain dead for now. name + Time stamp of when we get the close
-    // container message.
-    setHash(DigestUtils.sha256Hex(this.getContainerName() +
-        Long.toString(Time.monotonicNow())));
-  }
-
-  /**
-   * Final hash for this container.
-   * @return - Hash
-   */
-  public String getHash() {
-    return hash;
-  }
-
-  public void setHash(String hash) {
-    this.hash = hash;
-  }
-
-  public void setMaxSize(long maxSize) {
-    this.maxSize = maxSize;
-  }
-
-  public long getMaxSize() {
-    return maxSize;
-  }
-
-  public long getKeyCount() {
-    return metadata.size();
-  }
-
-  public void setBytesUsed(long used) {
-    this.bytesUsed.set(used);
-  }
-
-  public long addBytesUsed(long delta) {
-    return this.bytesUsed.addAndGet(delta);
-  }
-
-  public long getBytesUsed() {
-    return bytesUsed.get();
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[50/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/pom.xml b/hadoop-cblock/server/pom.xml
new file mode 100644
index 0000000..17af85d
--- /dev/null
+++ b/hadoop-cblock/server/pom.xml
@@ -0,0 +1,169 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-cblock</artifactId>
+    <version>3.2.0-SNAPSHOT</version>
+  </parent>
+  <artifactId>hadoop-cblock-server</artifactId>
+  <version>3.2.0-SNAPSHOT</version>
+  <description>Apache Hadoop CBlock Server</description>
+  <name>Apache Hadoop CBlock Server</name>
+  <packaging>jar</packaging>
+
+  <properties>
+    <hadoop.component>cblock</hadoop.component>
+    <is.hadoop.component>true</is.hadoop.component>
+  </properties>
+
+  <dependencies>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdsl-server-framework</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-ozone-common</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdsl-common</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdsl-client</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-ozone-integration-test</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+
+    <dependency>
+      <groupId>org.jscsi</groupId>
+      <artifactId>target</artifactId>
+      <version>2.6.0</version>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>ch.qos.logback</groupId>
+          <artifactId>logback-classic</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>io.kubernetes</groupId>
+      <artifactId>client-java</artifactId>
+      <version>1.0.0-beta1</version>
+      <exclusions>
+        <exclusion>
+          <groupId>io.swagger</groupId>
+          <artifactId>swagger-annotations</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.github.stefanbirkner</groupId>
+          <artifactId>system-rules</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+
+  </dependencies>
+
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <configuration>
+          <excludes>
+            <exclude>src/test/resources/dynamicprovisioner/expected1-pv.json</exclude>
+            <exclude>src/test/resources/dynamicprovisioner/input1-pvc.json</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-maven-plugins</artifactId>
+        <executions>
+          <execution>
+            <id>compile-protoc</id>
+            <goals>
+              <goal>protoc</goal>
+            </goals>
+            <configuration>
+              <protocVersion>${protobuf.version}</protocVersion>
+              <protocCommand>${protoc.path}</protocCommand>
+              <imports>
+                <param>
+                  ${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto
+                </param>
+                <param>
+                  ${basedir}/../../hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/
+                </param>
+                <param>
+                  ${basedir}/../../hadoop-hdfs-project/hadoop-hdfs/src/main/proto/
+                </param>
+                <param>
+                  ${basedir}/../../hadoop-hdsl/common/src/main/proto/
+                </param>
+                <param>${basedir}/src/main/proto</param>
+              </imports>
+              <source>
+                <directory>${basedir}/src/main/proto</directory>
+                <includes>
+                  <include>CBlockClientServerProtocol.proto</include>
+                  <include>CBlockServiceProtocol.proto</include>
+                </includes>
+              </source>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>findbugs-maven-plugin</artifactId>
+        <configuration>
+          <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
new file mode 100644
index 0000000..ec5d7d2
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import static java.lang.Thread.NORM_PRIORITY;
+
+/**
+ * This class contains constants for configuration keys used in CBlock.
+ */
+public final class CBlockConfigKeys {
+
+  public static final String DFS_CBLOCK_SERVICERPC_ADDRESS_KEY =
+      "dfs.cblock.servicerpc-address";
+  public static final int DFS_CBLOCK_SERVICERPC_PORT_DEFAULT =
+      9810;
+  public static final String DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT =
+      "0.0.0.0";
+
+  public static final String DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY =
+      "dfs.cblock.jscsi-address";
+
+  //The port on CBlockManager node for jSCSI to ask
+  public static final String DFS_CBLOCK_JSCSI_PORT_KEY =
+      "dfs.cblock.jscsi.port";
+  public static final int DFS_CBLOCK_JSCSI_PORT_DEFAULT =
+      9811;
+
+  public static final String DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY =
+      "dfs.cblock.service.rpc-bind-host";
+  public static final String DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY =
+      "dfs.cblock.jscsi.rpc-bind-host";
+
+  // default block size is 4KB
+  public static final int DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT =
+      4096;
+
+  public static final String DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY =
+      "dfs.cblock.service.handler.count";
+  public static final int DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT = 10;
+
+  public static final String DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY =
+      "dfs.cblock.service.leveldb.path";
+  //TODO : find a better place
+  public static final String DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT =
+      "/tmp/cblock_levelDB.dat";
+
+
+  public static final String DFS_CBLOCK_DISK_CACHE_PATH_KEY =
+      "dfs.cblock.disk.cache.path";
+  public static final String DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT =
+      "/tmp/cblockCacheDB";
+  /**
+   * Setting this flag to true makes the block layer compute a sha256 hash of
+   * the data and log that information along with block ID. This is very
+   * useful for doing trace based simulation of various workloads. Since it is
+   * computing a hash for each block this could be expensive, hence default
+   * is false.
+   */
+  public static final String DFS_CBLOCK_TRACE_IO = "dfs.cblock.trace.io";
+  public static final boolean DFS_CBLOCK_TRACE_IO_DEFAULT = false;
+
+  public static final String DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO =
+      "dfs.cblock.short.circuit.io";
+  public static final boolean DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO_DEFAULT =
+      false;
+
+  /**
+   * Cache size in 1000s of entries. 256 indicates 256 * 1024.
+   */
+  public static final String DFS_CBLOCK_CACHE_QUEUE_SIZE_KB =
+      "dfs.cblock.cache.queue.size.in.kb";
+  public static final int DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT = 256;
+
+  /**
+   *  Minimum Number of threads that cache pool will use for background I/O.
+   */
+  public static final String DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE =
+      "dfs.cblock.cache.core.min.pool.size";
+  public static final int DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT = 16;
+
+  /**
+   *  Maximum Number of threads that cache pool will use for background I/O.
+   */
+
+  public static final String DFS_CBLOCK_CACHE_MAX_POOL_SIZE =
+      "dfs.cblock.cache.max.pool.size";
+  public static final int DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT = 256;
+
+  /**
+   * Number of seconds to keep the Thread alive when it is idle.
+   */
+  public static final String DFS_CBLOCK_CACHE_KEEP_ALIVE =
+      "dfs.cblock.cache.keep.alive";
+  public static final String DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT = "60s";
+
+  /**
+   * Priority of cache flusher thread, affecting the relative performance of
+   * write and read.
+   */
+  public static final String DFS_CBLOCK_CACHE_THREAD_PRIORITY =
+      "dfs.cblock.cache.thread.priority";
+  public static final int DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT =
+      NORM_PRIORITY;
+
+  /**
+   * Block Buffer size in terms of blockID entries, 512 means 512 blockIDs.
+   */
+  public static final String DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE =
+      "dfs.cblock.cache.block.buffer.size";
+  public static final int DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT = 512;
+
+  public static final String DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL =
+      "dfs.cblock.block.buffer.flush.interval";
+  public static final String DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL_DEFAULT =
+      "60s";
+
+  // jscsi server settings
+  public static final String DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY =
+      "dfs.cblock.jscsi.server.address";
+  public static final String DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT =
+      "0.0.0.0";
+  public static final String DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY =
+      "dfs.cblock.jscsi.cblock.server.address";
+  public static final String DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT =
+      "127.0.0.1";
+
+  // to what address cblock server should talk to scm?
+  public static final String DFS_CBLOCK_SCM_IPADDRESS_KEY =
+      "dfs.cblock.scm.ipaddress";
+  public static final String DFS_CBLOCK_SCM_IPADDRESS_DEFAULT =
+      "127.0.0.1";
+  public static final String DFS_CBLOCK_SCM_PORT_KEY =
+      "dfs.cblock.scm.port";
+  public static final int DFS_CBLOCK_SCM_PORT_DEFAULT = 9860;
+
+  public static final String DFS_CBLOCK_CONTAINER_SIZE_GB_KEY =
+      "dfs.cblock.container.size.gb";
+  public static final int DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT =
+      5;
+
+  // LevelDB cache file uses an off-heap cache in LevelDB of 256 MB.
+  public static final String DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_KEY =
+      "dfs.cblock.cache.leveldb.cache.size.mb";
+  public static final int DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_DEFAULT = 256;
+
+  /**
+   * Cache does an best case attempt to write a block to a container.
+   * At some point of time, we will need to handle the case where we did try
+   * 64K times and is till not able to write to the container.
+   *
+   * TODO: We will need cBlock Server to allow us to do a remapping of the
+   * block location in case of failures, at that point we should reduce the
+   * retry count to a more normal number. This is approximately 18 hours of
+   * retry.
+   */
+  public static final String DFS_CBLOCK_CACHE_MAX_RETRY_KEY =
+      "dfs.cblock.cache.max.retry";
+  public static final int DFS_CBLOCK_CACHE_MAX_RETRY_DEFAULT =
+      64 * 1024;
+
+  /**
+   * Cblock CLI configs.
+   */
+  public static final String DFS_CBLOCK_MANAGER_POOL_SIZE =
+      "dfs.cblock.manager.pool.size";
+  public static final int DFS_CBLOCK_MANAGER_POOL_SIZE_DEFAULT = 16;
+
+  /**
+   * currently the largest supported volume is about 8TB, which might take
+   * > 20 seconds to finish creating containers. thus set timeout to 30 sec.
+   */
+  public static final String DFS_CBLOCK_RPC_TIMEOUT =
+      "dfs.cblock.rpc.timeout";
+  public static final String DFS_CBLOCK_RPC_TIMEOUT_DEFAULT = "300s";
+
+  public static final String DFS_CBLOCK_ISCSI_ADVERTISED_IP =
+      "dfs.cblock.iscsi.advertised.ip";
+
+  public static final String DFS_CBLOCK_ISCSI_ADVERTISED_PORT =
+      "dfs.cblock.iscsi.advertised.port";
+
+  public static final int DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT = 3260;
+
+
+  public static final String
+      DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED
+        = "dfs.cblock.kubernetes.dynamic-provisioner.enabled";
+
+  public static final boolean
+      DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED_DEFAULT = false;
+
+  public static final String
+      DFS_CBLOCK_KUBERNETES_CBLOCK_USER =
+         "dfs.cblock.kubernetes.cblock-user";
+
+  public static final String
+      DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT =
+         "iqn.2001-04.org.apache.hadoop";
+
+  public static final String
+      DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY =
+         "dfs.cblock.kubernetes.configfile";
+
+  private CBlockConfigKeys() {
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockManager.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
new file mode 100644
index 0000000..788d01e
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
@@ -0,0 +1,426 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.BlockingService;
+import org.apache.hadoop.cblock.kubernetes.DynamicProvisioner;
+import org.apache.hadoop.cblock.meta.VolumeDescriptor;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.proto.CBlockClientProtocol;
+import org.apache.hadoop.cblock.proto.CBlockServiceProtocol;
+import org.apache.hadoop.cblock.proto.MountVolumeResponse;
+import org.apache.hadoop.cblock.protocol.proto
+    .CBlockClientServerProtocolProtos;
+import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
+import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB;
+import org.apache.hadoop.cblock.protocolPB
+    .CBlockClientServerProtocolServerSideTranslatorPB;
+import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
+import org.apache.hadoop.cblock.protocolPB
+    .CBlockServiceProtocolServerSideTranslatorPB;
+import org.apache.hadoop.ipc.Client;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.apache.hadoop.scm.client.ContainerOperationClient;
+import org.apache.hadoop.scm.client.ScmClient;
+import org.apache.hadoop.cblock.storage.StorageManager;
+import org.apache.hadoop.cblock.util.KeyUtil;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.scm.protocolPB
+    .StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.utils.LevelDBStore;
+
+import static org.apache.hadoop.cblock.CblockUtils.getCblockServerRpcAddr;
+import static org.apache.hadoop.cblock.CblockUtils.getCblockServiceRpcAddr;
+import static org.apache.hadoop.ozone.web.util.ServerUtils
+    .updateRPCListenAddress;
+import org.iq80.leveldb.DBIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_CONTAINER_SIZE_GB_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SCM_IPADDRESS_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SCM_IPADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SCM_PORT_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SCM_PORT_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED_DEFAULT;
+
+
+/**
+ * The main entry point of CBlock operations, ALL the CBlock operations
+ * will go through this class. But NOTE that:
+ *
+ * volume operations (create/
+ * delete/info) are:
+ *    client -> CBlockManager -> StorageManager -> CBlock client
+ *
+ * IO operations (put/get block) are;
+ *    client -> CBlock client -> container
+ *
+ */
+public class CBlockManager implements CBlockServiceProtocol,
+    CBlockClientProtocol {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(CBlockManager.class);
+
+  private final RPC.Server cblockService;
+  private final RPC.Server cblockServer;
+
+  private final StorageManager storageManager;
+
+  private final LevelDBStore levelDBStore;
+  private final String dbPath;
+
+  private final DynamicProvisioner kubernetesDynamicProvisioner;
+
+  private Charset encoding = Charset.forName("UTF-8");
+
+  public CBlockManager(OzoneConfiguration conf,
+      ScmClient storageClient) throws IOException {
+    // Fix the cBlockManagerId generattion code here. Should support
+    // cBlockManager --init command which will generate a cBlockManagerId and
+    // persist it locally.
+    storageManager =
+        new StorageManager(storageClient, conf, "CBLOCK");
+
+    dbPath = conf.getTrimmed(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY,
+        DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT);
+    levelDBStore = new LevelDBStore(new File(dbPath), true);
+    LOG.info("Try to load exising volume information");
+    readFromPersistentStore();
+
+    RPC.setProtocolEngine(conf, CBlockServiceProtocolPB.class,
+        ProtobufRpcEngine.class);
+    RPC.setProtocolEngine(conf, CBlockClientServerProtocolPB.class,
+        ProtobufRpcEngine.class);
+    // start service for client command-to-cblock server service
+    InetSocketAddress serviceRpcAddr =
+        getCblockServiceRpcAddr(conf);
+    BlockingService cblockProto =
+        CBlockServiceProtocolProtos
+            .CBlockServiceProtocolService
+            .newReflectiveBlockingService(
+                new CBlockServiceProtocolServerSideTranslatorPB(this)
+            );
+    cblockService = startRpcServer(conf, CBlockServiceProtocolPB.class,
+        cblockProto, serviceRpcAddr,
+        DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY,
+        DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY,
+        DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT);
+    InetSocketAddress cblockServiceRpcAddress =
+        updateRPCListenAddress(conf,
+            DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, serviceRpcAddr, cblockService);
+    LOG.info("CBlock manager listening for client commands on: {}",
+        cblockServiceRpcAddress);
+    // now start service for cblock client-to-cblock server communication
+
+    InetSocketAddress serverRpcAddr =
+        getCblockServerRpcAddr(conf);
+    BlockingService serverProto =
+        CBlockClientServerProtocolProtos
+            .CBlockClientServerProtocolService
+            .newReflectiveBlockingService(
+                new CBlockClientServerProtocolServerSideTranslatorPB(this)
+            );
+    cblockServer = startRpcServer(
+        conf, CBlockClientServerProtocolPB.class,
+        serverProto, serverRpcAddr,
+        DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY,
+        DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY,
+        DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT);
+    InetSocketAddress cblockServerRpcAddress =
+        updateRPCListenAddress(conf,
+            DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, serverRpcAddr, cblockServer);
+    LOG.info("CBlock server listening for client commands on: {}",
+        cblockServerRpcAddress);
+
+    if (conf.getBoolean(DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED,
+        DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED_DEFAULT)) {
+
+      kubernetesDynamicProvisioner =
+          new DynamicProvisioner(conf, storageManager);
+      kubernetesDynamicProvisioner.init();
+
+    } else {
+      kubernetesDynamicProvisioner = null;
+    }
+  }
+
+  public void start() {
+    cblockService.start();
+    cblockServer.start();
+    if (kubernetesDynamicProvisioner != null) {
+      kubernetesDynamicProvisioner.start();
+    }
+    LOG.info("CBlock manager started!");
+  }
+
+  public void stop() {
+    cblockService.stop();
+    cblockServer.stop();
+    if (kubernetesDynamicProvisioner != null) {
+      kubernetesDynamicProvisioner.stop();
+    }
+  }
+
+  public void join() {
+    try {
+      cblockService.join();
+      cblockServer.join();
+    } catch (InterruptedException e) {
+      LOG.error("Interrupted during join");
+      Thread.currentThread().interrupt();
+    }
+  }
+
+  /**
+   * Starts an RPC server, if configured.
+   *
+   * @param conf configuration
+   * @param protocol RPC protocol provided by RPC server
+   * @param instance RPC protocol implementation instance
+   * @param addr configured address of RPC server
+   * @param bindHostKey configuration key for setting explicit bind host.  If
+   *     the property is not configured, then the bind host is taken from addr.
+   * @param handlerCountKey configuration key for RPC server handler count
+   * @param handlerCountDefault default RPC server handler count if unconfigured
+   * @return RPC server, or null if addr is null
+   * @throws IOException if there is an I/O error while creating RPC server
+   */
+  private static RPC.Server startRpcServer(OzoneConfiguration conf,
+      Class<?> protocol, BlockingService instance,
+      InetSocketAddress addr, String bindHostKey,
+      String handlerCountKey, int handlerCountDefault) throws IOException {
+    if (addr == null) {
+      return null;
+    }
+    String bindHost = conf.getTrimmed(bindHostKey);
+    if (bindHost == null || bindHost.isEmpty()) {
+      bindHost = addr.getHostName();
+    }
+    int numHandlers = conf.getInt(handlerCountKey, handlerCountDefault);
+    RPC.Server rpcServer = new RPC.Builder(conf)
+        .setProtocol(protocol)
+        .setInstance(instance)
+        .setBindAddress(bindHost)
+        .setPort(addr.getPort())
+        .setNumHandlers(numHandlers)
+        .setVerbose(false)
+        .setSecretManager(null)
+        .build();
+    return rpcServer;
+  }
+
+  @Override
+  public synchronized MountVolumeResponse mountVolume(
+      String userName, String volumeName) throws IOException {
+    return storageManager.isVolumeValid(userName, volumeName);
+  }
+
+  @Override
+  public List<VolumeInfo> listVolumes() throws IOException {
+    return listVolume(null);
+  }
+
+  @Override
+  public synchronized void createVolume(String userName, String volumeName,
+      long volumeSize, int blockSize) throws IOException {
+    LOG.info("Create volume received: userName: {} volumeName: {} " +
+            "volumeSize: {} blockSize: {}", userName, volumeName,
+        volumeSize, blockSize);
+    // It is important to create in-memory representation of the
+    // volume first, then writes to persistent storage (levelDB)
+    // such that it is guaranteed that when there is an entry in
+    // levelDB, the volume is allocated. (more like a UNDO log fashion)
+    // TODO: what if creation failed? we allocated containers but lost
+    // the reference to the volume and all it's containers. How to release
+    // the containers?
+    storageManager.createVolume(userName, volumeName, volumeSize, blockSize);
+    VolumeDescriptor volume = storageManager.getVolume(userName, volumeName);
+    if (volume == null) {
+      throw new IOException("Volume creation failed!");
+    }
+    String volumeKey = KeyUtil.getVolumeKey(userName, volumeName);
+    writeToPersistentStore(volumeKey.getBytes(encoding),
+        volume.toProtobuf().toByteArray());
+  }
+
+  @Override
+  public synchronized void deleteVolume(String userName,
+      String volumeName, boolean force) throws IOException {
+    LOG.info("Delete volume received: volume: {} {} ", volumeName, force);
+    storageManager.deleteVolume(userName, volumeName, force);
+    // being here means volume is successfully deleted now
+    String volumeKey = KeyUtil.getVolumeKey(userName, volumeName);
+    removeFromPersistentStore(volumeKey.getBytes(encoding));
+  }
+
+  // No need to synchronize on the following three methods, since write and
+  // remove's caller are synchronized. read's caller is the constructor and
+  // no other method call can happen at that time.
+  @VisibleForTesting
+  public void writeToPersistentStore(byte[] key, byte[] value) {
+    levelDBStore.put(key, value);
+  }
+
+  @VisibleForTesting
+  public void removeFromPersistentStore(byte[] key) {
+    levelDBStore.delete(key);
+  }
+
+  public void readFromPersistentStore() throws IOException {
+    try (DBIterator iter = levelDBStore.getIterator()) {
+      iter.seekToFirst();
+      while (iter.hasNext()) {
+        Map.Entry<byte[], byte[]> entry = iter.next();
+        String volumeKey = new String(entry.getKey(), encoding);
+        try {
+          VolumeDescriptor volumeDescriptor =
+              VolumeDescriptor.fromProtobuf(entry.getValue());
+          storageManager.addVolume(volumeDescriptor);
+        } catch (IOException e) {
+          LOG.error("Loading volume " + volumeKey + " error " + e);
+        }
+      }
+    }
+  }
+
+  @Override
+  public synchronized VolumeInfo infoVolume(String userName, String volumeName
+  ) throws IOException {
+    LOG.info("Info volume received: volume: {}", volumeName);
+    return storageManager.infoVolume(userName, volumeName);
+  }
+
+  @VisibleForTesting
+  public synchronized List<VolumeDescriptor> getAllVolumes() {
+    return storageManager.getAllVolume(null);
+  }
+
+  public synchronized List<VolumeDescriptor> getAllVolumes(String userName) {
+    return storageManager.getAllVolume(userName);
+  }
+
+  public synchronized void close() {
+    try {
+      levelDBStore.close();
+    } catch (IOException e) {
+      LOG.error("Error when closing levelDB " + e);
+    }
+  }
+
+  public synchronized void clean() {
+    try {
+      levelDBStore.close();
+      levelDBStore.destroy();
+    } catch (IOException e) {
+      LOG.error("Error when deleting levelDB " + e);
+    }
+  }
+
+  @Override
+  public synchronized List<VolumeInfo> listVolume(String userName)
+      throws IOException {
+    ArrayList<VolumeInfo> response = new ArrayList<>();
+    List<VolumeDescriptor> allVolumes =
+        storageManager.getAllVolume(userName);
+    for (VolumeDescriptor volume : allVolumes) {
+      VolumeInfo info =
+          new VolumeInfo(volume.getUserName(), volume.getVolumeName(),
+              volume.getVolumeSize(), volume.getBlockSize());
+      response.add(info);
+    }
+    return response;
+  }
+
+  public static void main(String[] args) throws Exception {
+    long version = RPC.getProtocolVersion(
+        StorageContainerLocationProtocolPB.class);
+    CblockUtils.activateConfigs();
+    OzoneConfiguration ozoneConf = new OzoneConfiguration();
+    String scmAddress = ozoneConf.get(DFS_CBLOCK_SCM_IPADDRESS_KEY,
+        DFS_CBLOCK_SCM_IPADDRESS_DEFAULT);
+    int scmPort = ozoneConf.getInt(DFS_CBLOCK_SCM_PORT_KEY,
+        DFS_CBLOCK_SCM_PORT_DEFAULT);
+    int containerSizeGB = ozoneConf.getInt(DFS_CBLOCK_CONTAINER_SIZE_GB_KEY,
+        DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT);
+    ContainerOperationClient.setContainerSizeB(containerSizeGB* OzoneConsts.GB);
+    InetSocketAddress address = new InetSocketAddress(scmAddress, scmPort);
+
+    ozoneConf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
+        OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
+    LOG.info(
+        "Creating StorageContainerLocationProtocol RPC client with address {}",
+        address);
+    RPC.setProtocolEngine(ozoneConf, StorageContainerLocationProtocolPB.class,
+        ProtobufRpcEngine.class);
+    StorageContainerLocationProtocolClientSideTranslatorPB client =
+        new StorageContainerLocationProtocolClientSideTranslatorPB(
+            RPC.getProxy(StorageContainerLocationProtocolPB.class, version,
+                address, UserGroupInformation.getCurrentUser(), ozoneConf,
+                NetUtils.getDefaultSocketFactory(ozoneConf),
+                Client.getRpcTimeout(ozoneConf)));
+    ScmClient storageClient = new ContainerOperationClient(
+        client, new XceiverClientManager(ozoneConf));
+    CBlockManager cbm = new CBlockManager(ozoneConf, storageClient);
+    cbm.start();
+    cbm.join();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CblockUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CblockUtils.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CblockUtils.java
new file mode 100644
index 0000000..99ffde0
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CblockUtils.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.cblock;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+
+import com.google.common.base.Optional;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_JSCSI_PORT_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_SERVICERPC_PORT_DEFAULT;
+import static org.apache.hadoop.hdsl.HdslUtils.getHostNameFromConfigKeys;
+import static org.apache.hadoop.hdsl.HdslUtils.getPortNumberFromConfigKeys;
+
+/**
+ * Generic stateless utility functions for CBlock components.
+ */
+public class CblockUtils {
+
+  private CblockUtils() {
+  }
+
+  /**
+   * Retrieve the socket address that is used by CBlock Service.
+   *
+   * @param conf
+   * @return Target InetSocketAddress for the CBlock Service endpoint.
+   */
+  public static InetSocketAddress getCblockServiceRpcAddr(Configuration conf) {
+    final Optional<String> host =
+        getHostNameFromConfigKeys(conf, DFS_CBLOCK_SERVICERPC_ADDRESS_KEY);
+
+    // If no port number is specified then we'll just try the defaultBindPort.
+    final Optional<Integer> port =
+        getPortNumberFromConfigKeys(conf, DFS_CBLOCK_SERVICERPC_ADDRESS_KEY);
+
+    return NetUtils.createSocketAddr(
+        host.or(DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT) + ":" + port
+            .or(DFS_CBLOCK_SERVICERPC_PORT_DEFAULT));
+  }
+
+  /**
+   * Retrieve the socket address that is used by CBlock Server.
+   *
+   * @param conf
+   * @return Target InetSocketAddress for the CBlock Server endpoint.
+   */
+  public static InetSocketAddress getCblockServerRpcAddr(Configuration conf) {
+    final Optional<String> host =
+        getHostNameFromConfigKeys(conf, DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY);
+
+    // If no port number is specified then we'll just try the defaultBindPort.
+    final Optional<Integer> port =
+        getPortNumberFromConfigKeys(conf, DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY);
+
+    return NetUtils.createSocketAddr(
+        host.or(DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT) + ":" + port
+            .or(DFS_CBLOCK_JSCSI_PORT_DEFAULT));
+  }
+
+  /**
+   * Parse size with size prefix string and return in bytes.
+   *
+   */
+  public static long parseSize(String volumeSizeArgs) throws IOException {
+    long multiplier = 1;
+
+    Pattern p = Pattern.compile("([0-9]+)([a-zA-Z]+)");
+    Matcher m = p.matcher(volumeSizeArgs);
+
+    if (!m.find()) {
+      throw new IOException("Invalid volume size args " + volumeSizeArgs);
+    }
+
+    int size = Integer.parseInt(m.group(1));
+    String s = m.group(2);
+
+    if (s.equalsIgnoreCase("MB") ||
+        s.equalsIgnoreCase("Mi")) {
+      multiplier = 1024L * 1024;
+    } else if (s.equalsIgnoreCase("GB") ||
+        s.equalsIgnoreCase("Gi")) {
+      multiplier = 1024L * 1024 * 1024;
+    } else if (s.equalsIgnoreCase("TB") ||
+        s.equalsIgnoreCase("Ti")) {
+      multiplier = 1024L * 1024 * 1024 * 1024;
+    } else {
+      throw new IOException("Invalid volume size args " + volumeSizeArgs);
+    }
+    return size * multiplier;
+  }
+
+  public static void activateConfigs(){
+    Configuration.addDefaultResource("hdfs-default.xml");
+    Configuration.addDefaultResource("hdfs-site.xml");
+    Configuration.addDefaultResource("ozone-default.xml");
+    Configuration.addDefaultResource("ozone-site.xml");
+    Configuration.addDefaultResource("cblock-default.xml");
+    Configuration.addDefaultResource("cblock-site.xml");
+
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java
new file mode 100644
index 0000000..ec38c9c
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.client;
+
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.proto.CBlockServiceProtocol;
+import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
+import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.ipc.ProtobufHelper;
+import org.apache.hadoop.ipc.ProtocolTranslator;
+import org.apache.hadoop.ipc.RPC;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * The client side implement of CBlockServiceProtocol.
+ */
+@InterfaceAudience.Private
+public final class CBlockServiceProtocolClientSideTranslatorPB
+    implements CBlockServiceProtocol, ProtocolTranslator, Closeable {
+
+  private final CBlockServiceProtocolPB rpcProxy;
+
+  public CBlockServiceProtocolClientSideTranslatorPB(
+      CBlockServiceProtocolPB rpcProxy) {
+    this.rpcProxy = rpcProxy;
+  }
+
+  @Override
+  public void close() throws IOException {
+    RPC.stopProxy(rpcProxy);
+  }
+
+  @Override
+  public void createVolume(String userName, String volumeName,
+      long volumeSize, int blockSize) throws IOException {
+    CBlockServiceProtocolProtos.CreateVolumeRequestProto.Builder req =
+        CBlockServiceProtocolProtos.CreateVolumeRequestProto.newBuilder();
+    req.setUserName(userName);
+    req.setVolumeName(volumeName);
+    req.setVolumeSize(volumeSize);
+    req.setBlockSize(blockSize);
+    try {
+      rpcProxy.createVolume(null, req.build());
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    }
+  }
+
+  @Override
+  public void deleteVolume(String userName, String volumeName, boolean force)
+      throws IOException {
+    CBlockServiceProtocolProtos.DeleteVolumeRequestProto.Builder req =
+        CBlockServiceProtocolProtos.DeleteVolumeRequestProto.newBuilder();
+    req.setUserName(userName);
+    req.setVolumeName(volumeName);
+    req.setForce(force);
+    try {
+      rpcProxy.deleteVolume(null, req.build());
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    }
+  }
+
+  @Override
+  public Object getUnderlyingProxyObject() {
+    return rpcProxy;
+  }
+
+  @Override
+  public VolumeInfo infoVolume(String userName, String volumeName)
+      throws IOException {
+    CBlockServiceProtocolProtos.InfoVolumeRequestProto.Builder req =
+        CBlockServiceProtocolProtos.InfoVolumeRequestProto.newBuilder();
+    req.setUserName(userName);
+    req.setVolumeName(volumeName);
+    try {
+      CBlockServiceProtocolProtos.InfoVolumeResponseProto resp =
+          rpcProxy.infoVolume(null, req.build());
+      return new VolumeInfo(resp.getVolumeInfo().getUserName(),
+          resp.getVolumeInfo().getVolumeName(),
+          resp.getVolumeInfo().getVolumeSize(),
+          resp.getVolumeInfo().getBlockSize(),
+          resp.getVolumeInfo().getUsage());
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    }
+  }
+
+  @Override
+  public List<VolumeInfo> listVolume(String userName) throws IOException {
+    CBlockServiceProtocolProtos.ListVolumeRequestProto.Builder req =
+        CBlockServiceProtocolProtos.ListVolumeRequestProto.newBuilder();
+    if (userName != null) {
+      req.setUserName(userName);
+    }
+    try {
+      CBlockServiceProtocolProtos.ListVolumeResponseProto resp =
+          rpcProxy.listVolume(null, req.build());
+      List<VolumeInfo> respList = new ArrayList<>();
+      for (CBlockServiceProtocolProtos.VolumeInfoProto entry :
+          resp.getVolumeEntryList()) {
+        VolumeInfo volumeInfo = new VolumeInfo(
+            entry.getUserName(), entry.getVolumeName(), entry.getVolumeSize(),
+            entry.getBlockSize());
+        respList.add(volumeInfo);
+      }
+      return respList;
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    } catch (Exception e) {
+      throw new IOException("got" + e.getCause() + " " + e.getMessage());
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java
new file mode 100644
index 0000000..4a3878a
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.client;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.cblock.CBlockConfigKeys;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import static org.apache.hadoop.cblock.CblockUtils.getCblockServiceRpcAddr;
+
+/**
+ * Implementation of client used by CBlock command line tool.
+ */
+public class CBlockVolumeClient {
+  private final CBlockServiceProtocolClientSideTranslatorPB cblockClient;
+
+  public CBlockVolumeClient(OzoneConfiguration conf) throws IOException {
+    this(conf, null);
+  }
+
+  public CBlockVolumeClient(OzoneConfiguration conf,
+      InetSocketAddress serverAddress) throws IOException {
+    InetSocketAddress address = serverAddress != null ? serverAddress :
+        getCblockServiceRpcAddr(conf);
+    long version = RPC.getProtocolVersion(CBlockServiceProtocolPB.class);
+    int rpcTimeout = Math.toIntExact(
+        conf.getTimeDuration(CBlockConfigKeys.DFS_CBLOCK_RPC_TIMEOUT,
+            CBlockConfigKeys.DFS_CBLOCK_RPC_TIMEOUT_DEFAULT,
+            TimeUnit.MILLISECONDS));
+    cblockClient = new CBlockServiceProtocolClientSideTranslatorPB(
+        RPC.getProtocolProxy(CBlockServiceProtocolPB.class, version,
+            address, UserGroupInformation.getCurrentUser(), conf,
+            NetUtils.getDefaultSocketFactory(conf), rpcTimeout, RetryPolicies
+                .retryUpToMaximumCountWithFixedSleep(
+                    300, 1, TimeUnit.SECONDS)).getProxy());
+  }
+
+  public void createVolume(String userName, String volumeName,
+      long volumeSize, int blockSize) throws IOException {
+    cblockClient.createVolume(userName, volumeName,
+        volumeSize, blockSize);
+  }
+
+  public void deleteVolume(String userName, String volumeName, boolean force)
+      throws IOException {
+    cblockClient.deleteVolume(userName, volumeName, force);
+  }
+
+  public VolumeInfo infoVolume(String userName, String volumeName)
+      throws IOException {
+    return cblockClient.infoVolume(userName, volumeName);
+  }
+
+  public List<VolumeInfo> listVolume(String userName)
+      throws IOException {
+    return cblockClient.listVolume(userName);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/package-info.java
new file mode 100644
index 0000000..761b71e
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.client;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java
new file mode 100644
index 0000000..8f6b82b
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.exception;
+
+import java.io.IOException;
+
+/**
+ * The exception class used in CBlock.
+ */
+public class CBlockException extends IOException {
+  public CBlockException(String message) {
+    super(message);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/package-info.java
new file mode 100644
index 0000000..268b8cb
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.exception;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java
new file mode 100644
index 0000000..04fe3a4
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper;
+
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Longs;
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import org.apache.hadoop.cblock.jscsiHelper.cache.impl.AsyncBlockWriter;
+import org.apache.hadoop.scm.XceiverClientSpi;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.utils.LevelDBStore;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Paths;
+
+/**
+ * The blockWriter task.
+ */
+public class BlockWriterTask implements Runnable {
+  private final LogicalBlock block;
+  private int tryCount;
+  private final ContainerCacheFlusher flusher;
+  private final String dbPath;
+  private final String fileName;
+  private final int maxRetryCount;
+
+  /**
+   * Constructs a BlockWriterTask.
+   *
+   * @param block - Block Information.
+   * @param flusher - ContainerCacheFlusher.
+   */
+  public BlockWriterTask(LogicalBlock block, ContainerCacheFlusher flusher,
+      String dbPath, int tryCount, String fileName, int maxRetryCount) {
+    this.block = block;
+    this.flusher = flusher;
+    this.dbPath = dbPath;
+    this.tryCount = tryCount;
+    this.fileName = fileName;
+    this.maxRetryCount = maxRetryCount;
+  }
+
+  /**
+   * When an object implementing interface <code>Runnable</code> is used
+   * to create a thread, starting the thread causes the object's
+   * <code>run</code> method to be called in that separately executing
+   * thread.
+   * <p>
+   * The general contract of the method <code>run</code> is that it may
+   * take any action whatsoever.
+   *
+   * @see Thread#run()
+   */
+  @Override
+  public void run() {
+    String containerName = null;
+    XceiverClientSpi client = null;
+    LevelDBStore levelDBStore = null;
+    String traceID = flusher.getTraceID(new File(dbPath), block.getBlockID());
+    flusher.getLOG().debug(
+        "Writing block to remote. block ID: {}", block.getBlockID());
+    try {
+      incTryCount();
+      Pipeline pipeline = flusher.getPipeline(this.dbPath, block.getBlockID());
+      client = flusher.getXceiverClientManager().acquireClient(pipeline);
+      containerName = pipeline.getContainerName();
+      byte[] keybuf = Longs.toByteArray(block.getBlockID());
+      byte[] data;
+      long startTime = Time.monotonicNow();
+      levelDBStore = flusher.getCacheDB(this.dbPath);
+      data = levelDBStore.get(keybuf);
+      Preconditions.checkNotNull(data);
+      long endTime = Time.monotonicNow();
+      Preconditions.checkState(data.length > 0, "Block data is zero length");
+      startTime = Time.monotonicNow();
+      ContainerProtocolCalls.writeSmallFile(client, containerName,
+          Long.toString(block.getBlockID()), data, traceID);
+      endTime = Time.monotonicNow();
+      flusher.getTargetMetrics().updateContainerWriteLatency(
+          endTime - startTime);
+      flusher.getLOG().debug("Time taken for Write Small File : {} ms",
+          endTime - startTime);
+
+      flusher.incrementRemoteIO();
+
+    } catch (Exception ex) {
+      flusher.getLOG().error("Writing of block:{} failed, We have attempted " +
+              "to write this block {} times to the container {}.Trace ID:{}",
+          block.getBlockID(), this.getTryCount(), containerName, traceID, ex);
+      writeRetryBlock(block);
+      if (ex instanceof IOException) {
+        flusher.getTargetMetrics().incNumWriteIOExceptionRetryBlocks();
+      } else {
+        flusher.getTargetMetrics().incNumWriteGenericExceptionRetryBlocks();
+      }
+      if (this.getTryCount() >= maxRetryCount) {
+        flusher.getTargetMetrics().incNumWriteMaxRetryBlocks();
+      }
+    } finally {
+      flusher.incFinishCount(fileName);
+      if (levelDBStore != null) {
+        flusher.releaseCacheDB(dbPath);
+      }
+      if(client != null) {
+        flusher.getXceiverClientManager().releaseClient(client);
+      }
+    }
+  }
+
+
+  private void writeRetryBlock(LogicalBlock currentBlock) {
+    boolean append = false;
+    String retryFileName =
+        String.format("%s.%d.%s.%s", AsyncBlockWriter.RETRY_LOG_PREFIX,
+            currentBlock.getBlockID(), Time.monotonicNow(), tryCount);
+    File logDir = new File(this.dbPath);
+    if (!logDir.exists() && !logDir.mkdirs()) {
+      flusher.getLOG().error(
+          "Unable to create the log directory, Critical error cannot continue");
+      return;
+    }
+    String log = Paths.get(this.dbPath, retryFileName).toString();
+    ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE / Byte.SIZE);
+    buffer.putLong(currentBlock.getBlockID());
+    buffer.flip();
+    try {
+      FileChannel channel = new FileOutputStream(log, append).getChannel();
+      channel.write(buffer);
+      channel.close();
+      flusher.processDirtyBlocks(this.dbPath, retryFileName);
+    } catch (IOException e) {
+      flusher.getTargetMetrics().incNumFailedRetryLogFileWrites();
+      flusher.getLOG().error("Unable to write the retry block. Block ID: {}",
+          currentBlock.getBlockID(), e);
+    }
+  }
+
+  /**
+   * Increments the try count. This is done each time we try this block
+   * write to the container.
+   */
+  private void incTryCount() {
+    tryCount++;
+  }
+
+  /**
+   * Get the retry count.
+   *
+   * @return int
+   */
+  public int getTryCount() {
+    return tryCount;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java
new file mode 100644
index 0000000..84b68e3
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper;
+
+import com.google.common.primitives.Longs;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.cblock.exception.CBlockException;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.proto.CBlockClientProtocol;
+import org.apache.hadoop.cblock.proto.MountVolumeResponse;
+import org.apache.hadoop.cblock.protocol.proto
+    .CBlockClientServerProtocolProtos.ContainerIDProto;
+import org.apache.hadoop.cblock.protocol.proto
+    .CBlockClientServerProtocolProtos.ListVolumesRequestProto;
+import org.apache.hadoop.cblock.protocol.proto
+    .CBlockClientServerProtocolProtos.ListVolumesResponseProto;
+import org.apache.hadoop.cblock.protocol.proto
+    .CBlockClientServerProtocolProtos.MountVolumeRequestProto;
+import org.apache.hadoop.cblock.protocol.proto
+    .CBlockClientServerProtocolProtos.MountVolumeResponseProto;
+import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos
+    .VolumeInfoProto;
+import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB;
+import org.apache.hadoop.ipc.ProtobufHelper;
+import org.apache.hadoop.ipc.ProtocolTranslator;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ * The client side of CBlockClientProtocol.
+ *
+ * CBlockClientProtocol is the protocol used between cblock client side
+ * and cblock manager (cblock client side is just the node where jscsi daemon
+ * process runs. a machines talks to jscsi daemon for mounting a volume).
+ *
+ * Right now, the only communication carried by this protocol is for client side
+ * to request mounting a volume.
+ */
+public class CBlockClientProtocolClientSideTranslatorPB
+    implements CBlockClientProtocol, ProtocolTranslator, Closeable {
+
+  private final CBlockClientServerProtocolPB rpcProxy;
+
+  public CBlockClientProtocolClientSideTranslatorPB(
+      CBlockClientServerProtocolPB rpcProxy) {
+    this.rpcProxy = rpcProxy;
+  }
+
+
+  @Override
+  public void close() throws IOException {
+    RPC.stopProxy(rpcProxy);
+  }
+
+  @Override
+  public Object getUnderlyingProxyObject() {
+    return rpcProxy;
+  }
+
+  @Override
+  public MountVolumeResponse mountVolume(
+      String userName, String volumeName) throws IOException {
+    MountVolumeRequestProto.Builder
+        request
+        = MountVolumeRequestProto
+        .newBuilder();
+    request.setUserName(userName);
+    request.setVolumeName(volumeName);
+    try {
+      MountVolumeResponseProto resp
+          = rpcProxy.mountVolume(null, request.build());
+      if (!resp.getIsValid()) {
+        throw new CBlockException(
+            "Not a valid volume:" + userName + ":" + volumeName);
+      }
+      List<Pipeline> containerIDs = new ArrayList<>();
+      HashMap<String, Pipeline> containerPipelines = new HashMap<>();
+      if (resp.getAllContainerIDsList().size() == 0) {
+        throw new CBlockException("Mount volume request returned no container");
+      }
+      for (ContainerIDProto containerID :
+          resp.getAllContainerIDsList()) {
+        if (containerID.hasPipeline()) {
+          // it should always have a pipeline only except for tests.
+          Pipeline p = Pipeline.getFromProtoBuf(containerID.getPipeline());
+          p.setData(Longs.toByteArray(containerID.getIndex()));
+          containerIDs.add(p);
+          containerPipelines.put(containerID.getContainerID(), p);
+        } else {
+          throw new CBlockException("ContainerID does not have pipeline!");
+        }
+      }
+      return new MountVolumeResponse(
+          resp.getIsValid(),
+          resp.getUserName(),
+          resp.getVolumeName(),
+          resp.getVolumeSize(),
+          resp.getBlockSize(),
+          containerIDs,
+          containerPipelines);
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    }
+  }
+
+  @Override
+  public List<VolumeInfo> listVolumes() throws IOException {
+    try {
+      List<VolumeInfo> result = new ArrayList<>();
+      ListVolumesResponseProto
+          listVolumesResponseProto = this.rpcProxy.listVolumes(null,
+          ListVolumesRequestProto.newBuilder()
+              .build());
+      for (VolumeInfoProto volumeInfoProto :
+          listVolumesResponseProto
+          .getVolumeEntryList()) {
+        result.add(new VolumeInfo(volumeInfoProto.getUserName(),
+            volumeInfoProto.getVolumeName(), volumeInfoProto.getVolumeSize(),
+            volumeInfoProto.getBlockSize()));
+      }
+      return result;
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java
new file mode 100644
index 0000000..2f35668
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java
@@ -0,0 +1,440 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.cblock.jscsiHelper.cache.CacheModule;
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.jscsi.target.storage.IStorageModule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_TRACE_IO;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_TRACE_IO_DEFAULT;
+
+/**
+ * The SCSI Target class for CBlockSCSIServer.
+ */
+final public class CBlockIStorageImpl implements IStorageModule {
+  private static final Logger LOGGER =
+      LoggerFactory.getLogger(CBlockIStorageImpl.class);
+  private static final Logger TRACER =
+      LoggerFactory.getLogger("TraceIO");
+
+  private CacheModule cache;
+  private final long volumeSize;
+  private final int blockSize;
+  private final String userName;
+  private final String volumeName;
+  private final boolean traceEnabled;
+  private final Configuration conf;
+  private final ContainerCacheFlusher flusher;
+  private List<Pipeline> fullContainerList;
+
+  /**
+   * private: constructs a SCSI Target.
+   *
+   * @param config - config
+   * @param userName - Username
+   * @param volumeName - Name of the volume
+   * @param volumeSize - Size of the volume
+   * @param blockSize - Size of the block
+   * @param fullContainerList - Ordered list of containers that make up this
+   * volume.
+   * @param flusher - flusher which is used to flush data from
+   *                  level db cache to containers
+   * @throws IOException - Throws IOException.
+   */
+  private CBlockIStorageImpl(Configuration config, String userName,
+      String volumeName, long volumeSize, int blockSize,
+      List<Pipeline> fullContainerList, ContainerCacheFlusher flusher) {
+    this.conf = config;
+    this.userName = userName;
+    this.volumeName = volumeName;
+    this.volumeSize = volumeSize;
+    this.blockSize = blockSize;
+    this.fullContainerList = new ArrayList<>(fullContainerList);
+    this.flusher = flusher;
+    this.traceEnabled = conf.getBoolean(DFS_CBLOCK_TRACE_IO,
+        DFS_CBLOCK_TRACE_IO_DEFAULT);
+  }
+
+  /**
+   * private: initialize the cache.
+   *
+   * @param xceiverClientManager - client manager that is used for creating new
+   * connections to containers.
+   * @param metrics  - target metrics to maintain metrics for target server
+   * @throws IOException - Throws IOException.
+   */
+  private void initCache(XceiverClientManager xceiverClientManager,
+      CBlockTargetMetrics metrics) throws IOException {
+    this.cache = CBlockLocalCache.newBuilder()
+        .setConfiguration(conf)
+        .setVolumeName(this.volumeName)
+        .setUserName(this.userName)
+        .setPipelines(this.fullContainerList)
+        .setClientManager(xceiverClientManager)
+        .setBlockSize(blockSize)
+        .setVolumeSize(volumeSize)
+        .setFlusher(flusher)
+        .setCBlockTargetMetrics(metrics)
+        .build();
+    this.cache.start();
+  }
+
+  /**
+   * Gets a new builder for CBlockStorageImpl.
+   *
+   * @return builder
+   */
+  public static Builder newBuilder() {
+    return new Builder();
+  }
+
+  /**
+   * Get Cache.
+   *
+   * @return - Cache
+   */
+  public CacheModule getCache() {
+    return cache;
+  }
+
+  /**
+   * Returns block size of this volume.
+   *
+   * @return int size of block for this volume.
+   */
+  @Override
+  public int getBlockSize() {
+    return blockSize;
+  }
+
+  /**
+   * Checks the index boundary of a block address.
+   *
+   * @param logicalBlockAddress the index of the first block of data to be read
+   * or written
+   * @param transferLengthInBlocks the total number of consecutive blocks about
+   * to be read or written
+   * @return 0 == Success, 1 indicates the LBA address is out of bounds and 2
+   * indicates that LBA + transfer size is out of bounds.
+   */
+  @Override
+  public int checkBounds(long logicalBlockAddress, int transferLengthInBlocks) {
+    long sizeInBlocks = volumeSize / blockSize;
+    int res = 0;
+    if (logicalBlockAddress < 0 || logicalBlockAddress >= sizeInBlocks) {
+      res = 1;
+    }
+
+    if (transferLengthInBlocks < 0 ||
+        logicalBlockAddress + transferLengthInBlocks > sizeInBlocks) {
+      if (res == 0) {
+        res = 2;
+      }
+    }
+    return res;
+  }
+
+  /**
+   * Number of blocks that make up this volume.
+   *
+   * @return long - count of blocks.
+   */
+  @Override
+  public long getSizeInBlocks() {
+    return volumeSize / blockSize;
+  }
+
+  /**
+   * Reads the number of bytes that can be read into the bytes buffer from the
+   * location indicated.
+   *
+   * @param bytes the array into which the data will be copied will be filled
+   * with data from storage
+   * @param storageIndex the position of the first byte to be copied
+   * @throws IOException
+   */
+  @Override
+  public void read(byte[] bytes, long storageIndex) throws IOException {
+    int startingIdxInBlock = (int) storageIndex % blockSize;
+    int idxInBytes = 0;
+    if (this.traceEnabled) {
+      TRACER.info("Task=ReadStart,length={},location={}",
+          bytes.length, storageIndex);
+    }
+    while (idxInBytes < bytes.length - 1) {
+      long blockId = (storageIndex + idxInBytes) / blockSize;
+      byte[] dataBytes;
+
+      try {
+        LogicalBlock block = this.cache.get(blockId);
+        dataBytes = block.getData().array();
+
+        if (this.traceEnabled) {
+          TRACER.info("Task=ReadBlock,BlockID={},length={},SHA={}",
+              blockId,
+              dataBytes.length,
+              dataBytes.length > 0 ? DigestUtils.sha256Hex(dataBytes) : null);
+        }
+      } catch (IOException e) {
+        // For an non-existing block cache.get will return a block with zero
+        // bytes filled. So any error here is a real error.
+        LOGGER.error("getting errors when reading data:" + e);
+        throw e;
+      }
+
+      int length = blockSize - startingIdxInBlock;
+      if (length > bytes.length - idxInBytes) {
+        length = bytes.length - idxInBytes;
+      }
+      if (dataBytes.length >= length) {
+        System.arraycopy(dataBytes, startingIdxInBlock, bytes, idxInBytes,
+            length);
+      }
+      startingIdxInBlock = 0;
+      idxInBytes += length;
+    }
+    if (this.traceEnabled) {
+      TRACER.info("Task=ReadEnd,length={},location={},SHA={}",
+          bytes.length, storageIndex, DigestUtils.sha256Hex(bytes));
+    }
+  }
+
+  @Override
+  public void write(byte[] bytes, long storageIndex) throws IOException {
+    int startingIdxInBlock = (int) storageIndex % blockSize;
+    int idxInBytes = 0;
+    if (this.traceEnabled) {
+      TRACER.info("Task=WriteStart,length={},location={},SHA={}",
+          bytes.length, storageIndex,
+          bytes.length > 0 ? DigestUtils.sha256Hex(bytes) : null);
+    }
+
+    ByteBuffer dataByte = ByteBuffer.allocate(blockSize);
+    while (idxInBytes < bytes.length - 1) {
+      long blockId = (storageIndex + idxInBytes) / blockSize;
+      int length = blockSize - startingIdxInBlock;
+      if (length > bytes.length - idxInBytes) {
+        length = bytes.length - idxInBytes;
+      }
+      System.arraycopy(bytes, idxInBytes, dataByte.array(), startingIdxInBlock,
+          length);
+      this.cache.put(blockId, dataByte.array());
+
+      if (this.traceEnabled) {
+        TRACER.info("Task=WriteBlock,BlockID={},length={},SHA={}",
+            blockId, dataByte.array().length,
+            dataByte.array().length > 0 ?
+                DigestUtils.sha256Hex(dataByte.array()) : null);
+      }
+      dataByte.clear();
+      startingIdxInBlock = 0;
+      idxInBytes += length;
+    }
+
+    if (this.traceEnabled) {
+      TRACER.info("Task=WriteEnd,length={},location={} ",
+          bytes.length, storageIndex);
+    }
+  }
+
+  @Override
+  public void close() throws IOException {
+    try {
+      cache.close();
+    } catch (IllegalStateException ise) {
+      LOGGER.error("Can not close the storage {}", ise);
+      throw ise;
+    }
+  }
+
+  /**
+   * Builder class for CBlocklocalCache.
+   */
+  public static class Builder {
+    private String userName;
+    private String volumeName;
+    private long volumeSize;
+    private int blockSize;
+    private List<Pipeline> containerList;
+    private Configuration conf;
+    private XceiverClientManager clientManager;
+    private ContainerCacheFlusher flusher;
+    private CBlockTargetMetrics metrics;
+
+    /**
+     * Constructs a builder.
+     */
+    Builder() {
+
+    }
+
+    public Builder setFlusher(ContainerCacheFlusher cacheFlusher) {
+      this.flusher = cacheFlusher;
+      return this;
+    }
+
+    /**
+     * set config.
+     *
+     * @param config - config
+     * @return Builder
+     */
+    public Builder setConf(Configuration config) {
+      this.conf = config;
+      return this;
+    }
+
+    /**
+     * set user name.
+     *
+     * @param cblockUserName - user name
+     * @return Builder
+     */
+    public Builder setUserName(String cblockUserName) {
+      this.userName = cblockUserName;
+      return this;
+    }
+
+    /**
+     * set volume name.
+     *
+     * @param cblockVolumeName -- volume name
+     * @return Builder
+     */
+    public Builder setVolumeName(String cblockVolumeName) {
+      this.volumeName = cblockVolumeName;
+      return this;
+    }
+
+    /**
+     * set volume size.
+     *
+     * @param cblockVolumeSize -- set volume size.
+     * @return Builder
+     */
+    public Builder setVolumeSize(long cblockVolumeSize) {
+      this.volumeSize = cblockVolumeSize;
+      return this;
+    }
+
+    /**
+     * set block size.
+     *
+     * @param cblockBlockSize -- block size
+     * @return Builder
+     */
+    public Builder setBlockSize(int cblockBlockSize) {
+      this.blockSize = cblockBlockSize;
+      return this;
+    }
+
+    /**
+     * Set contianer list.
+     *
+     * @param cblockContainerList - set the pipeline list
+     * @return Builder
+     */
+    public Builder setContainerList(List<Pipeline> cblockContainerList) {
+      this.containerList = cblockContainerList;
+      return this;
+    }
+
+    /**
+     * Set client manager.
+     *
+     * @param xceiverClientManager -- sets the client manager.
+     * @return Builder
+     */
+    public Builder setClientManager(XceiverClientManager xceiverClientManager) {
+      this.clientManager = xceiverClientManager;
+      return this;
+    }
+
+    /**
+     * Set Cblock Target Metrics.
+     *
+     * @param targetMetrics -- sets the cblock target metrics
+     * @return Builder
+     */
+    public Builder setCBlockTargetMetrics(CBlockTargetMetrics targetMetrics) {
+      this.metrics = targetMetrics;
+      return this;
+    }
+
+    /**
+     * Builds the CBlockStorageImpl.
+     *
+     * @return builds the CBlock Scsi Target.
+     */
+    public CBlockIStorageImpl build() throws IOException {
+      if (StringUtils.isBlank(userName)) {
+        throw new IllegalArgumentException("User name cannot be null or empty" +
+            ".");
+      }
+      if (StringUtils.isBlank(volumeName)) {
+        throw new IllegalArgumentException("Volume name cannot be null or " +
+            "empty");
+      }
+
+      if (volumeSize < 1) {
+        throw new IllegalArgumentException("Volume size cannot be negative or" +
+            " zero.");
+      }
+
+      if (blockSize < 1) {
+        throw new IllegalArgumentException("Block size cannot be negative or " +
+            "zero.");
+      }
+
+      if (containerList == null || containerList.size() == 0) {
+        throw new IllegalArgumentException("Container list cannot be null or " +
+            "empty");
+      }
+      if (clientManager == null) {
+        throw new IllegalArgumentException("Client manager cannot be null");
+      }
+      if (conf == null) {
+        throw new IllegalArgumentException("Configuration cannot be null");
+      }
+
+      if (flusher == null) {
+        throw new IllegalArgumentException("Flusher Cannot be null.");
+      }
+      CBlockIStorageImpl impl = new CBlockIStorageImpl(this.conf, this.userName,
+          this.volumeName, this.volumeSize, this.blockSize, this.containerList,
+          this.flusher);
+      impl.initCache(this.clientManager, this.metrics);
+      return impl;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java
new file mode 100644
index 0000000..6367c61
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper;
+
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.proto.MountVolumeResponse;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * This class is the handler of CBlockManager used by target server
+ * to communicate with CBlockManager.
+ *
+ * More specifically, this class will expose local methods to target
+ * server, and make RPC calls to CBlockManager accordingly
+ */
+public class CBlockManagerHandler {
+
+  private final CBlockClientProtocolClientSideTranslatorPB handler;
+
+  public CBlockManagerHandler(
+      CBlockClientProtocolClientSideTranslatorPB handler) {
+    this.handler = handler;
+  }
+
+  public MountVolumeResponse mountVolume(
+      String userName, String volumeName) throws IOException {
+    return handler.mountVolume(userName, volumeName);
+  }
+
+  public List<VolumeInfo> listVolumes() throws IOException {
+    return handler.listVolumes();
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[47/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java
new file mode 100644
index 0000000..b9ec462
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java
@@ -0,0 +1,427 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.storage;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.cblock.CBlockConfigKeys;
+import org.apache.hadoop.cblock.exception.CBlockException;
+import org.apache.hadoop.cblock.meta.ContainerDescriptor;
+import org.apache.hadoop.cblock.meta.VolumeDescriptor;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.proto.MountVolumeResponse;
+import org.apache.hadoop.cblock.util.KeyUtil;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.hdsl.protocol.proto.HdslProtos;
+import org.apache.hadoop.scm.client.ScmClient;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * This class maintains the key space of CBlock, more specifically, the
+ * volume to container mapping. The core data structure
+ * is a map from users to their volumes info, where volume info is a handler
+ * to a volume, containing information for IO on that volume and a storage
+ * client responsible for talking to the SCM.
+ */
+public class StorageManager {
+  private static final Logger LOGGER =
+      LoggerFactory.getLogger(StorageManager.class);
+  private final ScmClient storageClient;
+  private final int numThreads;
+  private static final int MAX_THREADS =
+      Runtime.getRuntime().availableProcessors() * 2;
+  private static final int MAX_QUEUE_CAPACITY = 1024;
+  private final String cblockId;
+
+  /**
+   * We will NOT have the situation where same kv pair getting
+   * processed, but it is possible to have multiple kv pair being
+   * processed at same time.
+   *
+   * So using just ConcurrentHashMap should be sufficient
+   *
+   * Again since currently same user accessing from multiple places
+   * is not allowed, no need to consider concurrency of volume map
+   * within one user
+   */
+  private ConcurrentHashMap<String, HashMap<String, VolumeDescriptor>>
+      user2VolumeMap;
+  // size of an underlying container.
+  // TODO : assuming all containers are of the same size
+  private long containerSizeB;
+
+  public StorageManager(ScmClient storageClient,
+      OzoneConfiguration ozoneConfig, String cblockId) throws IOException {
+    this.storageClient = storageClient;
+    this.user2VolumeMap = new ConcurrentHashMap<>();
+    this.containerSizeB = storageClient.getContainerSize(null);
+    this.numThreads =
+        ozoneConfig.getInt(CBlockConfigKeys.DFS_CBLOCK_MANAGER_POOL_SIZE,
+            CBlockConfigKeys.DFS_CBLOCK_MANAGER_POOL_SIZE_DEFAULT);
+    this.cblockId = cblockId;
+  }
+
+  /**
+   * This call will put the volume into in-memory map.
+   *
+   * more specifically, make the volume discoverable on jSCSI server
+   * and keep it's reference in-memory for look up.
+   * @param userName the user name of the volume.
+   * @param volumeName the name of the volume,
+   * @param volume a {@link VolumeDescriptor} object encapsulating the
+   *               information about the volume.
+   */
+  private void makeVolumeReady(String userName, String volumeName,
+      VolumeDescriptor volume) {
+    HashMap<String, VolumeDescriptor> userVolumes;
+    if (user2VolumeMap.containsKey(userName)) {
+      userVolumes = user2VolumeMap.get(userName);
+    } else {
+      userVolumes = new HashMap<>();
+      user2VolumeMap.put(userName, userVolumes);
+    }
+    userVolumes.put(volumeName, volume);
+  }
+
+  /**
+   * Called by CBlockManager to add volumes read from persistent store into
+   * memory, need to contact SCM to setup the reference to the containers given
+   * their id.
+   *
+   * Only for failover process where container meta info is read from
+   * persistent store, and containers themselves are alive.
+   *
+   * TODO : Currently, this method is not being called as failover process
+   * is not implemented yet.
+   *
+   * @param volumeDescriptor a {@link VolumeDescriptor} object encapsulating
+   *                         the information about a volume.
+   * @throws IOException when adding the volume failed. e.g. volume already
+   * exist, or no more container available.
+   */
+  public synchronized void addVolume(VolumeDescriptor volumeDescriptor)
+      throws IOException{
+    String userName = volumeDescriptor.getUserName();
+    String volumeName = volumeDescriptor.getVolumeName();
+    LOGGER.info("addVolume:" + userName + ":" + volumeName);
+    if (user2VolumeMap.containsKey(userName)
+        && user2VolumeMap.get(userName).containsKey(volumeName)) {
+      throw new CBlockException("Volume already exist for "
+          + userName + ":" + volumeName);
+    }
+    // the container ids are read from levelDB, setting up the
+    // container handlers here.
+    String[] containerIds = volumeDescriptor.getContainerIDs();
+
+    for (String containerId : containerIds) {
+      try {
+        Pipeline pipeline = storageClient.getContainer(containerId);
+        ContainerDescriptor containerDescriptor =
+            new ContainerDescriptor(containerId);
+        containerDescriptor.setPipeline(pipeline);
+        volumeDescriptor.addContainer(containerDescriptor);
+      } catch (IOException e) {
+        LOGGER.error("Getting container failed! Container:{} error:{}",
+            containerId, e);
+        throw e;
+      }
+    }
+    // now ready to put into in-memory map.
+    makeVolumeReady(userName, volumeName, volumeDescriptor);
+  }
+
+  private class CreateContainerTask implements Runnable {
+    private final VolumeDescriptor volume;
+    private final int containerIdx;
+    private final ArrayList<String> containerIds;
+    private final AtomicInteger numFailed;
+
+    CreateContainerTask(VolumeDescriptor volume, int containerIdx,
+                        ArrayList<String> containerIds,
+                        AtomicInteger numFailed) {
+      this.volume = volume;
+      this.containerIdx = containerIdx;
+      this.containerIds = containerIds;
+      this.numFailed = numFailed;
+    }
+
+    /**
+     * When an object implementing interface <code>Runnable</code> is used
+     * to create a thread, starting the thread causes the object's
+     * <code>run</code> method to be called in that separately executing
+     * thread.
+     * <p>
+     * The general contract of the method <code>run</code> is that it may
+     * take any action whatsoever.
+     *
+     * @see Thread#run()
+     */
+    public void run() {
+      ContainerDescriptor container = null;
+      try {
+        Pipeline pipeline = storageClient.createContainer(
+            HdslProtos.ReplicationType.STAND_ALONE,
+            HdslProtos.ReplicationFactor.ONE,
+            KeyUtil.getContainerName(volume.getUserName(),
+                volume.getVolumeName(), containerIdx), cblockId);
+
+        container = new ContainerDescriptor(pipeline.getContainerName());
+
+        container.setPipeline(pipeline);
+        container.setContainerIndex(containerIdx);
+        volume.addContainer(container);
+        containerIds.set(containerIdx, container.getContainerID());
+      } catch (Exception e) {
+        numFailed.incrementAndGet();
+        if (container != null) {
+          LOGGER.error("Error creating container Container:{}:" +
+              " index:{} error:{}", container.getContainerID(),
+              containerIdx, e);
+        } else {
+          LOGGER.error("Error creating container.", e);
+        }
+      }
+    }
+  }
+
+  private boolean createVolumeContainers(VolumeDescriptor volume) {
+    ArrayList<String> containerIds = new ArrayList<>();
+    ThreadPoolExecutor executor = new ThreadPoolExecutor(
+        Math.min(numThreads, MAX_THREADS),
+        MAX_THREADS, 1, TimeUnit.SECONDS,
+        new ArrayBlockingQueue<>(MAX_QUEUE_CAPACITY),
+        new ThreadPoolExecutor.CallerRunsPolicy());
+
+    AtomicInteger numFailedCreates = new AtomicInteger(0);
+    long allocatedSize = 0;
+    int containerIdx = 0;
+    while (allocatedSize < volume.getVolumeSize()) {
+      // adding null to allocate space in ArrayList
+      containerIds.add(containerIdx, null);
+      Runnable task = new CreateContainerTask(volume, containerIdx,
+          containerIds, numFailedCreates);
+      executor.submit(task);
+      allocatedSize += containerSizeB;
+      containerIdx += 1;
+    }
+
+    // issue the command and then wait for it to finish
+    executor.shutdown();
+    try {
+      executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
+    } catch (InterruptedException e) {
+      LOGGER.error("Error creating volume:{} error:{}",
+          volume.getVolumeName(), e);
+      executor.shutdownNow();
+      Thread.currentThread().interrupt();
+    }
+
+    volume.setContainerIDs(containerIds);
+    return numFailedCreates.get() == 0;
+  }
+
+  private void deleteContainer(String containerID, boolean force) {
+    try {
+      Pipeline pipeline = storageClient.getContainer(containerID);
+      storageClient.deleteContainer(pipeline, force);
+    } catch (Exception e) {
+      LOGGER.error("Error deleting container Container:{} error:{}",
+          containerID, e);
+    }
+  }
+
+  private void deleteVolumeContainers(List<String> containers, boolean force)
+      throws CBlockException {
+    ThreadPoolExecutor executor = new ThreadPoolExecutor(
+        Math.min(numThreads, MAX_THREADS),
+        MAX_THREADS, 1, TimeUnit.SECONDS,
+        new ArrayBlockingQueue<>(MAX_QUEUE_CAPACITY),
+        new ThreadPoolExecutor.CallerRunsPolicy());
+
+    for (String deleteContainer : containers) {
+      if (deleteContainer != null) {
+        Runnable task = () -> deleteContainer(deleteContainer, force);
+        executor.submit(task);
+      }
+    }
+
+    // issue the command and then wait for it to finish
+    executor.shutdown();
+    try {
+      executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
+    } catch (InterruptedException e) {
+      LOGGER.error("Error deleting containers error:{}", e);
+      executor.shutdownNow();
+      Thread.currentThread().interrupt();
+    }
+  }
+
+  /**
+   * Called by CBlock server when creating a fresh volume. The core
+   * logic is adding needed information into in-memory meta data.
+   *
+   * @param userName the user name of the volume.
+   * @param volumeName the name of the volume.
+   * @param volumeSize the size of the volume.
+   * @param blockSize the block size of the volume.
+   * @throws CBlockException when the volume can not be created.
+   */
+  public synchronized void createVolume(String userName, String volumeName,
+      long volumeSize, int blockSize) throws CBlockException {
+    LOGGER.debug("createVolume:" + userName + ":" + volumeName);
+    if (user2VolumeMap.containsKey(userName)
+        && user2VolumeMap.get(userName).containsKey(volumeName)) {
+      throw new CBlockException("Volume already exist for "
+          + userName + ":" + volumeName);
+    }
+    if (volumeSize < blockSize) {
+      throw new CBlockException("Volume size smaller than block size? " +
+          "volume size:" + volumeSize + " block size:" + blockSize);
+    }
+    VolumeDescriptor volume
+        = new VolumeDescriptor(userName, volumeName, volumeSize, blockSize);
+    boolean success = createVolumeContainers(volume);
+    if (!success) {
+      // cleanup the containers and throw the exception
+      deleteVolumeContainers(volume.getContainerIDsList(), true);
+      throw new CBlockException("Error when creating volume:" + volumeName);
+    }
+    makeVolumeReady(userName, volumeName, volume);
+  }
+
+  /**
+   * Called by CBlock server to delete a specific volume. Mainly
+   * to check whether it can be deleted, and remove it from in-memory meta
+   * data.
+   *
+   * @param userName the user name of the volume.
+   * @param volumeName the name of the volume.
+   * @param force if set to false, only delete volume it is empty, otherwise
+   *              throw exception. if set to true, delete regardless.
+   * @throws CBlockException when the volume can not be deleted.
+   */
+  public synchronized void deleteVolume(String userName, String volumeName,
+      boolean force) throws CBlockException {
+    if (!user2VolumeMap.containsKey(userName)
+        || !user2VolumeMap.get(userName).containsKey(volumeName)) {
+      throw new CBlockException("Deleting non-exist volume "
+          + userName + ":" + volumeName);
+    }
+    if (!force && !user2VolumeMap.get(userName).get(volumeName).isEmpty()) {
+      throw new CBlockException("Deleting a non-empty volume without force!");
+    }
+    VolumeDescriptor volume = user2VolumeMap.get(userName).remove(volumeName);
+    deleteVolumeContainers(volume.getContainerIDsList(), force);
+    if (user2VolumeMap.get(userName).size() == 0) {
+      user2VolumeMap.remove(userName);
+    }
+  }
+
+  /**
+   * Called by CBlock server to get information of a specific volume.
+   *
+   * @param userName the user name of the volume.
+   * @param volumeName the name of the volume.
+   * @return a {@link VolumeInfo} object encapsulating the information of the
+   * volume.
+   * @throws CBlockException when the information can not be retrieved.
+   */
+  public synchronized VolumeInfo infoVolume(String userName, String volumeName)
+      throws CBlockException {
+    if (!user2VolumeMap.containsKey(userName)
+        || !user2VolumeMap.get(userName).containsKey(volumeName)) {
+      throw new CBlockException("Getting info for non-exist volume "
+          + userName + ":" + volumeName);
+    }
+    return user2VolumeMap.get(userName).get(volumeName).getInfo();
+  }
+
+  /**
+   * Called by CBlock server to check whether the given volume can be
+   * mounted, i.e. whether it can be found in the meta data.
+   *
+   * return a {@link MountVolumeResponse} with isValid flag to indicate
+   * whether the volume can be mounted or not.
+   *
+   * @param userName the user name of the volume.
+   * @param volumeName the name of the volume
+   * @return a {@link MountVolumeResponse} object encapsulating whether the
+   * volume is valid, and if yes, the requried information for client to
+   * read/write the volume.
+   */
+  public synchronized MountVolumeResponse isVolumeValid(
+      String userName, String volumeName) {
+    if (!user2VolumeMap.containsKey(userName)
+        || !user2VolumeMap.get(userName).containsKey(volumeName)) {
+      // in the case of invalid volume, no need to set any value other than
+      // isValid flag.
+      return new MountVolumeResponse(false, null, null, 0, 0, null, null);
+    }
+    VolumeDescriptor volume = user2VolumeMap.get(userName).get(volumeName);
+    return new MountVolumeResponse(true, userName,
+        volumeName, volume.getVolumeSize(), volume.getBlockSize(),
+        volume.getContainerPipelines(), volume.getPipelines());
+  }
+
+  /**
+   * Called by CBlock manager to list all volumes.
+   *
+   * @param userName the userName whose volume to be listed, if set to null,
+   *                 all volumes will be listed.
+   * @return a list of {@link VolumeDescriptor} representing all volumes
+   * requested.
+   */
+  public synchronized List<VolumeDescriptor> getAllVolume(String userName) {
+    ArrayList<VolumeDescriptor> allVolumes = new ArrayList<>();
+    if (userName == null) {
+      for (Map.Entry<String, HashMap<String, VolumeDescriptor>> entry
+          : user2VolumeMap.entrySet()) {
+        allVolumes.addAll(entry.getValue().values());
+      }
+    } else {
+      if (user2VolumeMap.containsKey(userName)) {
+        allVolumes.addAll(user2VolumeMap.get(userName).values());
+      }
+    }
+    return allVolumes;
+  }
+
+  /**
+   * Only for testing the behavior of create/delete volumes.
+   */
+  @VisibleForTesting
+  public VolumeDescriptor getVolume(String userName, String volumeName) {
+    if (!user2VolumeMap.containsKey(userName)
+        || !user2VolumeMap.get(userName).containsKey(volumeName)) {
+      return null;
+    }
+    return user2VolumeMap.get(userName).get(volumeName);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/package-info.java
new file mode 100644
index 0000000..4426e6d
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.storage;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java
new file mode 100644
index 0000000..beb9e32
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.util;
+
+/**
+ * A simply class that generates key mappings. (e.g. from (userName, volumeName)
+ * pair to a single string volumeKey.
+ */
+public final class KeyUtil {
+  public static String getVolumeKey(String userName, String volumeName) {
+    return userName + ":" + volumeName;
+  }
+
+  public static String getContainerName(String userName, String volumeName,
+      int containerID) {
+    return getVolumeKey(userName, volumeName) + "#" + containerID;
+  }
+
+  public static String getUserNameFromVolumeKey(String key) {
+    return key.split(":")[0];
+  }
+
+  public static String getVolumeFromVolumeKey(String key) {
+    return key.split(":")[1];
+  }
+
+  public static boolean isValidVolumeKey(String key) {
+    return key.contains(":");
+  }
+
+  private KeyUtil() {
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/package-info.java
new file mode 100644
index 0000000..5b9aa0c
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.util;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/proto/CBlockClientServerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/proto/CBlockClientServerProtocol.proto b/hadoop-cblock/server/src/main/proto/CBlockClientServerProtocol.proto
new file mode 100644
index 0000000..160b254
--- /dev/null
+++ b/hadoop-cblock/server/src/main/proto/CBlockClientServerProtocol.proto
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * These .proto interfaces are private and unstable.
+ * Please see http://wiki.apache.org/hadoop/Compatibility
+ * for what changes are allowed for a *unstable* .proto interface.
+ */
+option java_package = "org.apache.hadoop.cblock.protocol.proto";
+option java_outer_classname = "CBlockClientServerProtocolProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.cblock;
+
+import "hdsl.proto";
+import "CBlockServiceProtocol.proto";
+/**
+* This message is sent from CBlock client side to CBlock server to
+* mount a volume specified by owner name and volume name.
+*
+* Right now, this is the only communication between client and server.
+* After the volume is mounted, CBlock client will talk to containers
+* by itself, nothing to do with CBlock server.
+*/
+message MountVolumeRequestProto {
+    required string userName = 1;
+    required string volumeName = 2;
+}
+
+/**
+* This message is sent from CBlock server to CBlock client as response
+* of mount a volume. It checks the whether the volume is valid to access
+* at all.(e.g. volume exist)
+*
+* And include enough information (volume size, block size, list of
+* containers for this volume) for client side to perform read/write on
+* the volume.
+*/
+message MountVolumeResponseProto {
+    required bool isValid = 1;
+    optional string userName = 2;
+    optional string volumeName = 3;
+    optional uint64 volumeSize = 4;
+    optional uint32 blockSize = 5;
+    repeated ContainerIDProto allContainerIDs = 6;
+}
+
+/**
+* This message include ID of container which can be used to locate the
+* container. Since the order of containers needs to be maintained, also
+* includes a index field to verify the correctness of the order.
+*/
+message ContainerIDProto {
+    required string containerID = 1;
+    required uint64 index = 2;
+    // making pipeline optional to be compatible with exisiting tests
+    optional hadoop.hdsl.Pipeline pipeline = 3;
+}
+
+
+message ListVolumesRequestProto {
+
+}
+
+message ListVolumesResponseProto {
+    repeated VolumeInfoProto volumeEntry = 1;
+}
+
+
+service CBlockClientServerProtocolService {
+    /**
+    * mount the volume.
+    */
+    rpc mountVolume(MountVolumeRequestProto) returns (MountVolumeResponseProto);
+
+    rpc listVolumes(ListVolumesRequestProto) returns(ListVolumesResponseProto);
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/proto/CBlockServiceProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/proto/CBlockServiceProtocol.proto b/hadoop-cblock/server/src/main/proto/CBlockServiceProtocol.proto
new file mode 100644
index 0000000..36e4b59
--- /dev/null
+++ b/hadoop-cblock/server/src/main/proto/CBlockServiceProtocol.proto
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * These .proto interfaces are private and unstable.
+ * Please see http://wiki.apache.org/hadoop/Compatibility
+ * for what changes are allowed for a *unstable* .proto interface.
+ */
+
+option java_package = "org.apache.hadoop.cblock.protocol.proto";
+option java_outer_classname = "CBlockServiceProtocolProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.cblock;
+
+/**
+* This message is sent to CBlock server to create a volume. Creating
+* volume requries four parameters: owner of the volume, name of the volume
+* size of volume and block size of the volume.
+*/
+message CreateVolumeRequestProto {
+    required string userName = 1;
+    required string volumeName = 2;
+    required uint64 volumeSize = 3;
+    optional uint32 blockSize = 4 [default = 4096];
+}
+
+/**
+* Empty response message.
+*/
+message CreateVolumeResponseProto {
+
+}
+
+/**
+* This message is sent to CBlock server to delete a volume. The volume
+* is specified by owner name and volume name. If force is set to
+* false, volume will be deleted only if it is empty. Otherwise delete it
+* regardless.
+*/
+message DeleteVolumeRequestProto {
+    required string userName = 1;
+    required string volumeName = 2;
+    optional bool force = 3;
+}
+
+/**
+* Empty response message.
+*/
+message DeleteVolumeResponseProto {
+
+}
+
+/**
+* This message is sent to CBlock server to request info of a volume. The
+* volume is specified by owner name and volume name.
+*/
+message InfoVolumeRequestProto {
+    required string userName = 1;
+    required string volumeName = 2;
+}
+
+/**
+* This message describes the information of a volume.
+* Currently, the info includes the volume creation parameters and a number
+* as the usage of the volume, in terms of number of bytes.
+*/
+message VolumeInfoProto {
+    required string userName = 1;
+    required string volumeName = 2;
+    required uint64 volumeSize = 3;
+    required uint64 blockSize = 4;
+    optional uint64 usage = 5;
+    // TODO : potentially volume ACL
+}
+
+/**
+* This message is sent from CBlock server as response of info volume request.
+*/
+message InfoVolumeResponseProto {
+    optional VolumeInfoProto volumeInfo = 1;
+}
+
+/**
+* This message is sent to CBlock server to list all available volume.
+*/
+message ListVolumeRequestProto {
+    optional string userName = 1;
+}
+
+/**
+* This message is sent from CBlock server as response of volume listing.
+*/
+message ListVolumeResponseProto {
+    repeated VolumeInfoProto volumeEntry = 1;
+}
+
+service CBlockServiceProtocolService {
+    /**
+    * Create a volume.
+    */
+    rpc createVolume(CreateVolumeRequestProto) returns(CreateVolumeResponseProto);
+
+    /**
+    * Delete a volume.
+    */
+    rpc deleteVolume(DeleteVolumeRequestProto) returns(DeleteVolumeResponseProto);
+
+    /**
+    * Get info of a volume.
+    */
+    rpc infoVolume(InfoVolumeRequestProto) returns(InfoVolumeResponseProto);
+
+    /**
+    * List all available volumes.
+    */
+    rpc listVolume(ListVolumeRequestProto) returns(ListVolumeResponseProto);
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/resources/cblock-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/resources/cblock-default.xml b/hadoop-cblock/server/src/main/resources/cblock-default.xml
new file mode 100644
index 0000000..ebf36cd
--- /dev/null
+++ b/hadoop-cblock/server/src/main/resources/cblock-default.xml
@@ -0,0 +1,347 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<!-- Do not modify this file directly.  Instead, copy entries that you -->
+<!-- wish to modify from this file into ozone-site.xml and change them -->
+<!-- there.  If ozone-site.xml does not already exist, create it.      -->
+
+<!--Tags supported are OZONE, CBLOCK, MANAGEMENT, SECURITY, PERFORMANCE,   -->
+<!--DEBUG, CLIENT, SERVER, KSM, SCM, CRITICAL, RATIS, CONTAINER, REQUIRED, -->
+<!--REST, STORAGE, PIPELINE, STANDALONE                                    -->
+
+<configuration>
+
+  <!--CBlock Settings-->
+  <property>
+    <name>dfs.cblock.block.buffer.flush.interval</name>
+    <value>60s</value>
+    <tag>CBLOCK, PERFORMANCE</tag>
+    <description>
+      Controls the frequency at this the local cache flushes the
+      blocks to the remote containers.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.cache.block.buffer.size</name>
+    <value>512</value>
+    <tag>CBLOCK, PERFORMANCE</tag>
+    <description>
+      Size of the local cache for blocks. So cache size will be block
+      size multiplied by this number.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.cache.core.min.pool.size</name>
+    <value>16</value>
+    <tag>CBLOCK, PERFORMANCE</tag>
+    <description>
+      A minimum number of threads in the pool that cBlock cache will
+      use for the background I/O to remote containers.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.cache.max.pool.size</name>
+    <value>256</value>
+    <tag>CBLOCK, PERFORMANCE</tag>
+    <description>
+      Maximum number of threads in the pool that cBlock cache will
+      use for background I/O to remote containers.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.cache.keep.alive</name>
+    <value>60s</value>
+    <tag>CBLOCK, PERFORMANCE</tag>
+    <description>
+      If the cblock cache has no I/O, then the threads in the cache
+      pool are kept idle for this amount of time before shutting down.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.cache.leveldb.cache.size.mb</name>
+    <value>256</value>
+    <tag>CBLOCK, PERFORMANCE</tag>
+    <description>
+      The amount of physical memory allocated to the local cache. The
+      SCSI driver will allocate this much RAM cache instances.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.cache.max.retry</name>
+    <value>65536</value>
+    <tag>CBLOCK, PERFORMANCE</tag>
+    <description>
+      If the local cache is enabled then, CBlock writes to the local
+      cache when I/O happens. Then the background I/O threads write this
+      block to the remote containers. This value controls how many times the
+      background thread should attempt to do I/O to the remote containers
+      before giving up.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.cache.queue.size.in.kb</name>
+    <value>256</value>
+    <tag>CBLOCK, PERFORMANCE</tag>
+    <description>
+      Size of the in memory cache queue, that is flushed to local
+      disk.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.cache.thread.priority</name>
+    <value>5</value>
+    <tag>CBLOCK, PERFORMANCE</tag>
+    <description>
+      Priority of cache flusher thread, affecting the relative performance of
+      write and read. Supported values are 1, 5, 10.
+      Use 10 for high priority and 1 for low priority.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.container.size.gb</name>
+    <value>5</value>
+    <tag>CBLOCK, MANAGEMENT</tag>
+    <description>
+      The size of ozone container in the number of GBs. Note that
+      this is not setting container size for ozone. This setting is
+      instructing CBlock to manage containers at a standard size.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.disk.cache.path</name>
+    <value>${hadoop.tmp.dir}/cblockCacheDB</value>
+    <tag>CBLOCK, REQUIRED</tag>
+    <description>
+      The default path for the cblock local cache. If the cblock
+      local cache is enabled, then it must be set to a valid path. This cache
+      *should* be mapped to the fastest disk on a given machine, For example,
+      an SSD drive would be a good idea. Currently, all mounted disk on a
+      data node is mapped to a single path, so having a large number of IOPS
+      is essential.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.jscsi-address</name>
+    <value/>
+    <tag>CBLOCK, MANAGEMENT</tag>
+    <description>
+      The address that cblock will be bind to, should be a host:port
+      format, This setting is required for cblock server to start.
+      This address to be used by jscsi to mount volume.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.jscsi.cblock.server.address</name>
+    <value>127.0.0.1</value>
+    <tag>CBLOCK, MANAGEMENT</tag>
+    <description>
+      The address local jscsi server will use to talk to cblock manager.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.jscsi.port</name>
+    <value>9811</value>
+    <tag>CBLOCK, MANAGEMENT</tag>
+    <description>
+      The port on CBlockManager node for jSCSI to talk to.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.jscsi.rpc-bind-host</name>
+    <value>0.0.0.0</value>
+    <tag>CBLOCK, MANAGEMENT</tag>
+    <description>
+      The actual address the cblock jscsi rpc server will bind to. If
+      this optional address is set, it overrides only the hostname portion of
+      dfs.cblock.jscsi-address.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.jscsi.server.address</name>
+    <value>0.0.0.0</value>
+    <tag>CBLOCK, MANAGEMENT</tag>
+    <description>
+      The address that jscsi server will be running, it is nice have one
+      local jscsi server for each client(Linux JSCSI client) that tries to
+      mount cblock.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.manager.pool.size</name>
+    <value>16</value>
+    <tag>CBLOCK, PERFORMANCE</tag>
+    <description>
+      Number of active threads that cblock manager will use for container
+      operations. The maximum number of the threads are limited to the
+      processor count * 2.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.rpc.timeout</name>
+    <value>300s</value>
+    <tag>CBLOCK, MANAGEMENT</tag>
+    <description>
+      RPC timeout used for cblock CLI operations. When you
+      create very large disks, like 5TB, etc. The number of containers
+      allocated in the system is huge. It is will 5TB/5GB, which is 1000
+      containers. The client CLI might timeout even though the cblock manager
+      creates the specified disk. This value allows the user to wait for a
+      longer period.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.scm.ipaddress</name>
+    <value>127.0.0.1</value>
+    <tag>CBLOCK, MANAGEMENT</tag>
+    <description>
+      IP address used by cblock to connect to SCM.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.scm.port</name>
+    <value>9860</value>
+    <tag>CBLOCK, MANAGEMENT</tag>
+    <description>
+      Port used by cblock to connect to SCM.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.service.handler.count</name>
+    <value>10</value>
+    <tag>CBLOCK, MANAGEMENT</tag>
+    <description>
+      Default number of handlers for CBlock service rpc.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.service.leveldb.path</name>
+    <value>${hadoop.tmp.dir}/cblock_server.dat</value>
+    <tag>CBLOCK, REQUIRED</tag>
+    <description>
+      Default path for the cblock meta data store.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.service.rpc-bind-host</name>
+    <value>0.0.0.0</value>
+    <tag>CBLOCK, MANAGEMENT</tag>
+    <description>
+      The actual address the cblock service RPC server will bind to.
+      If the optional address is set, it overrides only the hostname portion of
+      dfs.cblock.servicerpc-address.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.servicerpc-address</name>
+    <value/>
+    <tag>CBLOCK, MANAGEMENT, REQUIRED</tag>
+    <description>
+      The address that cblock will be bind to, should be a host:port
+      format, this setting is required for cblock server to start.
+      This address is used for cblock management operations like create, delete,
+      info and list volumes
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.short.circuit.io</name>
+    <value>false</value>
+    <tag>CBLOCK, PERFORMANCE</tag>
+    <description>
+      Enables use of the local cache in cblock. Enabling this allows
+      I/O against the local cache and background threads do actual I/O against
+      the
+      containers.
+    </description>
+  </property>
+  <property>
+    <name>dfs.cblock.trace.io</name>
+    <value>false</value>
+    <tag>CBLOCK, DEBUG</tag>
+    <description>Default flag for enabling trace io, Trace I/O logs all I/O with
+      hashes of
+      data. This is useful for detecting things like data corruption.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.cblock.iscsi.advertised.ip</name>
+    <value>0.0.0.0</value>
+    <tag>CBLOCK</tag>
+    <description>
+      IP address returned during the iscsi discovery.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.cblock.iscsi.advertised.port</name>
+    <value>3260</value>
+    <tag>CBLOCK</tag>
+    <description>
+      TCP port returned during the iscsi discovery.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.cblock.kubernetes.dynamic-provisioner.enabled</name>
+    <value>false</value>
+    <tag>CBLOCK, KUBERNETES</tag>
+    <description>Flag to enable automatic creation of cblocks and
+      kubernetes PersitentVolumes in kubernetes environment.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.cblock.kubernetes.cblock-user</name>
+    <value>iqn.2001-04.org.apache.hadoop</value>
+    <tag>CBLOCK, KUBERNETES</tag>
+    <description>CBlock user to use for the dynamic provisioner.
+      This user will own all of the auto-created cblocks.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.cblock.kubernetes.configfile</name>
+    <value></value>
+    <tag>CBLOCK, KUBERNETES</tag>
+    <description>Location of the kubernetes configuration file
+      to access the kubernetes cluster. Not required inside a pod
+      as the default service account will be if this value is
+      empty.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.cblock.iscsi.advertised.ip</name>
+    <value></value>
+    <tag>CBLOCK, KUBERNETES</tag>
+    <description>IP where the cblock target server is available
+      from the kubernetes nodes. Usually it's a cluster ip address
+      which is defined by a deployed Service.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.cblock.iscsi.advertised.port</name>
+    <value>3260</value>
+    <tag>CBLOCK, KUBERNETES</tag>
+    <description>Port where the cblock target server is available
+      from the kubernetes nodes. Could be different from the
+      listening port if jscsi is behind a Service.
+    </description>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java
new file mode 100644
index 0000000..e1eb36f
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java
@@ -0,0 +1,456 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import com.google.common.primitives.Longs;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics;
+import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher;
+import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.apache.hadoop.scm.XceiverClientSpi;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.MiniOzoneClassicCluster;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.concurrent.TimeoutException;
+
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+    DFS_CBLOCK_DISK_CACHE_PATH_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+    DFS_CBLOCK_TRACE_IO;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+    DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+    DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL;
+
+/**
+ * Tests for Local Cache Buffer Manager.
+ */
+public class TestBufferManager {
+  private final static long GB = 1024 * 1024 * 1024;
+  private final static int KB = 1024;
+  private static MiniOzoneCluster cluster;
+  private static OzoneConfiguration config;
+  private static StorageContainerLocationProtocolClientSideTranslatorPB
+      storageContainerLocationClient;
+  private static XceiverClientManager xceiverClientManager;
+
+  @BeforeClass
+  public static void init() throws IOException {
+    config = new OzoneConfiguration();
+    String path = GenericTestUtils.getTempPath(
+        TestBufferManager.class.getSimpleName());
+    config.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+    config.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+    config.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+    cluster = new MiniOzoneClassicCluster.Builder(config)
+        .numDataNodes(1).setHandlerType("distributed").build();
+    storageContainerLocationClient = cluster
+        .createStorageContainerLocationClient();
+    xceiverClientManager = new XceiverClientManager(config);
+  }
+
+  @AfterClass
+  public static void shutdown() throws InterruptedException {
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+    IOUtils.cleanupWithLogger(null, storageContainerLocationClient, cluster);
+  }
+
+  /**
+   * createContainerAndGetPipeline creates a set of containers and returns the
+   * Pipelines that define those containers.
+   *
+   * @param count - Number of containers to create.
+   * @return - List of Pipelines.
+   * @throws IOException
+   */
+  private List<Pipeline> createContainerAndGetPipeline(int count)
+      throws IOException {
+    List<Pipeline> containerPipelines = new LinkedList<>();
+    for (int x = 0; x < count; x++) {
+      String traceID = "trace" + RandomStringUtils.randomNumeric(4);
+      String containerName = "container" + RandomStringUtils.randomNumeric(10);
+      Pipeline pipeline =
+          storageContainerLocationClient.allocateContainer(
+              xceiverClientManager.getType(),
+              xceiverClientManager.getFactor(), containerName, "CBLOCK");
+      XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline);
+      ContainerProtocolCalls.createContainer(client, traceID);
+      // This step is needed since we set private data on pipelines, when we
+      // read the list from CBlockServer. So we mimic that action here.
+      pipeline.setData(Longs.toByteArray(x));
+      containerPipelines.add(pipeline);
+      xceiverClientManager.releaseClient(client);
+    }
+    return containerPipelines;
+  }
+
+  /**
+   * This test writes some block to the cache and then shuts down the cache.
+   * The cache is then restarted to check that the
+   * correct number of blocks are read from Dirty Log
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testEmptyBlockBufferHandling() throws IOException,
+      InterruptedException, TimeoutException {
+    // Create a new config so that this tests write metafile to new location
+    OzoneConfiguration flushTestConfig = new OzoneConfiguration();
+    String path = GenericTestUtils
+        .getTempPath(TestBufferManager.class.getSimpleName()
+            + RandomStringUtils.randomNumeric(4));
+    flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+    flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+    String userName = "user" + RandomStringUtils.randomNumeric(4);
+    String data = RandomStringUtils.random(4 * KB);
+    List<Pipeline> pipelines = createContainerAndGetPipeline(10);
+    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
+        xceiverClientManager, metrics);
+    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+        .setConfiguration(flushTestConfig)
+        .setVolumeName(volumeName)
+        .setUserName(userName)
+        .setPipelines(pipelines)
+        .setClientManager(xceiverClientManager)
+        .setBlockSize(4 * KB)
+        .setVolumeSize(50 * GB)
+        .setFlusher(flusher)
+        .setCBlockTargetMetrics(metrics)
+        .build();
+    cache.start();
+    // Write data to the cache
+    cache.put(1, data.getBytes(StandardCharsets.UTF_8));
+    Assert.assertEquals(0, metrics.getNumDirectBlockWrites());
+    Assert.assertEquals(1, metrics.getNumWriteOps());
+    cache.put(2, data.getBytes(StandardCharsets.UTF_8));
+    Assert.assertEquals(0, metrics.getNumDirectBlockWrites());
+    Assert.assertEquals(2, metrics.getNumWriteOps());
+
+    // Store the previous block buffer position
+    Assert.assertEquals(2, metrics.getNumBlockBufferUpdates());
+    // Simulate a shutdown by closing the cache
+    cache.close();
+    Thread.sleep(1000);
+    Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
+    Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
+    Assert.assertEquals(2 * (Long.SIZE/ Byte.SIZE),
+                                metrics.getNumBytesDirtyLogWritten());
+    Assert.assertEquals(0, metrics.getNumFailedBlockBufferFlushes());
+    Assert.assertEquals(0, metrics.getNumInterruptedBufferWaits());
+
+    // Restart cache and check that right number of entries are read
+    CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher newFlusher =
+        new ContainerCacheFlusher(flushTestConfig,
+            xceiverClientManager, newMetrics);
+    CBlockLocalCache newCache = CBlockLocalCache.newBuilder()
+        .setConfiguration(flushTestConfig)
+        .setVolumeName(volumeName)
+        .setUserName(userName)
+        .setPipelines(pipelines)
+        .setClientManager(xceiverClientManager)
+        .setBlockSize(4 * KB)
+        .setVolumeSize(50 * GB)
+        .setFlusher(newFlusher)
+        .setCBlockTargetMetrics(newMetrics)
+        .build();
+    newCache.start();
+    Thread fllushListenerThread = new Thread(newFlusher);
+    fllushListenerThread.setDaemon(true);
+    fllushListenerThread.start();
+
+    Thread.sleep(5000);
+    Assert.assertEquals(metrics.getNumBlockBufferUpdates(),
+                                      newMetrics.getNumDirtyLogBlockRead());
+    Assert.assertEquals(newMetrics.getNumDirtyLogBlockRead()
+            * (Long.SIZE/ Byte.SIZE), newMetrics.getNumBytesDirtyLogReads());
+    // Now shutdown again, nothing should be flushed
+    newFlusher.shutdown();
+    Assert.assertEquals(0, newMetrics.getNumBlockBufferUpdates());
+    Assert.assertEquals(0, newMetrics.getNumBytesDirtyLogWritten());
+  }
+
+  @Test
+  public void testPeriodicFlush() throws IOException,
+      InterruptedException, TimeoutException{
+    // Create a new config so that this tests write metafile to new location
+    OzoneConfiguration flushTestConfig = new OzoneConfiguration();
+    String path = GenericTestUtils
+        .getTempPath(TestBufferManager.class.getSimpleName()
+            + RandomStringUtils.randomNumeric(4));
+    flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+    flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+    flushTestConfig
+        .setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, 5, SECONDS);
+
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+    String userName = "user" + RandomStringUtils.randomNumeric(4);
+    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
+        xceiverClientManager, metrics);
+    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+        .setConfiguration(flushTestConfig)
+        .setVolumeName(volumeName)
+        .setUserName(userName)
+        .setPipelines(createContainerAndGetPipeline(10))
+        .setClientManager(xceiverClientManager)
+        .setBlockSize(4 * KB)
+        .setVolumeSize(50 * GB)
+        .setFlusher(flusher)
+        .setCBlockTargetMetrics(metrics)
+        .build();
+    cache.start();
+    Thread.sleep(8000);
+    // Ticks will be at 5s, 10s and so on, so this count should be 1
+    Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
+    // Nothing pushed to cache, so nothing should be written
+    Assert.assertEquals(0, metrics.getNumBytesDirtyLogWritten());
+    Assert.assertEquals(0, metrics.getNumBlockBufferFlushCompleted());
+    cache.close();
+    // After close, another trigger should happen but still no data written
+    Assert.assertEquals(2, metrics.getNumBlockBufferFlushTriggered());
+    Assert.assertEquals(0, metrics.getNumBytesDirtyLogWritten());
+    Assert.assertEquals(0, metrics.getNumBlockBufferFlushCompleted());
+    Assert.assertEquals(0, metrics.getNumFailedBlockBufferFlushes());
+  }
+
+  @Test
+  public void testSingleBufferFlush() throws IOException,
+      InterruptedException, TimeoutException {
+    // Create a new config so that this tests write metafile to new location
+    OzoneConfiguration flushTestConfig = new OzoneConfiguration();
+    String path = GenericTestUtils
+        .getTempPath(TestBufferManager.class.getSimpleName()
+            + RandomStringUtils.randomNumeric(4));
+    flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+    flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+    String userName = "user" + RandomStringUtils.randomNumeric(4);
+    String data = RandomStringUtils.random(4 * KB);
+    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
+        xceiverClientManager, metrics);
+    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+        .setConfiguration(flushTestConfig)
+        .setVolumeName(volumeName)
+        .setUserName(userName)
+        .setPipelines(createContainerAndGetPipeline(10))
+        .setClientManager(xceiverClientManager)
+        .setBlockSize(4 * KB)
+        .setVolumeSize(50 * GB)
+        .setFlusher(flusher)
+        .setCBlockTargetMetrics(metrics)
+        .build();
+    cache.start();
+
+    for (int i = 0; i < 511; i++) {
+      cache.put(i, data.getBytes(StandardCharsets.UTF_8));
+    }
+    // After writing 511 block no flush should happen
+    Assert.assertEquals(0, metrics.getNumBlockBufferFlushTriggered());
+    Assert.assertEquals(0, metrics.getNumBlockBufferFlushCompleted());
+
+
+    // After one more block it should
+    cache.put(512, data.getBytes(StandardCharsets.UTF_8));
+    Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
+    Thread.sleep(1000);
+    Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
+    cache.close();
+    Assert.assertEquals(512 * (Long.SIZE / Byte.SIZE),
+                                metrics.getNumBytesDirtyLogWritten());
+  }
+
+  @Test
+  public void testMultipleBuffersFlush() throws IOException,
+      InterruptedException, TimeoutException {
+    // Create a new config so that this tests write metafile to new location
+    OzoneConfiguration flushTestConfig = new OzoneConfiguration();
+    String path = GenericTestUtils
+        .getTempPath(TestBufferManager.class.getSimpleName()
+            + RandomStringUtils.randomNumeric(4));
+    flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+    flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+    flushTestConfig
+        .setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, 120, SECONDS);
+
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+    String userName = "user" + RandomStringUtils.randomNumeric(4);
+    String data = RandomStringUtils.random(4 * KB);
+    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
+        xceiverClientManager, metrics);
+    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+        .setConfiguration(flushTestConfig)
+        .setVolumeName(volumeName)
+        .setUserName(userName)
+        .setPipelines(createContainerAndGetPipeline(10))
+        .setClientManager(xceiverClientManager)
+        .setBlockSize(4 * KB)
+        .setVolumeSize(50 * GB)
+        .setFlusher(flusher)
+        .setCBlockTargetMetrics(metrics)
+        .build();
+    cache.start();
+
+    for (int i = 0; i < 4; i++) {
+      for (int j = 0; j < 512; j++) {
+        cache.put(i * 512 + j, data.getBytes(StandardCharsets.UTF_8));
+      }
+      // Flush should be triggered after every 512 block write
+      Assert.assertEquals(i + 1, metrics.getNumBlockBufferFlushTriggered());
+    }
+    Assert.assertEquals(0, metrics.getNumIllegalDirtyLogFiles());
+    Assert.assertEquals(0, metrics.getNumFailedDirtyLogFileDeletes());
+    cache.close();
+    Assert.assertEquals(4 * 512 * (Long.SIZE / Byte.SIZE),
+        metrics.getNumBytesDirtyLogWritten());
+    Assert.assertEquals(5, metrics.getNumBlockBufferFlushTriggered());
+    Assert.assertEquals(4, metrics.getNumBlockBufferFlushCompleted());
+  }
+
+  @Test
+  public void testSingleBlockFlush() throws IOException,
+      InterruptedException, TimeoutException{
+    // Create a new config so that this tests write metafile to new location
+    OzoneConfiguration flushTestConfig = new OzoneConfiguration();
+    String path = GenericTestUtils
+        .getTempPath(TestBufferManager.class.getSimpleName()
+            + RandomStringUtils.randomNumeric(4));
+    flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+    flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+    flushTestConfig
+        .setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL,
+            5, SECONDS);
+
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+    String userName = "user" + RandomStringUtils.randomNumeric(4);
+    String data = RandomStringUtils.random(4 * KB);
+    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
+        xceiverClientManager, metrics);
+    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+        .setConfiguration(flushTestConfig)
+        .setVolumeName(volumeName)
+        .setUserName(userName)
+        .setPipelines(createContainerAndGetPipeline(10))
+        .setClientManager(xceiverClientManager)
+        .setBlockSize(4 * KB)
+        .setVolumeSize(50 * GB)
+        .setFlusher(flusher)
+        .setCBlockTargetMetrics(metrics)
+        .build();
+    cache.start();
+    cache.put(0, data.getBytes(StandardCharsets.UTF_8));
+    Thread.sleep(8000);
+    // Ticks will be at 5s, 10s and so on, so this count should be 1
+    Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
+    // 1 block written to cache, which should be flushed
+    Assert.assertEquals(8, metrics.getNumBytesDirtyLogWritten());
+    Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
+    cache.close();
+    // After close, another trigger should happen but no data should be written
+    Assert.assertEquals(2, metrics.getNumBlockBufferFlushTriggered());
+    Assert.assertEquals(8, metrics.getNumBytesDirtyLogWritten());
+    Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
+    Assert.assertEquals(0, metrics.getNumFailedBlockBufferFlushes());
+  }
+
+  @Test
+  public void testRepeatedBlockWrites() throws IOException,
+      InterruptedException, TimeoutException{
+    // Create a new config so that this tests write metafile to new location
+    OzoneConfiguration flushTestConfig = new OzoneConfiguration();
+    String path = GenericTestUtils
+        .getTempPath(TestBufferManager.class.getSimpleName()
+            + RandomStringUtils.randomNumeric(4));
+    flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+    flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+    String userName = "user" + RandomStringUtils.randomNumeric(4);
+    String data = RandomStringUtils.random(4 * KB);
+    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
+        xceiverClientManager, metrics);
+    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+        .setConfiguration(flushTestConfig)
+        .setVolumeName(volumeName)
+        .setUserName(userName)
+        .setPipelines(createContainerAndGetPipeline(10))
+        .setClientManager(xceiverClientManager)
+        .setBlockSize(4 * KB)
+        .setVolumeSize(50 * GB)
+        .setFlusher(flusher)
+        .setCBlockTargetMetrics(metrics)
+        .build();
+    Thread fllushListenerThread = new Thread(flusher);
+    fllushListenerThread.setDaemon(true);
+    fllushListenerThread.start();
+    cache.start();
+    for (int i = 0; i < 512; i++) {
+      cache.put(i, data.getBytes(StandardCharsets.UTF_8));
+    }
+    Assert.assertEquals(512, metrics.getNumWriteOps());
+    Assert.assertEquals(512, metrics.getNumBlockBufferUpdates());
+    Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
+    Thread.sleep(5000);
+    Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
+
+
+    for (int i = 0; i < 512; i++) {
+      cache.put(i, data.getBytes(StandardCharsets.UTF_8));
+    }
+    Assert.assertEquals(1024, metrics.getNumWriteOps());
+    Assert.assertEquals(1024, metrics.getNumBlockBufferUpdates());
+    Assert.assertEquals(2, metrics.getNumBlockBufferFlushTriggered());
+
+    Thread.sleep(5000);
+    Assert.assertEquals(0, metrics.getNumWriteIOExceptionRetryBlocks());
+    Assert.assertEquals(0, metrics.getNumWriteGenericExceptionRetryBlocks());
+    Assert.assertEquals(2, metrics.getNumBlockBufferFlushCompleted());
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockConfigurationFields.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockConfigurationFields.java
new file mode 100644
index 0000000..4139ac6
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockConfigurationFields.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import org.apache.hadoop.conf.TestConfigurationFieldsBase;
+
+/**
+ * Tests if configuration constants documented in ozone-defaults.xml.
+ */
+public class TestCBlockConfigurationFields extends TestConfigurationFieldsBase {
+
+  @Override
+  public void initializeMemberVariables() {
+    xmlFilename = new String("cblock-default.xml");
+    configurationClasses =
+        new Class[] {CBlockConfigKeys.class};
+    errorIfMissingConfigProps = true;
+    errorIfMissingXmlProps = true;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java
new file mode 100644
index 0000000..d995ba6
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java
@@ -0,0 +1,377 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import com.google.common.primitives.Longs;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics;
+import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher;
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.MiniOzoneClassicCluster;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.hdsl.protocol.proto.HdslProtos.LifeCycleState;
+import org.apache.hadoop.hdsl.protocol.proto.HdslProtos.ReplicationFactor;
+import org.apache.hadoop.hdsl.protocol.proto.HdslProtos.ReplicationType;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.apache.hadoop.scm.XceiverClientSpi;
+import org.apache.hadoop.scm.container.common.helpers.PipelineChannel;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.scm.protocolPB
+    .StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_DISK_CACHE_PATH_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_TRACE_IO;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE;
+
+/**
+ * Tests for Cblock read write functionality.
+ */
+public class TestCBlockReadWrite {
+  private final static long GB = 1024 * 1024 * 1024;
+  private final static int KB = 1024;
+  private static MiniOzoneCluster cluster;
+  private static OzoneConfiguration config;
+  private static StorageContainerLocationProtocolClientSideTranslatorPB
+      storageContainerLocationClient;
+  private static XceiverClientManager xceiverClientManager;
+
+  @BeforeClass
+  public static void init() throws IOException {
+    config = new OzoneConfiguration();
+    String path = GenericTestUtils
+        .getTempPath(TestCBlockReadWrite.class.getSimpleName());
+    config.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+    config.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+    config.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+    cluster = new MiniOzoneClassicCluster.Builder(config)
+        .numDataNodes(1)
+        .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
+    storageContainerLocationClient = cluster
+        .createStorageContainerLocationClient();
+    xceiverClientManager = new XceiverClientManager(config);
+  }
+
+  @AfterClass
+  public static void shutdown() throws InterruptedException {
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+    IOUtils.cleanupWithLogger(null, storageContainerLocationClient, cluster);
+  }
+
+  /**
+   * getContainerPipelines creates a set of containers and returns the
+   * Pipelines that define those containers.
+   *
+   * @param count - Number of containers to create.
+   * @return - List of Pipelines.
+   * @throws IOException throws Exception
+   */
+  private List<Pipeline> getContainerPipeline(int count) throws IOException {
+    List<Pipeline> containerPipelines = new LinkedList<>();
+    for (int x = 0; x < count; x++) {
+      String traceID = "trace" + RandomStringUtils.randomNumeric(4);
+      String containerName = "container" + RandomStringUtils.randomNumeric(10);
+      Pipeline pipeline =
+          storageContainerLocationClient.allocateContainer(
+              xceiverClientManager.getType(),
+              xceiverClientManager.getFactor(), containerName, "CBLOCK");
+      XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline);
+      ContainerProtocolCalls.createContainer(client, traceID);
+      // This step is needed since we set private data on pipelines, when we
+      // read the list from CBlockServer. So we mimic that action here.
+      pipeline.setData(Longs.toByteArray(x));
+      containerPipelines.add(pipeline);
+    }
+    return containerPipelines;
+  }
+
+  /**
+   * This test creates a cache and performs a simple write / read.
+   * The operations are done by bypassing the cache.
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testDirectIO() throws IOException,
+      InterruptedException, TimeoutException {
+    OzoneConfiguration cConfig = new OzoneConfiguration();
+    cConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, false);
+    cConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+    final long blockID = 0;
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+    String userName = "user" + RandomStringUtils.randomNumeric(4);
+    String data = RandomStringUtils.random(4 * KB);
+    String dataHash = DigestUtils.sha256Hex(data);
+    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher flusher = new ContainerCacheFlusher(cConfig,
+        xceiverClientManager, metrics);
+    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+        .setConfiguration(cConfig)
+        .setVolumeName(volumeName)
+        .setUserName(userName)
+        .setPipelines(getContainerPipeline(10))
+        .setClientManager(xceiverClientManager)
+        .setBlockSize(4 * KB)
+        .setVolumeSize(50 * GB)
+        .setFlusher(flusher)
+        .setCBlockTargetMetrics(metrics)
+        .build();
+    cache.start();
+    Assert.assertFalse(cache.isShortCircuitIOEnabled());
+    cache.put(blockID, data.getBytes(StandardCharsets.UTF_8));
+    Assert.assertEquals(1, metrics.getNumDirectBlockWrites());
+    Assert.assertEquals(1, metrics.getNumWriteOps());
+    // Please note that this read is directly from remote container
+    LogicalBlock block = cache.get(blockID);
+    Assert.assertEquals(1, metrics.getNumReadOps());
+    Assert.assertEquals(0, metrics.getNumReadCacheHits());
+    Assert.assertEquals(1, metrics.getNumReadCacheMiss());
+    Assert.assertEquals(0, metrics.getNumReadLostBlocks());
+    Assert.assertEquals(0, metrics.getNumFailedDirectBlockWrites());
+
+    cache.put(blockID + 1, data.getBytes(StandardCharsets.UTF_8));
+    Assert.assertEquals(2, metrics.getNumDirectBlockWrites());
+    Assert.assertEquals(2, metrics.getNumWriteOps());
+    Assert.assertEquals(0, metrics.getNumFailedDirectBlockWrites());
+    // Please note that this read is directly from remote container
+    block = cache.get(blockID + 1);
+    Assert.assertEquals(2, metrics.getNumReadOps());
+    Assert.assertEquals(0, metrics.getNumReadCacheHits());
+    Assert.assertEquals(2, metrics.getNumReadCacheMiss());
+    Assert.assertEquals(0, metrics.getNumReadLostBlocks());
+    String readHash = DigestUtils.sha256Hex(block.getData().array());
+    Assert.assertEquals("File content does not match.", dataHash, readHash);
+    GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
+    cache.close();
+  }
+
+  /**
+   * This test writes some block to the cache and then shuts down the cache
+   * The cache is then restarted with "short.circuit.io" disable to check
+   * that the blocks are read correctly from the container.
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testContainerWrites() throws IOException,
+      InterruptedException, TimeoutException {
+    // Create a new config so that this tests write metafile to new location
+    OzoneConfiguration flushTestConfig = new OzoneConfiguration();
+    String path = GenericTestUtils
+        .getTempPath(TestCBlockReadWrite.class.getSimpleName());
+    flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+    flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+    flushTestConfig.setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, 3,
+        TimeUnit.SECONDS);
+    XceiverClientManager xcm = new XceiverClientManager(flushTestConfig);
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+    String userName = "user" + RandomStringUtils.randomNumeric(4);
+
+    int numUniqueBlocks = 4;
+    String[] data = new String[numUniqueBlocks];
+    String[] dataHash = new String[numUniqueBlocks];
+    for (int i = 0; i < numUniqueBlocks; i++) {
+      data[i] = RandomStringUtils.random(4 * KB);
+      dataHash[i] = DigestUtils.sha256Hex(data[i]);
+    }
+
+    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
+        xcm, metrics);
+    List<Pipeline> pipelines = getContainerPipeline(10);
+    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+        .setConfiguration(flushTestConfig)
+        .setVolumeName(volumeName)
+        .setUserName(userName)
+        .setPipelines(pipelines)
+        .setClientManager(xcm)
+        .setBlockSize(4 * KB)
+        .setVolumeSize(50 * GB)
+        .setFlusher(flusher)
+        .setCBlockTargetMetrics(metrics)
+        .build();
+    cache.start();
+    Thread flushListenerThread = new Thread(flusher);
+    flushListenerThread.setDaemon(true);
+    flushListenerThread.start();
+    Assert.assertTrue(cache.isShortCircuitIOEnabled());
+    // Write data to the cache
+    for (int i = 0; i < 512; i++) {
+      cache.put(i, data[i % numUniqueBlocks].getBytes(StandardCharsets.UTF_8));
+    }
+    // Close the cache and flush the data to the containers
+    cache.close();
+    Assert.assertEquals(0, metrics.getNumDirectBlockWrites());
+    Assert.assertEquals(512, metrics.getNumWriteOps());
+    Thread.sleep(3000);
+    flusher.shutdown();
+    Assert.assertTrue(metrics.getNumBlockBufferFlushTriggered() > 1);
+    Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
+    Assert.assertEquals(0, metrics.getNumWriteIOExceptionRetryBlocks());
+    Assert.assertEquals(0, metrics.getNumWriteGenericExceptionRetryBlocks());
+    Assert.assertEquals(0, metrics.getNumFailedReleaseLevelDB());
+    // Now disable DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO and restart cache
+    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, false);
+    CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher newFlusher =
+        new ContainerCacheFlusher(flushTestConfig, xcm, newMetrics);
+    CBlockLocalCache newCache = CBlockLocalCache.newBuilder()
+        .setConfiguration(flushTestConfig)
+        .setVolumeName(volumeName)
+        .setUserName(userName)
+        .setPipelines(pipelines)
+        .setClientManager(xcm)
+        .setBlockSize(4 * KB)
+        .setVolumeSize(50 * GB)
+        .setFlusher(newFlusher)
+        .setCBlockTargetMetrics(newMetrics)
+        .build();
+    newCache.start();
+    Assert.assertFalse(newCache.isShortCircuitIOEnabled());
+    // this read will be from the container, also match the hash
+    for (int i = 0; i < 512; i++) {
+      LogicalBlock block = newCache.get(i);
+      String readHash = DigestUtils.sha256Hex(block.getData().array());
+      Assert.assertEquals("File content does not match, for index:"
+          + i, dataHash[i % numUniqueBlocks], readHash);
+    }
+    Assert.assertEquals(0, newMetrics.getNumReadLostBlocks());
+    Assert.assertEquals(0, newMetrics.getNumFailedReadBlocks());
+    newCache.close();
+    newFlusher.shutdown();
+  }
+
+  @Test
+  public void testRetryLog() throws IOException,
+      InterruptedException, TimeoutException {
+    // Create a new config so that this tests write metafile to new location
+    OzoneConfiguration flushTestConfig = new OzoneConfiguration();
+    String path = GenericTestUtils
+        .getTempPath(TestCBlockReadWrite.class.getSimpleName());
+    flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+    flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+    flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+    flushTestConfig.setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL,
+        3,
+        TimeUnit.SECONDS);
+
+    int numblocks = 10;
+    flushTestConfig.setInt(DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE, numblocks);
+
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+    String userName = "user" + RandomStringUtils.randomNumeric(4);
+    String data = RandomStringUtils.random(4 * KB);
+
+    List<Pipeline> fakeContainerPipelines = new LinkedList<>();
+    PipelineChannel pipelineChannel = new PipelineChannel("fake",
+        LifeCycleState.OPEN, ReplicationType.STAND_ALONE, ReplicationFactor.ONE,
+        "fake");
+    Pipeline fakePipeline = new Pipeline("fake", pipelineChannel);
+    fakePipeline.setData(Longs.toByteArray(1));
+    fakeContainerPipelines.add(fakePipeline);
+
+    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
+        xceiverClientManager, metrics);
+    CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+        .setConfiguration(flushTestConfig)
+        .setVolumeName(volumeName)
+        .setUserName(userName)
+        .setPipelines(fakeContainerPipelines)
+        .setClientManager(xceiverClientManager)
+        .setBlockSize(4 * KB)
+        .setVolumeSize(50 * GB)
+        .setFlusher(flusher)
+        .setCBlockTargetMetrics(metrics)
+        .build();
+    cache.start();
+    Thread flushListenerThread = new Thread(flusher);
+    flushListenerThread.setDaemon(true);
+    flushListenerThread.start();
+
+    for (int i = 0; i < numblocks; i++) {
+      cache.put(i, data.getBytes(StandardCharsets.UTF_8));
+    }
+    Assert.assertEquals(numblocks, metrics.getNumWriteOps());
+    Thread.sleep(3000);
+
+    // all the writes to the container will fail because of fake pipelines
+    Assert.assertEquals(numblocks, metrics.getNumDirtyLogBlockRead());
+    Assert.assertTrue(
+        metrics.getNumWriteGenericExceptionRetryBlocks() >= numblocks);
+    Assert.assertEquals(0, metrics.getNumWriteIOExceptionRetryBlocks());
+    Assert.assertEquals(0, metrics.getNumFailedRetryLogFileWrites());
+    Assert.assertEquals(0, metrics.getNumFailedReleaseLevelDB());
+    cache.close();
+    flusher.shutdown();
+
+    // restart cache with correct pipelines, now blocks should be uploaded
+    // correctly
+    CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create();
+    ContainerCacheFlusher newFlusher =
+        new ContainerCacheFlusher(flushTestConfig,
+            xceiverClientManager, newMetrics);
+    CBlockLocalCache newCache = CBlockLocalCache.newBuilder()
+        .setConfiguration(flushTestConfig)
+        .setVolumeName(volumeName)
+        .setUserName(userName)
+        .setPipelines(getContainerPipeline(10))
+        .setClientManager(xceiverClientManager)
+        .setBlockSize(4 * KB)
+        .setVolumeSize(50 * GB)
+        .setFlusher(newFlusher)
+        .setCBlockTargetMetrics(newMetrics)
+        .build();
+    newCache.start();
+    Thread newFlushListenerThread = new Thread(newFlusher);
+    newFlushListenerThread.setDaemon(true);
+    newFlushListenerThread.start();
+    Thread.sleep(3000);
+    Assert.assertTrue(newMetrics.getNumRetryLogBlockRead() >= numblocks);
+    Assert.assertEquals(0, newMetrics.getNumWriteGenericExceptionRetryBlocks());
+    Assert.assertEquals(0, newMetrics.getNumWriteIOExceptionRetryBlocks());
+    Assert.assertEquals(0, newMetrics.getNumFailedReleaseLevelDB());
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[21/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ListContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ListContainerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ListContainerHandler.java
deleted file mode 100644
index d0ac974..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ListContainerHandler.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli.container;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.ozone.scm.cli.OzoneCommandHandler;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.CMD_WIDTH;
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.HELP_OP;
-
-/**
- * This is the handler that process container list command.
- */
-public class ListContainerHandler extends OzoneCommandHandler {
-
-  public static final String CONTAINER_LIST = "list";
-  public static final String OPT_START_CONTAINER = "start";
-  public static final String OPT_PREFIX_CONTAINER = "prefix";
-  public static final String OPT_COUNT = "count";
-
-  /**
-   * Constructs a handler object.
-   *
-   * @param scmClient scm client
-   */
-  public ListContainerHandler(ScmClient scmClient) {
-    super(scmClient);
-  }
-
-  @Override
-  public void execute(CommandLine cmd) throws IOException {
-    if (!cmd.hasOption(CONTAINER_LIST)) {
-      throw new IOException("Expecting container list");
-    }
-    if (cmd.hasOption(HELP_OP)) {
-      displayHelp();
-      return;
-    }
-
-    if (!cmd.hasOption(OPT_COUNT)) {
-      displayHelp();
-      if (!cmd.hasOption(HELP_OP)) {
-        throw new IOException("Expecting container count");
-      } else {
-        return;
-      }
-    }
-
-    String startName = cmd.getOptionValue(OPT_START_CONTAINER);
-    String prefixName = cmd.getOptionValue(OPT_PREFIX_CONTAINER);
-    int count = 0;
-
-    if (cmd.hasOption(OPT_COUNT)) {
-      count = Integer.parseInt(cmd.getOptionValue(OPT_COUNT));
-      if (count < 0) {
-        displayHelp();
-        throw new IOException("-count should not be negative");
-      }
-    }
-
-    List<ContainerInfo> containerList =
-        getScmClient().listContainer(startName, prefixName, count);
-
-    // Output data list
-    for (ContainerInfo container : containerList) {
-      outputContainerPipeline(container.getPipeline());
-    }
-  }
-
-  private void outputContainerPipeline(Pipeline pipeline) throws IOException {
-    // Print container report info.
-    logOut("%s", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
-        pipeline.toJsonString()));
-  }
-
-  @Override
-  public void displayHelp() {
-    Options options = new Options();
-    addOptions(options);
-    HelpFormatter helpFormatter = new HelpFormatter();
-    helpFormatter.printHelp(CMD_WIDTH, "hdfs scm -container -list <option>",
-        "where <option> can be the following", options, "");
-  }
-
-  public static void addOptions(Options options) {
-    Option startContainerOpt = new Option(OPT_START_CONTAINER,
-        true, "Specify start container name");
-    Option endContainerOpt = new Option(OPT_PREFIX_CONTAINER,
-        true, "Specify prefix container name");
-    Option countOpt = new Option(OPT_COUNT, true,
-        "Specify count number, required");
-    options.addOption(countOpt);
-    options.addOption(startContainerOpt);
-    options.addOption(endContainerOpt);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/package-info.java
deleted file mode 100644
index 1c9f40d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.cli.container;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/package-info.java
deleted file mode 100644
index d1f9775..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.cli;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
deleted file mode 100644
index 5950d9a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
+++ /dev/null
@@ -1,606 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * <p>http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * <p>Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.lease.Lease;
-import org.apache.hadoop.ozone.lease.LeaseException;
-import org.apache.hadoop.ozone.lease.LeaseManager;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.scm.container.closer.ContainerCloser;
-import org.apache.hadoop.ozone.scm.container.replication.ContainerSupervisor;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.scm.pipelines.PipelineSelector;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter;
-import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import static org.apache.hadoop.ozone.OzoneConsts.SCM_CONTAINER_DB;
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
-    .FAILED_TO_CHANGE_CONTAINER_STATE;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_CONTAINER_SIZE_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB;
-
-/**
- * Mapping class contains the mapping from a name to a pipeline mapping. This
- * is used by SCM when
- * allocating new locations and when looking up a key.
- */
-public class ContainerMapping implements Mapping {
-  private static final Logger LOG = LoggerFactory.getLogger(ContainerMapping
-      .class);
-
-  private final NodeManager nodeManager;
-  private final long cacheSize;
-  private final Lock lock;
-  private final Charset encoding = Charset.forName("UTF-8");
-  private final MetadataStore containerStore;
-  private final PipelineSelector pipelineSelector;
-  private final ContainerStateManager containerStateManager;
-  private final LeaseManager<ContainerInfo> containerLeaseManager;
-  private final ContainerSupervisor containerSupervisor;
-  private final float containerCloseThreshold;
-  private final ContainerCloser closer;
-  private final long size;
-
-  /**
-   * Constructs a mapping class that creates mapping between container names
-   * and pipelines.
-   *
-   * @param nodeManager - NodeManager so that we can get the nodes that are
-   * healthy to place new
-   * containers.
-   * @param cacheSizeMB - Amount of memory reserved for the LSM tree to cache
-   * its nodes. This is
-   * passed to LevelDB and this memory is allocated in Native code space.
-   * CacheSize is specified
-   * in MB.
-   * @throws IOException on Failure.
-   */
-  @SuppressWarnings("unchecked")
-  public ContainerMapping(
-      final Configuration conf, final NodeManager nodeManager, final int
-      cacheSizeMB) throws IOException {
-    this.nodeManager = nodeManager;
-    this.cacheSize = cacheSizeMB;
-    this.closer = new ContainerCloser(nodeManager, conf);
-
-    File metaDir = OzoneUtils.getOzoneMetaDirPath(conf);
-
-    // Write the container name to pipeline mapping.
-    File containerDBPath = new File(metaDir, SCM_CONTAINER_DB);
-    containerStore =
-        MetadataStoreBuilder.newBuilder()
-            .setConf(conf)
-            .setDbFile(containerDBPath)
-            .setCacheSize(this.cacheSize * OzoneConsts.MB)
-            .build();
-
-    this.lock = new ReentrantLock();
-
-    this.pipelineSelector = new PipelineSelector(nodeManager, conf);
-
-    // To be replaced with code getStorageSize once it is committed.
-    size = conf.getLong(OZONE_SCM_CONTAINER_SIZE_GB,
-        OZONE_SCM_CONTAINER_SIZE_DEFAULT) * 1024 * 1024 * 1024;
-    this.containerStateManager =
-        new ContainerStateManager(conf, this);
-    this.containerSupervisor =
-        new ContainerSupervisor(conf, nodeManager,
-            nodeManager.getNodePoolManager());
-    this.containerCloseThreshold = conf.getFloat(
-        ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD,
-        ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD_DEFAULT);
-    LOG.trace("Container State Manager created.");
-
-    long containerCreationLeaseTimeout = conf.getTimeDuration(
-        ScmConfigKeys.OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT,
-        ScmConfigKeys.OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT_DEFAULT,
-        TimeUnit.MILLISECONDS);
-    LOG.trace("Starting Container Lease Manager.");
-    containerLeaseManager = new LeaseManager<>(containerCreationLeaseTimeout);
-    containerLeaseManager.start();
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public ContainerInfo getContainer(final String containerName) throws
-      IOException {
-    ContainerInfo containerInfo;
-    lock.lock();
-    try {
-      byte[] containerBytes = containerStore.get(containerName.getBytes(
-          encoding));
-      if (containerBytes == null) {
-        throw new SCMException(
-            "Specified key does not exist. key : " + containerName,
-            SCMException.ResultCodes.FAILED_TO_FIND_CONTAINER);
-      }
-
-      OzoneProtos.SCMContainerInfo temp = OzoneProtos.SCMContainerInfo.PARSER
-          .parseFrom(containerBytes);
-      containerInfo = ContainerInfo.fromProtobuf(temp);
-      return containerInfo;
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public List<ContainerInfo> listContainer(String startName,
-      String prefixName, int count) throws IOException {
-    List<ContainerInfo> containerList = new ArrayList<>();
-    lock.lock();
-    try {
-      if (containerStore.isEmpty()) {
-        throw new IOException("No container exists in current db");
-      }
-      MetadataKeyFilter prefixFilter = new KeyPrefixFilter(prefixName);
-      byte[] startKey = startName == null ? null : DFSUtil.string2Bytes(
-          startName);
-      List<Map.Entry<byte[], byte[]>> range =
-          containerStore.getSequentialRangeKVs(startKey, count, prefixFilter);
-
-      // Transform the values into the pipelines.
-      // TODO: filter by container state
-      for (Map.Entry<byte[], byte[]> entry : range) {
-        ContainerInfo containerInfo =
-            ContainerInfo.fromProtobuf(
-                OzoneProtos.SCMContainerInfo.PARSER.parseFrom(
-                    entry.getValue()));
-        Preconditions.checkNotNull(containerInfo);
-        containerList.add(containerInfo);
-      }
-    } finally {
-      lock.unlock();
-    }
-    return containerList;
-  }
-
-  /**
-   * Allocates a new container.
-   *
-   * @param replicationFactor - replication factor of the container.
-   * @param containerName - Name of the container.
-   * @param owner - The string name of the Service that owns this container.
-   * @return - Pipeline that makes up this container.
-   * @throws IOException - Exception
-   */
-  @Override
-  public ContainerInfo allocateContainer(
-      ReplicationType type,
-      ReplicationFactor replicationFactor,
-      final String containerName,
-      String owner)
-      throws IOException {
-    Preconditions.checkNotNull(containerName);
-    Preconditions.checkState(!containerName.isEmpty());
-
-    ContainerInfo containerInfo;
-    if (!nodeManager.isOutOfChillMode()) {
-      throw new SCMException(
-          "Unable to create container while in chill mode",
-          SCMException.ResultCodes.CHILL_MODE_EXCEPTION);
-    }
-
-    lock.lock();
-    try {
-      byte[] containerBytes = containerStore.get(containerName.getBytes(
-          encoding));
-      if (containerBytes != null) {
-        throw new SCMException(
-            "Specified container already exists. key : " + containerName,
-            SCMException.ResultCodes.CONTAINER_EXISTS);
-      }
-      containerInfo =
-          containerStateManager.allocateContainer(
-              pipelineSelector, type, replicationFactor, containerName,
-              owner);
-      containerStore.put(
-          containerName.getBytes(encoding), containerInfo.getProtobuf()
-              .toByteArray());
-    } finally {
-      lock.unlock();
-    }
-    return containerInfo;
-  }
-
-  /**
-   * Deletes a container from SCM.
-   *
-   * @param containerName - Container name
-   * @throws IOException if container doesn't exist or container store failed
-   *                     to delete the
-   *                     specified key.
-   */
-  @Override
-  public void deleteContainer(String containerName) throws IOException {
-    lock.lock();
-    try {
-      byte[] dbKey = containerName.getBytes(encoding);
-      byte[] containerBytes = containerStore.get(dbKey);
-      if (containerBytes == null) {
-        throw new SCMException(
-            "Failed to delete container " + containerName + ", reason : " +
-                "container doesn't exist.",
-            SCMException.ResultCodes.FAILED_TO_FIND_CONTAINER);
-      }
-      containerStore.delete(dbKey);
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  /**
-   * {@inheritDoc} Used by client to update container state on SCM.
-   */
-  @Override
-  public OzoneProtos.LifeCycleState updateContainerState(
-      String containerName, OzoneProtos.LifeCycleEvent event) throws
-      IOException {
-    ContainerInfo containerInfo;
-    lock.lock();
-    try {
-      byte[] dbKey = containerName.getBytes(encoding);
-      byte[] containerBytes = containerStore.get(dbKey);
-      if (containerBytes == null) {
-        throw new SCMException(
-            "Failed to update container state"
-                + containerName
-                + ", reason : container doesn't exist.",
-            SCMException.ResultCodes.FAILED_TO_FIND_CONTAINER);
-      }
-      containerInfo =
-          ContainerInfo.fromProtobuf(OzoneProtos.SCMContainerInfo.PARSER
-              .parseFrom(containerBytes));
-
-      Preconditions.checkNotNull(containerInfo);
-      switch (event) {
-      case CREATE:
-        // Acquire lease on container
-        Lease<ContainerInfo> containerLease =
-            containerLeaseManager.acquire(containerInfo);
-        // Register callback to be executed in case of timeout
-        containerLease.registerCallBack(() -> {
-          updateContainerState(containerName,
-              OzoneProtos.LifeCycleEvent.TIMEOUT);
-          return null;
-        });
-        break;
-      case CREATED:
-        // Release the lease on container
-        containerLeaseManager.release(containerInfo);
-        break;
-      case FINALIZE:
-        // TODO: we don't need a lease manager here for closing as the
-        // container report will include the container state after HDFS-13008
-        // If a client failed to update the container close state, DN container
-        // report from 3 DNs will be used to close the container eventually.
-        break;
-      case CLOSE:
-        break;
-      case UPDATE:
-        break;
-      case DELETE:
-        break;
-      case TIMEOUT:
-        break;
-      case CLEANUP:
-        break;
-      default:
-        throw new SCMException("Unsupported container LifeCycleEvent.",
-            FAILED_TO_CHANGE_CONTAINER_STATE);
-      }
-      // If the below updateContainerState call fails, we should revert the
-      // changes made in switch case.
-      // Like releasing the lease in case of BEGIN_CREATE.
-      ContainerInfo updatedContainer = containerStateManager
-          .updateContainerState(containerInfo, event);
-      containerStore.put(dbKey, updatedContainer.getProtobuf().toByteArray());
-      return updatedContainer.getState();
-    } catch (LeaseException e) {
-      throw new IOException("Lease Exception.", e);
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  /**
-   * Returns the container State Manager.
-   *
-   * @return ContainerStateManager
-   */
-  @Override
-  public ContainerStateManager getStateManager() {
-    return containerStateManager;
-  }
-
-  /**
-   * Process container report from Datanode.
-   * <p>
-   * Processing follows a very simple logic for time being.
-   * <p>
-   * 1. Datanodes report the current State -- denoted by the datanodeState
-   * <p>
-   * 2. We are the older SCM state from the Database -- denoted by
-   * the knownState.
-   * <p>
-   * 3. We copy the usage etc. from currentState to newState and log that
-   * newState to the DB. This allows us SCM to bootup again and read the
-   * state of the world from the DB, and then reconcile the state from
-   * container reports, when they arrive.
-   *
-   * @param reports Container report
-   */
-  @Override
-  public void processContainerReports(ContainerReportsRequestProto reports)
-      throws IOException {
-    List<StorageContainerDatanodeProtocolProtos.ContainerInfo>
-        containerInfos = reports.getReportsList();
-    containerSupervisor.handleContainerReport(reports);
-    for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
-        containerInfos) {
-      byte[] dbKey = datanodeState.getContainerNameBytes().toByteArray();
-      lock.lock();
-      try {
-        byte[] containerBytes = containerStore.get(dbKey);
-        if (containerBytes != null) {
-          OzoneProtos.SCMContainerInfo knownState =
-              OzoneProtos.SCMContainerInfo.PARSER.parseFrom(containerBytes);
-
-          OzoneProtos.SCMContainerInfo newState =
-              reconcileState(datanodeState, knownState);
-
-          // FIX ME: This can be optimized, we write twice to memory, where a
-          // single write would work well.
-          //
-          // We need to write this to DB again since the closed only write
-          // the updated State.
-          containerStore.put(dbKey, newState.toByteArray());
-
-          // If the container is closed, then state is already written to SCM
-          // DB.TODO: So can we can write only once to DB.
-          if (closeContainerIfNeeded(newState)) {
-            LOG.info("Closing the Container: {}", newState.getContainerName());
-          }
-        } else {
-          // Container not found in our container db.
-          LOG.error("Error while processing container report from datanode :" +
-                  " {}, for container: {}, reason: container doesn't exist in" +
-                  "container database.", reports.getDatanodeID(),
-              datanodeState.getContainerName());
-        }
-      } finally {
-        lock.unlock();
-      }
-    }
-  }
-
-  /**
-   * Reconciles the state from Datanode with the state in SCM.
-   *
-   * @param datanodeState - State from the Datanode.
-   * @param knownState - State inside SCM.
-   * @return new SCM State for this container.
-   */
-  private OzoneProtos.SCMContainerInfo reconcileState(
-      StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState,
-      OzoneProtos.SCMContainerInfo knownState) {
-    OzoneProtos.SCMContainerInfo.Builder builder =
-        OzoneProtos.SCMContainerInfo.newBuilder();
-    builder.setContainerName(knownState.getContainerName());
-    builder.setPipeline(knownState.getPipeline());
-    // If used size is greater than allocated size, we will be updating
-    // allocated size with used size. This update is done as a fallback
-    // mechanism in case SCM crashes without properly updating allocated
-    // size. Correct allocated value will be updated by
-    // ContainerStateManager during SCM shutdown.
-    long usedSize = datanodeState.getUsed();
-    long allocated = knownState.getAllocatedBytes() > usedSize ?
-        knownState.getAllocatedBytes() : usedSize;
-    builder.setAllocatedBytes(allocated);
-    builder.setUsedBytes(usedSize);
-    builder.setNumberOfKeys(datanodeState.getKeyCount());
-    builder.setState(knownState.getState());
-    builder.setStateEnterTime(knownState.getStateEnterTime());
-    builder.setContainerID(knownState.getContainerID());
-    if (knownState.getOwner() != null) {
-      builder.setOwner(knownState.getOwner());
-    }
-    return builder.build();
-  }
-
-  /**
-   * Queues the close container command, to datanode and writes the new state
-   * to container DB.
-   * <p>
-   * TODO : Remove this 2 ContainerInfo definitions. It is brain dead to have
-   * one protobuf in one file and another definition in another file.
-   *
-   * @param newState - This is the state we maintain in SCM.
-   * @throws IOException
-   */
-  private boolean closeContainerIfNeeded(OzoneProtos.SCMContainerInfo newState)
-      throws IOException {
-    float containerUsedPercentage = 1.0f *
-        newState.getUsedBytes() / this.size;
-
-    ContainerInfo scmInfo = getContainer(newState.getContainerName());
-    if (containerUsedPercentage >= containerCloseThreshold
-        && !isClosed(scmInfo)) {
-      // We will call closer till get to the closed state.
-      // That is SCM will make this call repeatedly until we reach the closed
-      // state.
-      closer.close(newState);
-
-      if (shouldClose(scmInfo)) {
-        // This event moves the Container from Open to Closing State, this is
-        // a state inside SCM. This is the desired state that SCM wants this
-        // container to reach. We will know that a container has reached the
-        // closed state from container reports. This state change should be
-        // invoked once and only once.
-        OzoneProtos.LifeCycleState state = updateContainerState(
-            scmInfo.getContainerName(),
-            OzoneProtos.LifeCycleEvent.FINALIZE);
-        if (state != OzoneProtos.LifeCycleState.CLOSING) {
-          LOG.error("Failed to close container {}, reason : Not able " +
-                  "to " +
-                  "update container state, current container state: {}.",
-              newState.getContainerName(), state);
-          return false;
-        }
-        return true;
-      }
-    }
-    return false;
-  }
-
-  /**
-   * In Container is in closed state, if it is in closed, Deleting or Deleted
-   * State.
-   *
-   * @param info - ContainerInfo.
-   * @return true if is in open state, false otherwise
-   */
-  private boolean shouldClose(ContainerInfo info) {
-    return info.getState() == OzoneProtos.LifeCycleState.OPEN;
-  }
-
-  private boolean isClosed(ContainerInfo info) {
-    return info.getState() == OzoneProtos.LifeCycleState.CLOSED;
-  }
-
-  @VisibleForTesting
-  public ContainerCloser getCloser() {
-    return closer;
-  }
-
-  /**
-   * Closes this stream and releases any system resources associated with it.
-   * If the stream is
-   * already closed then invoking this method has no effect.
-   * <p>
-   * <p>As noted in {@link AutoCloseable#close()}, cases where the close may
-   * fail require careful
-   * attention. It is strongly advised to relinquish the underlying resources
-   * and to internally
-   * <em>mark</em> the {@code Closeable} as closed, prior to throwing the
-   * {@code IOException}.
-   *
-   * @throws IOException if an I/O error occurs
-   */
-  @Override
-  public void close() throws IOException {
-    if (containerLeaseManager != null) {
-      containerLeaseManager.shutdown();
-    }
-    if (containerStateManager != null) {
-      flushContainerInfo();
-      containerStateManager.close();
-    }
-    if (containerStore != null) {
-      containerStore.close();
-    }
-  }
-
-  /**
-   * Since allocatedBytes of a container is only in memory, stored in
-   * containerStateManager, when closing ContainerMapping, we need to update
-   * this in the container store.
-   *
-   * @throws IOException on failure.
-   */
-  @VisibleForTesting
-  public void flushContainerInfo() throws IOException {
-    List<ContainerInfo> containers = containerStateManager.getAllContainers();
-    List<String> failedContainers = new ArrayList<>();
-    for (ContainerInfo info : containers) {
-      // even if some container updated failed, others can still proceed
-      try {
-        byte[] dbKey = info.getContainerName().getBytes(encoding);
-        byte[] containerBytes = containerStore.get(dbKey);
-        // TODO : looks like when a container is deleted, the container is
-        // removed from containerStore but not containerStateManager, so it can
-        // return info of a deleted container. may revisit this in the future,
-        // for now, just skip a not-found container
-        if (containerBytes != null) {
-          OzoneProtos.SCMContainerInfo oldInfoProto =
-              OzoneProtos.SCMContainerInfo.PARSER.parseFrom(containerBytes);
-          ContainerInfo oldInfo = ContainerInfo.fromProtobuf(oldInfoProto);
-          ContainerInfo newInfo = new ContainerInfo.Builder()
-              .setAllocatedBytes(info.getAllocatedBytes())
-              .setContainerName(oldInfo.getContainerName())
-              .setNumberOfKeys(oldInfo.getNumberOfKeys())
-              .setOwner(oldInfo.getOwner())
-              .setPipeline(oldInfo.getPipeline())
-              .setState(oldInfo.getState())
-              .setUsedBytes(oldInfo.getUsedBytes())
-              .build();
-          containerStore.put(dbKey, newInfo.getProtobuf().toByteArray());
-        } else {
-          LOG.debug("Container state manager has container {} but not found " +
-                  "in container store, a deleted container?",
-              info.getContainerName());
-        }
-      } catch (IOException ioe) {
-        failedContainers.add(info.getContainerName());
-      }
-    }
-    if (!failedContainers.isEmpty()) {
-      throw new IOException("Error in flushing container info from container " +
-          "state manager: " + failedContainers);
-    }
-  }
-
-  @VisibleForTesting
-  public MetadataStore getContainerStore() {
-    return containerStore;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStateManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStateManager.java
deleted file mode 100644
index c586107..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStateManager.java
+++ /dev/null
@@ -1,451 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException;
-import org.apache.hadoop.ozone.common.statemachine.StateMachine;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleEvent;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.scm.container.ContainerStates.ContainerID;
-import org.apache.hadoop.ozone.scm.container.ContainerStates.ContainerState;
-import org.apache.hadoop.ozone.scm.container.ContainerStates.ContainerStateMap;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.scm.pipelines.PipelineSelector;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
-    .FAILED_TO_CHANGE_CONTAINER_STATE;
-
-/**
- * A container state manager keeps track of container states and returns
- * containers that match various queries.
- * <p>
- * This state machine is driven by a combination of server and client actions.
- * <p>
- * This is how a create container happens: 1. When a container is created, the
- * Server(or SCM) marks that Container as ALLOCATED state. In this state, SCM
- * has chosen a pipeline for container to live on. However, the container is not
- * created yet. This container along with the pipeline is returned to the
- * client.
- * <p>
- * 2. The client when it sees the Container state as ALLOCATED understands that
- * container needs to be created on the specified pipeline. The client lets the
- * SCM know that saw this flag and is initiating the on the data nodes.
- * <p>
- * This is done by calling into notifyObjectCreation(ContainerName,
- * BEGIN_CREATE) flag. When SCM gets this call, SCM puts the container state
- * into CREATING. All this state means is that SCM told Client to create a
- * container and client saw that request.
- * <p>
- * 3. Then client makes calls to datanodes directly, asking the datanodes to
- * create the container. This is done with the help of pipeline that supports
- * this container.
- * <p>
- * 4. Once the creation of the container is complete, the client will make
- * another call to the SCM, this time specifying the containerName and the
- * COMPLETE_CREATE as the Event.
- * <p>
- * 5. With COMPLETE_CREATE event, the container moves to an Open State. This is
- * the state when clients can write to a container.
- * <p>
- * 6. If the client does not respond with the COMPLETE_CREATE event with a
- * certain time, the state machine times out and triggers a delete operation of
- * the container.
- * <p>
- * Please see the function initializeStateMachine below to see how this looks in
- * code.
- * <p>
- * Reusing existing container :
- * <p>
- * The create container call is not made all the time, the system tries to use
- * open containers as much as possible. So in those cases, it looks thru the
- * list of open containers and will return containers that match the specific
- * signature.
- * <p>
- * Please note : Logically there are 3 separate state machines in the case of
- * containers.
- * <p>
- * The Create State Machine -- Commented extensively above.
- * <p>
- * Open/Close State Machine - Once the container is in the Open State,
- * eventually it will be closed, once sufficient data has been written to it.
- * <p>
- * TimeOut Delete Container State Machine - if the container creating times out,
- * then Container State manager decides to delete the container.
- */
-public class ContainerStateManager implements Closeable {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ContainerStateManager.class);
-
-  private final StateMachine<OzoneProtos.LifeCycleState,
-      OzoneProtos.LifeCycleEvent> stateMachine;
-
-  private final long containerSize;
-  private final ConcurrentHashMap<ContainerState, ContainerID> lastUsedMap;
-  private final ContainerStateMap containers;
-  private final AtomicLong containerCount;
-
-  /**
-   * Constructs a Container State Manager that tracks all containers owned by
-   * SCM for the purpose of allocation of blocks.
-   * <p>
-   * TODO : Add Container Tags so we know which containers are owned by SCM.
-   */
-  @SuppressWarnings("unchecked")
-  public ContainerStateManager(Configuration configuration,
-      Mapping containerMapping) {
-
-    // Initialize the container state machine.
-    Set<OzoneProtos.LifeCycleState> finalStates = new HashSet();
-
-    // These are the steady states of a container.
-    finalStates.add(LifeCycleState.OPEN);
-    finalStates.add(LifeCycleState.CLOSED);
-    finalStates.add(LifeCycleState.DELETED);
-
-    this.stateMachine = new StateMachine<>(LifeCycleState.ALLOCATED,
-        finalStates);
-    initializeStateMachine();
-
-    this.containerSize = OzoneConsts.GB * configuration.getInt(
-        ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB,
-        ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT);
-
-    lastUsedMap = new ConcurrentHashMap<>();
-    containerCount = new AtomicLong(0);
-    containers = new ContainerStateMap();
-    loadExistingContainers(containerMapping);
-  }
-
-  private void loadExistingContainers(Mapping containerMapping) {
-
-    List<ContainerInfo> containerList;
-    try {
-      containerList = containerMapping.listContainer(null,
-          null, Integer.MAX_VALUE);
-
-      // if there are no container to load, let us return.
-      if (containerList == null || containerList.size() == 0) {
-        LOG.info("No containers to load for this cluster.");
-        return;
-      }
-    } catch (IOException e) {
-      if (!e.getMessage().equals("No container exists in current db")) {
-        LOG.error("Could not list the containers", e);
-      }
-      return;
-    }
-
-    try {
-      long maxID = 0;
-      for (ContainerInfo container : containerList) {
-        containers.addContainer(container);
-
-        if (maxID < container.getContainerID()) {
-          maxID = container.getContainerID();
-        }
-
-        containerCount.set(maxID);
-      }
-    } catch (SCMException ex) {
-      LOG.error("Unable to create a container information. ", ex);
-      // Fix me, what is the proper shutdown procedure for SCM ??
-      // System.exit(1) // Should we exit here?
-    }
-  }
-
-  /**
-   * Return the info of all the containers kept by the in-memory mapping.
-   *
-   * @return the list of all container info.
-   */
-  public List<ContainerInfo> getAllContainers() {
-    List<ContainerInfo> list = new ArrayList<>();
-
-    //No Locking needed since the return value is an immutable map.
-    containers.getContainerMap().forEach((key, value) -> list.add(value));
-    return list;
-  }
-
-  /*
-   *
-   * Event and State Transition Mapping:
-   *
-   * State: ALLOCATED ---------------> CREATING
-   * Event:                CREATE
-   *
-   * State: CREATING  ---------------> OPEN
-   * Event:               CREATED
-   *
-   * State: OPEN      ---------------> CLOSING
-   * Event:               FINALIZE
-   *
-   * State: CLOSING   ---------------> CLOSED
-   * Event:                CLOSE
-   *
-   * State: CLOSED   ----------------> DELETING
-   * Event:                DELETE
-   *
-   * State: DELETING ----------------> DELETED
-   * Event:               CLEANUP
-   *
-   * State: CREATING  ---------------> DELETING
-   * Event:               TIMEOUT
-   *
-   *
-   * Container State Flow:
-   *
-   * [ALLOCATED]------->[CREATING]--------->[OPEN]---------->[CLOSING]------->[CLOSED]
-   *            (CREATE)     |    (CREATED)       (FINALIZE)          (CLOSE)    |
-   *                         |                                                   |
-   *                         |                                                   |
-   *                         |(TIMEOUT)                                  (DELETE)|
-   *                         |                                                   |
-   *                         +------------------> [DELETING] <-------------------+
-   *                                                   |
-   *                                                   |
-   *                                          (CLEANUP)|
-   *                                                   |
-   *                                               [DELETED]
-   */
-  private void initializeStateMachine() {
-    stateMachine.addTransition(LifeCycleState.ALLOCATED,
-        LifeCycleState.CREATING,
-        LifeCycleEvent.CREATE);
-
-    stateMachine.addTransition(LifeCycleState.CREATING,
-        LifeCycleState.OPEN,
-        LifeCycleEvent.CREATED);
-
-    stateMachine.addTransition(LifeCycleState.OPEN,
-        LifeCycleState.CLOSING,
-        LifeCycleEvent.FINALIZE);
-
-    stateMachine.addTransition(LifeCycleState.CLOSING,
-        LifeCycleState.CLOSED,
-        LifeCycleEvent.CLOSE);
-
-    stateMachine.addTransition(LifeCycleState.CLOSED,
-        LifeCycleState.DELETING,
-        LifeCycleEvent.DELETE);
-
-    stateMachine.addTransition(LifeCycleState.CREATING,
-        LifeCycleState.DELETING,
-        LifeCycleEvent.TIMEOUT);
-
-    stateMachine.addTransition(LifeCycleState.DELETING,
-        LifeCycleState.DELETED,
-        LifeCycleEvent.CLEANUP);
-  }
-
-  /**
-   * allocates a new container based on the type, replication etc.
-   *
-   * @param selector -- Pipeline selector class.
-   * @param type -- Replication type.
-   * @param replicationFactor - Replication replicationFactor.
-   * @param containerName - Container Name.
-   * @return Container Info.
-   * @throws IOException  on Failure.
-   */
-  public ContainerInfo allocateContainer(PipelineSelector selector, OzoneProtos
-      .ReplicationType type, OzoneProtos.ReplicationFactor replicationFactor,
-      final String containerName, String owner) throws
-      IOException {
-
-    Pipeline pipeline = selector.getReplicationPipeline(type,
-        replicationFactor, containerName);
-    ContainerInfo containerInfo = new ContainerInfo.Builder()
-        .setContainerName(containerName)
-        .setState(OzoneProtos.LifeCycleState.ALLOCATED)
-        .setPipeline(pipeline)
-        // This is bytes allocated for blocks inside container, not the
-        // container size
-        .setAllocatedBytes(0)
-        .setUsedBytes(0)
-        .setNumberOfKeys(0)
-        .setStateEnterTime(Time.monotonicNow())
-        .setOwner(owner)
-        .setContainerID(containerCount.incrementAndGet())
-        .build();
-    Preconditions.checkNotNull(containerInfo);
-    containers.addContainer(containerInfo);
-    LOG.trace("New container allocated: {}", containerInfo);
-    return containerInfo;
-  }
-
-  /**
-   * Update the Container State to the next state.
-   *
-   * @param info - ContainerInfo
-   * @param event - LifeCycle Event
-   * @return Updated ContainerInfo.
-   * @throws SCMException  on Failure.
-   */
-  public ContainerInfo updateContainerState(ContainerInfo
-      info, OzoneProtos.LifeCycleEvent event) throws SCMException {
-    LifeCycleState newState;
-    try {
-      newState = this.stateMachine.getNextState(info.getState(), event);
-    } catch (InvalidStateTransitionException ex) {
-      String error = String.format("Failed to update container state %s, " +
-              "reason: invalid state transition from state: %s upon " +
-              "event: %s.",
-          info.getPipeline().getContainerName(), info.getState(), event);
-      LOG.error(error);
-      throw new SCMException(error, FAILED_TO_CHANGE_CONTAINER_STATE);
-    }
-
-    // This is a post condition after executing getNextState.
-    Preconditions.checkNotNull(newState);
-    containers.updateState(info, info.getState(), newState);
-    return containers.getContainerInfo(info);
-  }
-
-  /**
-   * Update the container State.
-   * @param info - Container Info
-   * @return  ContainerInfo
-   * @throws SCMException - on Error.
-   */
-  public ContainerInfo updateContainerInfo(ContainerInfo info)
-      throws SCMException {
-    containers.updateContainerInfo(info);
-    return containers.getContainerInfo(info);
-  }
-
-
-  /**
-   * Return a container matching the attributes specified.
-   *
-   * @param size - Space needed in the Container.
-   * @param owner - Owner of the container - A specific nameservice.
-   * @param type - Replication Type {StandAlone, Ratis}
-   * @param factor - Replication Factor {ONE, THREE}
-   * @param state - State of the Container-- {Open, Allocated etc.}
-   * @return ContainerInfo, null if there is no match found.
-   */
-  public ContainerInfo getMatchingContainer(final long size,
-      String owner, ReplicationType type, ReplicationFactor factor,
-      LifeCycleState state) {
-
-    // Find containers that match the query spec, if no match return null.
-    NavigableSet<ContainerID> matchingSet =
-        containers.getMatchingContainerIDs(state, owner, factor, type);
-    if (matchingSet == null || matchingSet.size() == 0) {
-      return null;
-    }
-
-    // Get the last used container and find container above the last used
-    // container ID.
-    ContainerState key = new ContainerState(owner, type, factor);
-    ContainerID lastID = lastUsedMap.get(key);
-    if(lastID == null) {
-      lastID = matchingSet.first();
-    }
-
-    // There is a small issue here. The first time, we will skip the first
-    // container. But in most cases it will not matter.
-    NavigableSet<ContainerID> resultSet = matchingSet.tailSet(lastID, false);
-    if (resultSet.size() == 0) {
-      resultSet = matchingSet;
-    }
-
-    ContainerInfo selectedContainer =
-        findContainerWithSpace(size, resultSet, owner);
-    if (selectedContainer == null) {
-
-      // If we did not find any space in the tailSet, we need to look for
-      // space in the headset, we need to pass true to deal with the
-      // situation that we have a lone container that has space. That is we
-      // ignored the last used container under the assumption we can find
-      // other containers with space, but if have a single container that is
-      // not true. Hence we need to include the last used container as the
-      // last element in the sorted set.
-
-      resultSet = matchingSet.headSet(lastID, true);
-      selectedContainer = findContainerWithSpace(size, resultSet, owner);
-    }
-    // Update the allocated Bytes on this container.
-    if(selectedContainer != null) {
-      selectedContainer.updateAllocatedBytes(size);
-    }
-    return selectedContainer;
-
-  }
-
-  private ContainerInfo findContainerWithSpace(long size,
-      NavigableSet<ContainerID> searchSet, String owner) {
-    // Get the container with space to meet our request.
-    for (ContainerID id : searchSet) {
-      ContainerInfo containerInfo = containers.getContainerInfo(id.getId());
-      if (containerInfo.getAllocatedBytes() + size <= this.containerSize) {
-        containerInfo.updateLastUsedTime();
-
-        ContainerState key = new ContainerState(owner,
-            containerInfo.getPipeline().getType(),
-            containerInfo.getPipeline().getFactor());
-        lastUsedMap.put(key, containerInfo.containerID());
-        return containerInfo;
-      }
-    }
-    return null;
-  }
-
-  /**
-   * Returns a set of ContainerIDs that match the Container.
-   *
-   * @param owner  Owner of the Containers.
-   * @param type - Replication Type of the containers
-   * @param factor - Replication factor of the containers.
-   * @param state - Current State, like Open, Close etc.
-   * @return Set of containers that match the specific query parameters.
-   */
-  public NavigableSet<ContainerID> getMatchingContainerIDs(
-      String owner, ReplicationType type, ReplicationFactor factor,
-      LifeCycleState state) {
-    return containers.getMatchingContainerIDs(state, owner,
-        factor, type);
-  }
-
-  @Override
-  public void close() throws IOException {
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerAttribute.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerAttribute.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerAttribute.java
deleted file mode 100644
index 1372e7f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerAttribute.java
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-package org.apache.hadoop.ozone.scm.container.ContainerStates;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.TreeSet;
-
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
-    .FAILED_TO_CHANGE_CONTAINER_STATE;
-
-/**
- * Each Attribute that we manage for a container is maintained as a map.
- * <p>
- * Currently we manage the following attributes for a container.
- * <p>
- * 1. StateMap - LifeCycleState -> Set of ContainerIDs
- * 2. TypeMap  - ReplicationType -> Set of ContainerIDs
- * 3. OwnerMap - OwnerNames -> Set of ContainerIDs
- * 4. FactorMap - ReplicationFactor -> Set of ContainerIDs
- * <p>
- * This means that for a cluster size of 750 PB -- we will have around 150
- * Million containers, if we assume 5GB average container size.
- * <p>
- * That implies that these maps will take around 2/3 GB of RAM which will be
- * pinned down in the SCM. This is deemed acceptable since we can tune the
- * container size --say we make it 10GB average size, then we can deal with a
- * cluster size of 1.5 exa bytes with the same metadata in SCMs memory.
- * <p>
- * Please note: **This class is not thread safe**. This used to be thread safe,
- * while bench marking we found that ContainerStateMap would be taking 5
- * locks for a single container insert. If we remove locks in this class,
- * then we are able to perform about 540K operations per second, with the
- * locks in this class it goes down to 246K operations per second. Hence we
- * are going to rely on ContainerStateMap locks to maintain consistency of
- * data in these classes too, since ContainerAttribute is only used by
- * ContainerStateMap class.
- */
-public class ContainerAttribute<T> {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ContainerAttribute.class);
-
-  private final Map<T, NavigableSet<ContainerID>> attributeMap;
-  private static final NavigableSet<ContainerID> EMPTY_SET =  Collections
-      .unmodifiableNavigableSet(new TreeSet<>());
-
-  /**
-   * Creates a Container Attribute map from an existing Map.
-   *
-   * @param attributeMap - AttributeMap
-   */
-  public ContainerAttribute(Map<T, NavigableSet<ContainerID>> attributeMap) {
-    this.attributeMap = attributeMap;
-  }
-
-  /**
-   * Create an empty Container Attribute map.
-   */
-  public ContainerAttribute() {
-    this.attributeMap = new HashMap<>();
-  }
-
-  /**
-   * Insert or update the value in the Attribute map.
-   *
-   * @param key - The key to the set where the ContainerID should exist.
-   * @param value - Actual Container ID.
-   * @throws SCMException - on Error
-   */
-  public boolean insert(T key, ContainerID value) throws SCMException {
-    Preconditions.checkNotNull(key);
-    Preconditions.checkNotNull(value);
-
-    if (attributeMap.containsKey(key)) {
-      if (attributeMap.get(key).add(value)) {
-        return true; //we inserted the value as it doesn’t exist in the set.
-      } else { // Failure indicates that this ContainerID exists in the Set
-        if (!attributeMap.get(key).remove(value)) {
-          LOG.error("Failure to remove the object from the Map.Key:{}, " +
-              "ContainerID: {}", key, value);
-          throw new SCMException("Failure to remove the object from the Map",
-              FAILED_TO_CHANGE_CONTAINER_STATE);
-        }
-        attributeMap.get(key).add(value);
-        return true;
-      }
-    } else {
-      // This key does not exist, we need to allocate this key in the map.
-      // TODO: Replace TreeSet with FoldedTreeSet from HDFS Utils.
-      // Skipping for now, since FoldedTreeSet does not have implementations
-      // for headSet and TailSet. We need those calls.
-      this.attributeMap.put(key, new TreeSet<>());
-      // This should not fail, we just allocated this object.
-      attributeMap.get(key).add(value);
-      return true;
-    }
-  }
-
-  /**
-   * Returns true if have this bucket in the attribute map.
-   *
-   * @param key - Key to lookup
-   * @return true if we have the key
-   */
-  public boolean hasKey(T key) {
-    Preconditions.checkNotNull(key);
-    return this.attributeMap.containsKey(key);
-  }
-
-  /**
-   * Returns true if we have the key and the containerID in the bucket.
-   *
-   * @param key - Key to the bucket
-   * @param id - container ID that we want to lookup
-   * @return true or false
-   */
-  public boolean hasContainerID(T key, ContainerID id) {
-    Preconditions.checkNotNull(key);
-    Preconditions.checkNotNull(id);
-
-    return this.attributeMap.containsKey(key) &&
-        this.attributeMap.get(key).contains(id);
-  }
-
-  /**
-   * Returns true if we have the key and the containerID in the bucket.
-   *
-   * @param key - Key to the bucket
-   * @param id - container ID that we want to lookup
-   * @return true or false
-   */
-  public boolean hasContainerID(T key, int id) {
-    return hasContainerID(key, ContainerID.valueof(id));
-  }
-
-  /**
-   * Clears all entries for this key type.
-   *
-   * @param key - Key that identifies the Set.
-   */
-  public void clearSet(T key) {
-    Preconditions.checkNotNull(key);
-
-    if (attributeMap.containsKey(key)) {
-      attributeMap.get(key).clear();
-    } else {
-      LOG.debug("key: {} does not exist in the attributeMap", key);
-    }
-  }
-
-  /**
-   * Removes a container ID from the set pointed by the key.
-   *
-   * @param key - key to identify the set.
-   * @param value - Container ID
-   */
-  public boolean remove(T key, ContainerID value) {
-    Preconditions.checkNotNull(key);
-    Preconditions.checkNotNull(value);
-
-    if (attributeMap.containsKey(key)) {
-      if (!attributeMap.get(key).remove(value)) {
-        LOG.debug("ContainerID: {} does not exist in the set pointed by " +
-            "key:{}", value, key);
-        return false;
-      }
-      return true;
-    } else {
-      LOG.debug("key: {} does not exist in the attributeMap", key);
-      return false;
-    }
-  }
-
-  /**
-   * Returns the collection that maps to the given key.
-   *
-   * @param key - Key to the bucket.
-   * @return Underlying Set in immutable form.
-   */
-  public NavigableSet<ContainerID> getCollection(T key) {
-    Preconditions.checkNotNull(key);
-
-    if (this.attributeMap.containsKey(key)) {
-      return Collections.unmodifiableNavigableSet(this.attributeMap.get(key));
-    }
-    LOG.debug("No such Key. Key {}", key);
-    return EMPTY_SET;
-  }
-
-  /**
-   * Moves a ContainerID from one bucket to another.
-   *
-   * @param currentKey - Current Key
-   * @param newKey - newKey
-   * @param value - ContainerID
-   * @throws SCMException on Error
-   */
-  public void update(T currentKey, T newKey, ContainerID value)
-      throws SCMException {
-    Preconditions.checkNotNull(currentKey);
-    Preconditions.checkNotNull(newKey);
-
-    boolean removed = false;
-    try {
-      removed = remove(currentKey, value);
-      if (!removed) {
-        throw new SCMException("Unable to find key in the current key bucket",
-            FAILED_TO_CHANGE_CONTAINER_STATE);
-      }
-      insert(newKey, value);
-    } catch (SCMException ex) {
-      // if we removed the key, insert it back to original bucket, since the
-      // next insert failed.
-      LOG.error("error in update.", ex);
-      if (removed) {
-        insert(currentKey, value);
-        LOG.trace("reinserted the removed key. {}", currentKey);
-      }
-      throw ex;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerState.java
deleted file mode 100644
index 6c492ff..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerState.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.ozone.scm.container.ContainerStates;
-
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-/**
- * Class that acts as the container state.
- */
-public class ContainerState {
-  private final OzoneProtos.ReplicationType type;
-  private final String owner;
-  private final OzoneProtos.ReplicationFactor replicationFactor;
-
-  /**
-   * Constructs a Container Key.
-   *
-   * @param owner - Container Owners
-   * @param type - Replication Type.
-   * @param factor - Replication Factors
-   */
-  public ContainerState(String owner, OzoneProtos.ReplicationType type,
-      OzoneProtos.ReplicationFactor factor) {
-    this.type = type;
-    this.owner = owner;
-    this.replicationFactor = factor;
-  }
-
-
-  public OzoneProtos.ReplicationType getType() {
-    return type;
-  }
-
-  public String getOwner() {
-    return owner;
-  }
-
-  public OzoneProtos.ReplicationFactor getFactor() {
-    return replicationFactor;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-
-    ContainerState that = (ContainerState) o;
-
-    return new EqualsBuilder()
-        .append(type, that.type)
-        .append(owner, that.owner)
-        .append(replicationFactor, that.replicationFactor)
-        .isEquals();
-  }
-
-  @Override
-  public int hashCode() {
-    return new HashCodeBuilder(137, 757)
-        .append(type)
-        .append(owner)
-        .append(replicationFactor)
-        .toHashCode();
-  }
-
-  @Override
-  public String toString() {
-    return "ContainerKey{" +
-        ", type=" + type +
-        ", owner=" + owner +
-        ", replicationFactor=" + replicationFactor +
-        '}';
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerStateMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerStateMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerStateMap.java
deleted file mode 100644
index eebc6be..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerStateMap.java
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.ozone.scm.container.ContainerStates;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.util.AutoCloseableLock;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.TreeSet;
-
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
-    .CONTAINER_EXISTS;
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
-    .FAILED_TO_CHANGE_CONTAINER_STATE;
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
-    .FAILED_TO_FIND_CONTAINER;
-
-/**
- * Container State Map acts like a unified map for various attributes that are
- * used to select containers when we need allocated blocks.
- * <p>
- * This class provides the ability to query 4 classes of attributes. They are
- * <p>
- * 1. LifeCycleStates - LifeCycle States of container describe in which state
- * a container is. For example, a container needs to be in Open State for a
- * client to able to write to it.
- * <p>
- * 2. Owners - Each instance of Name service, for example, Namenode of HDFS or
- * Key Space Manager (KSM) of Ozone or CBlockServer --  is an owner. It is
- * possible to have many KSMs for a Ozone cluster and only one SCM. But SCM
- * keeps the data from each KSM in separate bucket, never mixing them. To
- * write data, often we have to find all open containers for a specific owner.
- * <p>
- * 3. ReplicationType - The clients are allowed to specify what kind of
- * replication pipeline they want to use. Each Container exists on top of a
- * pipeline, so we need to get ReplicationType that is specified by the user.
- * <p>
- * 4. ReplicationFactor - The replication factor represents how many copies
- * of data should be made, right now we support 2 different types, ONE
- * Replica and THREE Replica. User can specify how many copies should be made
- * for a ozone key.
- * <p>
- * The most common access pattern of this class is to select a container based
- * on all these parameters, for example, when allocating a block we will
- * select a container that belongs to user1, with Ratis replication which can
- * make 3 copies of data. The fact that we will look for open containers by
- * default and if we cannot find them we will add new containers.
- */
-public class ContainerStateMap {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ContainerStateMap.class);
-
-  private final ContainerAttribute<LifeCycleState> lifeCycleStateMap;
-  private final ContainerAttribute<String> ownerMap;
-  private final ContainerAttribute<ReplicationFactor> factorMap;
-  private final ContainerAttribute<ReplicationType> typeMap;
-
-  private final Map<ContainerID, ContainerInfo> containerMap;
-  private final static NavigableSet<ContainerID> EMPTY_SET  =
-      Collections.unmodifiableNavigableSet(new TreeSet<>());
-
-  // Container State Map lock should be held before calling into
-  // Update ContainerAttributes. The consistency of ContainerAttributes is
-  // protected by this lock.
-  private final AutoCloseableLock autoLock;
-
-  /**
-   * Create a ContainerStateMap.
-   */
-  public ContainerStateMap() {
-    lifeCycleStateMap = new ContainerAttribute<>();
-    ownerMap = new ContainerAttribute<>();
-    factorMap = new ContainerAttribute<>();
-    typeMap = new ContainerAttribute<>();
-    containerMap = new HashMap<>();
-    autoLock = new AutoCloseableLock();
-//        new InstrumentedLock(getClass().getName(), LOG,
-//            new ReentrantLock(),
-//            1000,
-//            300));
-  }
-
-  /**
-   * Adds a ContainerInfo Entry in the ContainerStateMap.
-   *
-   * @param info - container info
-   * @throws SCMException - throws if create failed.
-   */
-  public void addContainer(ContainerInfo info)
-      throws SCMException {
-
-    try (AutoCloseableLock lock = autoLock.acquire()) {
-      ContainerID id = ContainerID.valueof(info.getContainerID());
-      if (containerMap.putIfAbsent(id, info) != null) {
-        LOG.debug("Duplicate container ID detected. {}", id);
-        throw new
-            SCMException("Duplicate container ID detected.",
-            CONTAINER_EXISTS);
-      }
-
-      lifeCycleStateMap.insert(info.getState(), id);
-      ownerMap.insert(info.getOwner(), id);
-      factorMap.insert(info.getPipeline().getFactor(), id);
-      typeMap.insert(info.getPipeline().getType(), id);
-      LOG.trace("Created container with {} successfully.", id);
-    }
-  }
-
-  /**
-   * Returns the latest state of Container from SCM's Container State Map.
-   *
-   * @param info - ContainerInfo
-   * @return ContainerInfo
-   */
-  public ContainerInfo getContainerInfo(ContainerInfo info) {
-    return getContainerInfo(info.getContainerID());
-  }
-
-  /**
-   * Returns the latest state of Container from SCM's Container State Map.
-   *
-   * @param containerID - int
-   * @return container info, if found.
-   */
-  public ContainerInfo getContainerInfo(long containerID) {
-    ContainerID id = new ContainerID(containerID);
-    return containerMap.get(id);
-  }
-
-  /**
-   * Returns the full container Map.
-   *
-   * @return - Map
-   */
-  public Map<ContainerID, ContainerInfo> getContainerMap() {
-    try (AutoCloseableLock lock = autoLock.acquire()) {
-      return Collections.unmodifiableMap(containerMap);
-    }
-  }
-
-  /**
-   * Just update the container State.
-   * @param info ContainerInfo.
-   */
-  public void updateContainerInfo(ContainerInfo info) throws SCMException {
-    Preconditions.checkNotNull(info);
-    ContainerInfo currentInfo = null;
-    try (AutoCloseableLock lock = autoLock.acquire()) {
-      currentInfo = containerMap.get(
-          ContainerID.valueof(info.getContainerID()));
-
-      if (currentInfo == null) {
-        throw new SCMException("No such container.", FAILED_TO_FIND_CONTAINER);
-      }
-      containerMap.put(info.containerID(), info);
-    }
-  }
-
-  /**
-   * Update the State of a container.
-   *
-   * @param info - ContainerInfo
-   * @param currentState - CurrentState
-   * @param newState - NewState.
-   * @throws SCMException - in case of failure.
-   */
-  public void updateState(ContainerInfo info, LifeCycleState currentState,
-      LifeCycleState newState) throws SCMException {
-    Preconditions.checkNotNull(currentState);
-    Preconditions.checkNotNull(newState);
-
-    ContainerID id = new ContainerID(info.getContainerID());
-    ContainerInfo currentInfo = null;
-
-    try (AutoCloseableLock lock = autoLock.acquire()) {
-      currentInfo = containerMap.get(id);
-
-      if (currentInfo == null) {
-        throw new
-            SCMException("No such container.", FAILED_TO_FIND_CONTAINER);
-      }
-      // We are updating two places before this update is done, these can
-      // fail independently, since the code needs to handle it.
-
-      // We update the attribute map, if that fails it will throw an exception,
-      // so no issues, if we are successful, we keep track of the fact that we
-      // have updated the lifecycle state in the map, and update the container
-      // state. If this second update fails, we will attempt to roll back the
-      // earlier change we did. If the rollback fails, we can be in an
-      // inconsistent state,
-
-      info.setState(newState);
-      containerMap.put(id, info);
-      lifeCycleStateMap.update(currentState, newState, id);
-      LOG.trace("Updated the container {} to new state. Old = {}, new = " +
-          "{}", id, currentState, newState);
-    } catch (SCMException ex) {
-      LOG.error("Unable to update the container state. {}", ex);
-      // we need to revert the change in this attribute since we are not
-      // able to update the hash table.
-      LOG.info("Reverting the update to lifecycle state. Moving back to " +
-              "old state. Old = {}, Attempted state = {}", currentState,
-          newState);
-
-      containerMap.put(id, currentInfo);
-
-      // if this line throws, the state map can be in an inconsistent
-      // state, since we will have modified the attribute by the
-      // container state will not in sync since we were not able to put
-      // that into the hash table.
-      lifeCycleStateMap.update(newState, currentState, id);
-
-      throw new SCMException("Updating the container map failed.", ex,
-          FAILED_TO_CHANGE_CONTAINER_STATE);
-    }
-  }
-
-  /**
-   * Returns A list of containers owned by a name service.
-   *
-   * @param ownerName - Name of the NameService.
-   * @return - NavigableSet of ContainerIDs.
-   */
-  NavigableSet<ContainerID> getContainerIDsByOwner(String ownerName) {
-    Preconditions.checkNotNull(ownerName);
-
-    try (AutoCloseableLock lock = autoLock.acquire()) {
-      return ownerMap.getCollection(ownerName);
-    }
-  }
-
-  /**
-   * Returns Containers in the System by the Type.
-   *
-   * @param type - Replication type -- StandAlone, Ratis etc.
-   * @return NavigableSet
-   */
-  NavigableSet<ContainerID> getContainerIDsByType(ReplicationType type) {
-    Preconditions.checkNotNull(type);
-
-    try (AutoCloseableLock lock = autoLock.acquire()) {
-      return typeMap.getCollection(type);
-    }
-  }
-
-  /**
-   * Returns Containers by replication factor.
-   *
-   * @param factor - Replication Factor.
-   * @return NavigableSet.
-   */
-  NavigableSet<ContainerID> getContainerIDsByFactor(ReplicationFactor factor) {
-    Preconditions.checkNotNull(factor);
-
-    try (AutoCloseableLock lock = autoLock.acquire()) {
-      return factorMap.getCollection(factor);
-    }
-  }
-
-  /**
-   * Returns Containers by State.
-   *
-   * @param state - State - Open, Closed etc.
-   * @return List of containers by state.
-   */
-  NavigableSet<ContainerID> getContainerIDsByState(LifeCycleState state) {
-    Preconditions.checkNotNull(state);
-
-    try (AutoCloseableLock lock = autoLock.acquire()) {
-      return lifeCycleStateMap.getCollection(state);
-    }
-  }
-
-  /**
-   * Gets the containers that matches the  following filters.
-   *
-   * @param state - LifeCycleState
-   * @param owner - Owner
-   * @param factor - Replication Factor
-   * @param type - Replication Type
-   * @return ContainerInfo or Null if not container satisfies the criteria.
-   */
-  public NavigableSet<ContainerID> getMatchingContainerIDs(
-      LifeCycleState state, String owner,
-      ReplicationFactor factor, ReplicationType type) {
-
-    Preconditions.checkNotNull(state, "State cannot be null");
-    Preconditions.checkNotNull(owner, "Owner cannot be null");
-    Preconditions.checkNotNull(factor, "Factor cannot be null");
-    Preconditions.checkNotNull(type, "Type cannot be null");
-
-    try (AutoCloseableLock lock = autoLock.acquire()) {
-
-      // If we cannot meet any one condition we return EMPTY_SET immediately.
-      // Since when we intersect these sets, the result will be empty if any
-      // one is empty.
-      NavigableSet<ContainerID> stateSet =
-          lifeCycleStateMap.getCollection(state);
-      if (stateSet.size() == 0) {
-        return EMPTY_SET;
-      }
-
-      NavigableSet<ContainerID> ownerSet = ownerMap.getCollection(owner);
-      if (ownerSet.size() == 0) {
-        return EMPTY_SET;
-      }
-
-      NavigableSet<ContainerID> factorSet = factorMap.getCollection(factor);
-      if (factorSet.size() == 0) {
-        return EMPTY_SET;
-      }
-
-      NavigableSet<ContainerID> typeSet = typeMap.getCollection(type);
-      if (typeSet.size() == 0) {
-        return EMPTY_SET;
-      }
-
-
-      // if we add more constraints we will just add those sets here..
-      NavigableSet<ContainerID>[] sets = sortBySize(stateSet,
-          ownerSet, factorSet, typeSet);
-
-      NavigableSet<ContainerID> currentSet = sets[0];
-      // We take the smallest set and intersect against the larger sets. This
-      // allows us to reduce the lookups to the least possible number.
-      for (int x = 1; x < sets.length; x++) {
-        currentSet = intersectSets(currentSet, sets[x]);
-      }
-      return currentSet;
-    }
-  }
-
-  /**
-   * Calculates the intersection between sets and returns a new set.
-   *
-   * @param smaller - First Set
-   * @param bigger - Second Set
-   * @return resultSet which is the intersection of these two sets.
-   */
-  private NavigableSet<ContainerID> intersectSets(
-      NavigableSet<ContainerID> smaller,
-      NavigableSet<ContainerID> bigger) {
-    Preconditions.checkState(smaller.size() <= bigger.size(),
-        "This function assumes the first set is lesser or equal to second " +
-            "set");
-    NavigableSet<ContainerID> resultSet = new TreeSet<>();
-    for (ContainerID id : smaller) {
-      if (bigger.contains(id)) {
-        resultSet.add(id);
-      }
-    }
-    return resultSet;
-  }
-
-  /**
-   * Sorts a list of Sets based on Size. This is useful when we are
-   * intersecting the sets.
-   *
-   * @param sets - varagrs of sets
-   * @return Returns a sorted array of sets based on the size of the set.
-   */
-  @SuppressWarnings("unchecked")
-  private NavigableSet<ContainerID>[] sortBySize(
-      NavigableSet<ContainerID>... sets) {
-    for (int x = 0; x < sets.length - 1; x++) {
-      for (int y = 0; y < sets.length - x - 1; y++) {
-        if (sets[y].size() > sets[y + 1].size()) {
-          NavigableSet temp = sets[y];
-          sets[y] = sets[y + 1];
-          sets[y + 1] = temp;
-        }
-      }
-    }
-    return sets;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java
deleted file mode 100644
index 6a7e663..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-/**
- * Container States management package.
- */
-package org.apache.hadoop.ozone.scm.container.ContainerStates;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/Mapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/Mapping.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/Mapping.java
deleted file mode 100644
index 0d442d1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/Mapping.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container;
-
-
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Mapping class contains the mapping from a name to a pipeline mapping. This is
- * used by SCM when allocating new locations and when looking up a key.
- */
-public interface Mapping extends Closeable {
-  /**
-   * Returns the ContainerInfo from the container name.
-   *
-   * @param containerName - Name
-   * @return - ContainerInfo such as creation state and the pipeline.
-   * @throws IOException
-   */
-  ContainerInfo getContainer(String containerName) throws IOException;
-
-  /**
-   * Returns containers under certain conditions.
-   * Search container names from start name(exclusive),
-   * and use prefix name to filter the result. The max
-   * size of the searching range cannot exceed the
-   * value of count.
-   *
-   * @param startName start name, if null, start searching at the head.
-   * @param prefixName prefix name, if null, then filter is disabled.
-   * @param count count, if count < 0, the max size is unlimited.(
-   *              Usually the count will be replace with a very big
-   *              value instead of being unlimited in case the db is very big)
-   *
-   * @return a list of container.
-   * @throws IOException
-   */
-  List<ContainerInfo> listContainer(String startName, String prefixName,
-      int count) throws IOException;
-
-  /**
-   * Allocates a new container for a given keyName and replication factor.
-   *
-   * @param replicationFactor - replication factor of the container.
-   * @param containerName - Name.
-   * @param owner
-   * @return - Container Info.
-   * @throws IOException
-   */
-  ContainerInfo allocateContainer(OzoneProtos.ReplicationType type,
-      OzoneProtos.ReplicationFactor replicationFactor,
-      String containerName, String owner) throws IOException;
-
-  /**
-   * Deletes a container from SCM.
-   *
-   * @param containerName - Container Name
-   * @throws IOException
-   */
-  void deleteContainer(String containerName) throws IOException;
-
-  /**
-   * Update container state.
-   * @param containerName - Container Name
-   * @param event - container life cycle event
-   * @return - new container state
-   * @throws IOException
-   */
-  OzoneProtos.LifeCycleState updateContainerState(String containerName,
-      OzoneProtos.LifeCycleEvent event) throws IOException;
-
-  /**
-   * Returns the container State Manager.
-   * @return ContainerStateManager
-   */
-  ContainerStateManager getStateManager();
-
-  /**
-   * Process container report from Datanode.
-   *
-   * @param reports Container report
-   */
-  void processContainerReports(ContainerReportsRequestProto reports)
-      throws IOException;
-
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[39/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ContainerOperationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ContainerOperationClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ContainerOperationClient.java
deleted file mode 100644
index d411a6b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ContainerOperationClient.java
+++ /dev/null
@@ -1,403 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.scm.client;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerData;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ReadContainerResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocolPB
-    .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.UUID;
-
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState.ALLOCATED;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState.OPEN;
-
-/**
- * This class provides the client-facing APIs of container operations.
- */
-public class ContainerOperationClient implements ScmClient {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ContainerOperationClient.class);
-  private static long containerSizeB = -1;
-  private final StorageContainerLocationProtocolClientSideTranslatorPB
-      storageContainerLocationClient;
-  private final XceiverClientManager xceiverClientManager;
-
-  public ContainerOperationClient(
-      StorageContainerLocationProtocolClientSideTranslatorPB
-          storageContainerLocationClient,
-      XceiverClientManager xceiverClientManager) {
-    this.storageContainerLocationClient = storageContainerLocationClient;
-    this.xceiverClientManager = xceiverClientManager;
-  }
-
-  /**
-   * Return the capacity of containers. The current assumption is that all
-   * containers have the same capacity. Therefore one static is sufficient for
-   * any container.
-   * @return The capacity of one container in number of bytes.
-   */
-  public static long getContainerSizeB() {
-    return containerSizeB;
-  }
-
-  /**
-   * Set the capacity of container. Should be exactly once on system start.
-   * @param size Capacity of one container in number of bytes.
-   */
-  public static void setContainerSizeB(long size) {
-    containerSizeB = size;
-  }
-
-  /**
-   * @inheritDoc
-   */
-  @Override
-  public Pipeline createContainer(String containerId, String owner)
-      throws IOException {
-    XceiverClientSpi client = null;
-    try {
-      Pipeline pipeline =
-          storageContainerLocationClient.allocateContainer(
-              xceiverClientManager.getType(),
-              xceiverClientManager.getFactor(), containerId, owner);
-      client = xceiverClientManager.acquireClient(pipeline);
-
-      // Allocated State means that SCM has allocated this pipeline in its
-      // namespace. The client needs to create the pipeline on the machines
-      // which was choosen by the SCM.
-      Preconditions.checkState(pipeline.getLifeCycleState() == ALLOCATED ||
-          pipeline.getLifeCycleState() == OPEN, "Unexpected pipeline state");
-      if (pipeline.getLifeCycleState() == ALLOCATED) {
-        createPipeline(client, pipeline);
-      }
-      // TODO : Container Client State needs to be updated.
-      // TODO : Return ContainerInfo instead of Pipeline
-      createContainer(containerId, client, pipeline);
-      return pipeline;
-    } finally {
-      if (client != null) {
-        xceiverClientManager.releaseClient(client);
-      }
-    }
-  }
-
-  /**
-   * Create a container over pipeline specified by the SCM.
-   *
-   * @param containerId - Container ID
-   * @param client - Client to communicate with Datanodes
-   * @param pipeline - A pipeline that is already created.
-   * @throws IOException
-   */
-  public void createContainer(String containerId, XceiverClientSpi client,
-      Pipeline pipeline) throws IOException {
-    String traceID = UUID.randomUUID().toString();
-    storageContainerLocationClient.notifyObjectStageChange(
-        ObjectStageChangeRequestProto.Type.container,
-        containerId,
-        ObjectStageChangeRequestProto.Op.create,
-        ObjectStageChangeRequestProto.Stage.begin);
-    ContainerProtocolCalls.createContainer(client, traceID);
-    storageContainerLocationClient.notifyObjectStageChange(
-        ObjectStageChangeRequestProto.Type.container,
-        containerId,
-        ObjectStageChangeRequestProto.Op.create,
-        ObjectStageChangeRequestProto.Stage.complete);
-
-    // Let us log this info after we let SCM know that we have completed the
-    // creation state.
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Created container " + containerId
-          + " leader:" + pipeline.getLeader()
-          + " machines:" + pipeline.getMachines());
-    }
-  }
-
-  /**
-   * Creates a pipeline over the machines choosen by the SCM.
-   *
-   * @param client - Client
-   * @param pipeline - pipeline to be createdon Datanodes.
-   * @throws IOException
-   */
-  private void createPipeline(XceiverClientSpi client, Pipeline pipeline)
-      throws IOException {
-
-    Preconditions.checkNotNull(pipeline.getPipelineName(), "Pipeline " +
-        "name cannot be null when client create flag is set.");
-
-    // Pipeline creation is a three step process.
-    //
-    // 1. Notify SCM that this client is doing a create pipeline on
-    // datanodes.
-    //
-    // 2. Talk to Datanodes to create the pipeline.
-    //
-    // 3. update SCM that pipeline creation was successful.
-    storageContainerLocationClient.notifyObjectStageChange(
-        ObjectStageChangeRequestProto.Type.pipeline,
-        pipeline.getPipelineName(),
-        ObjectStageChangeRequestProto.Op.create,
-        ObjectStageChangeRequestProto.Stage.begin);
-
-    client.createPipeline(pipeline.getPipelineName(),
-        pipeline.getMachines());
-
-    storageContainerLocationClient.notifyObjectStageChange(
-        ObjectStageChangeRequestProto.Type.pipeline,
-        pipeline.getPipelineName(),
-        ObjectStageChangeRequestProto.Op.create,
-        ObjectStageChangeRequestProto.Stage.complete);
-
-    // TODO : Should we change the state on the client side ??
-    // That makes sense, but it is not needed for the client to work.
-    LOG.debug("Pipeline creation successful. Pipeline: {}",
-        pipeline.toString());
-  }
-
-  /**
-   * @inheritDoc
-   */
-  @Override
-  public Pipeline createContainer(OzoneProtos.ReplicationType type,
-      OzoneProtos.ReplicationFactor factor,
-      String containerId, String owner) throws IOException {
-    XceiverClientSpi client = null;
-    try {
-      // allocate container on SCM.
-      Pipeline pipeline =
-          storageContainerLocationClient.allocateContainer(type, factor,
-              containerId, owner);
-      client = xceiverClientManager.acquireClient(pipeline);
-
-      // Allocated State means that SCM has allocated this pipeline in its
-      // namespace. The client needs to create the pipeline on the machines
-      // which was choosen by the SCM.
-      if (pipeline.getLifeCycleState() == ALLOCATED) {
-        createPipeline(client, pipeline);
-      }
-
-      // TODO : Return ContainerInfo instead of Pipeline
-      // connect to pipeline leader and allocate container on leader datanode.
-      client = xceiverClientManager.acquireClient(pipeline);
-      createContainer(containerId, client, pipeline);
-      return pipeline;
-    } finally {
-      if (client != null) {
-        xceiverClientManager.releaseClient(client);
-      }
-    }
-  }
-
-  /**
-   * Returns a set of Nodes that meet a query criteria.
-   *
-   * @param nodeStatuses - A set of criteria that we want the node to have.
-   * @param queryScope - Query scope - Cluster or pool.
-   * @param poolName - if it is pool, a pool name is required.
-   * @return A set of nodes that meet the requested criteria.
-   * @throws IOException
-   */
-  @Override
-  public OzoneProtos.NodePool queryNode(EnumSet<OzoneProtos.NodeState>
-      nodeStatuses, OzoneProtos.QueryScope queryScope, String poolName)
-      throws IOException {
-    return storageContainerLocationClient.queryNode(nodeStatuses, queryScope,
-        poolName);
-  }
-
-  /**
-   * Creates a specified replication pipeline.
-   */
-  @Override
-  public Pipeline createReplicationPipeline(OzoneProtos.ReplicationType type,
-      OzoneProtos.ReplicationFactor factor, OzoneProtos.NodePool nodePool)
-      throws IOException {
-    return storageContainerLocationClient.createReplicationPipeline(type,
-        factor, nodePool);
-  }
-
-  /**
-   * Delete the container, this will release any resource it uses.
-   * @param pipeline - Pipeline that represents the container.
-   * @param force - True to forcibly delete the container.
-   * @throws IOException
-   */
-  @Override
-  public void deleteContainer(Pipeline pipeline, boolean force)
-      throws IOException {
-    XceiverClientSpi client = null;
-    try {
-      client = xceiverClientManager.acquireClient(pipeline);
-      String traceID = UUID.randomUUID().toString();
-      ContainerProtocolCalls.deleteContainer(client, force, traceID);
-      storageContainerLocationClient
-          .deleteContainer(pipeline.getContainerName());
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Deleted container {}, leader: {}, machines: {} ",
-            pipeline.getContainerName(),
-            pipeline.getLeader(),
-            pipeline.getMachines());
-      }
-    } finally {
-      if (client != null) {
-        xceiverClientManager.releaseClient(client);
-      }
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public List<ContainerInfo> listContainer(String startName,
-      String prefixName, int count)
-      throws IOException {
-    return storageContainerLocationClient.listContainer(
-        startName, prefixName, count);
-  }
-
-  /**
-   * Get meta data from an existing container.
-   *
-   * @param pipeline - pipeline that represents the container.
-   * @return ContainerInfo - a message of protobuf which has basic info
-   * of a container.
-   * @throws IOException
-   */
-  @Override
-  public ContainerData readContainer(Pipeline pipeline) throws IOException {
-    XceiverClientSpi client = null;
-    try {
-      client = xceiverClientManager.acquireClient(pipeline);
-      String traceID = UUID.randomUUID().toString();
-      ReadContainerResponseProto response =
-          ContainerProtocolCalls.readContainer(client,
-              pipeline.getContainerName(), traceID);
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Read container {}, leader: {}, machines: {} ",
-            pipeline.getContainerName(),
-            pipeline.getLeader(),
-            pipeline.getMachines());
-      }
-      return response.getContainerData();
-    } finally {
-      if (client != null) {
-        xceiverClientManager.releaseClient(client);
-      }
-    }
-  }
-
-  /**
-   * Given an id, return the pipeline associated with the container.
-   * @param containerId - String Container ID
-   * @return Pipeline of the existing container, corresponding to the given id.
-   * @throws IOException
-   */
-  @Override
-  public Pipeline getContainer(String containerId) throws
-      IOException {
-    return storageContainerLocationClient.getContainer(containerId);
-  }
-
-  /**
-   * Close a container.
-   *
-   * @param pipeline the container to be closed.
-   * @throws IOException
-   */
-  @Override
-  public void closeContainer(Pipeline pipeline) throws IOException {
-    XceiverClientSpi client = null;
-    try {
-      LOG.debug("Close container {}", pipeline);
-      /*
-      TODO: two orders here, revisit this later:
-      1. close on SCM first, then on data node
-      2. close on data node first, then on SCM
-
-      with 1: if client failed after closing on SCM, then there is a
-      container SCM thinks as closed, but is actually open. Then SCM will no
-      longer allocate block to it, which is fine. But SCM may later try to
-      replicate this "closed" container, which I'm not sure is safe.
-
-      with 2: if client failed after close on datanode, then there is a
-      container SCM thinks as open, but is actually closed. Then SCM will still
-      try to allocate block to it. Which will fail when actually doing the
-      write. No more data can be written, but at least the correctness and
-      consistency of existing data will maintain.
-
-      For now, take the #2 way.
-       */
-      // Actually close the container on Datanode
-      client = xceiverClientManager.acquireClient(pipeline);
-      String traceID = UUID.randomUUID().toString();
-
-      String containerId = pipeline.getContainerName();
-
-      storageContainerLocationClient.notifyObjectStageChange(
-          ObjectStageChangeRequestProto.Type.container,
-          containerId,
-          ObjectStageChangeRequestProto.Op.close,
-          ObjectStageChangeRequestProto.Stage.begin);
-
-      ContainerProtocolCalls.closeContainer(client, traceID);
-      // Notify SCM to close the container
-      storageContainerLocationClient.notifyObjectStageChange(
-          ObjectStageChangeRequestProto.Type.container,
-          containerId,
-          ObjectStageChangeRequestProto.Op.close,
-          ObjectStageChangeRequestProto.Stage.complete);
-    } finally {
-      if (client != null) {
-        xceiverClientManager.releaseClient(client);
-      }
-    }
-  }
-
-  /**
-   * Get the the current usage information.
-   * @param pipeline - Pipeline
-   * @return the size of the given container.
-   * @throws IOException
-   */
-  @Override
-  public long getContainerSize(Pipeline pipeline) throws IOException {
-    // TODO : Pipeline can be null, handle it correctly.
-    long size = getContainerSizeB();
-    if (size == -1) {
-      throw new IOException("Container size unknown!");
-    }
-    return size;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ScmClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ScmClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ScmClient.java
deleted file mode 100644
index ef2f1e0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ScmClient.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.scm.client;
-
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerData;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.List;
-
-/**
- * The interface to call into underlying container layer.
- *
- * Written as interface to allow easy testing: implement a mock container layer
- * for standalone testing of CBlock API without actually calling into remote
- * containers. Actual container layer can simply re-implement this.
- *
- * NOTE this is temporarily needed class. When SCM containers are full-fledged,
- * this interface will likely be removed.
- */
-@InterfaceStability.Unstable
-public interface ScmClient {
-  /**
-   * Creates a Container on SCM and returns the pipeline.
-   * @param containerId - String container ID
-   * @return Pipeline
-   * @throws IOException
-   */
-  Pipeline createContainer(String containerId, String owner) throws IOException;
-
-  /**
-   * Gets a container by Name -- Throws if the container does not exist.
-   * @param containerId - String Container ID
-   * @return Pipeline
-   * @throws IOException
-   */
-  Pipeline getContainer(String containerId) throws IOException;
-
-  /**
-   * Close a container by name.
-   *
-   * @param pipeline the container to be closed.
-   * @throws IOException
-   */
-  void closeContainer(Pipeline pipeline) throws IOException;
-
-  /**
-   * Deletes an existing container.
-   * @param pipeline - Pipeline that represents the container.
-   * @param force - true to forcibly delete the container.
-   * @throws IOException
-   */
-  void deleteContainer(Pipeline pipeline, boolean force) throws IOException;
-
-  /**
-   * Lists a range of containers and get their info.
-   *
-   * @param startName start name, if null, start searching at the head.
-   * @param prefixName prefix name, if null, then filter is disabled.
-   * @param count count, if count < 0, the max size is unlimited.(
-   *              Usually the count will be replace with a very big
-   *              value instead of being unlimited in case the db is very big)
-   *
-   * @return a list of pipeline.
-   * @throws IOException
-   */
-  List<ContainerInfo> listContainer(String startName, String prefixName,
-      int count) throws IOException;
-
-  /**
-   * Read meta data from an existing container.
-   * @param pipeline - Pipeline that represents the container.
-   * @return ContainerInfo
-   * @throws IOException
-   */
-  ContainerData readContainer(Pipeline pipeline) throws IOException;
-
-
-  /**
-   * Gets the container size -- Computed by SCM from Container Reports.
-   * @param pipeline - Pipeline
-   * @return number of bytes used by this container.
-   * @throws IOException
-   */
-  long getContainerSize(Pipeline pipeline) throws IOException;
-
-  /**
-   * Creates a Container on SCM and returns the pipeline.
-   * @param type - Replication Type.
-   * @param replicationFactor - Replication Factor
-   * @param containerId - Container ID
-   * @return Pipeline
-   * @throws IOException - in case of error.
-   */
-  Pipeline createContainer(OzoneProtos.ReplicationType type,
-      OzoneProtos.ReplicationFactor replicationFactor, String containerId,
-      String owner) throws IOException;
-
-  /**
-   * Returns a set of Nodes that meet a query criteria.
-   * @param nodeStatuses - A set of criteria that we want the node to have.
-   * @param queryScope - Query scope - Cluster or pool.
-   * @param poolName - if it is pool, a pool name is required.
-   * @return A set of nodes that meet the requested criteria.
-   * @throws IOException
-   */
-  OzoneProtos.NodePool queryNode(EnumSet<OzoneProtos.NodeState> nodeStatuses,
-      OzoneProtos.QueryScope queryScope, String poolName) throws IOException;
-
-  /**
-   * Creates a specified replication pipeline.
-   * @param type - Type
-   * @param factor - Replication factor
-   * @param nodePool - Set of machines.
-   * @throws IOException
-   */
-  Pipeline createReplicationPipeline(OzoneProtos.ReplicationType type,
-      OzoneProtos.ReplicationFactor factor, OzoneProtos.NodePool nodePool)
-      throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/package-info.java
deleted file mode 100644
index e85e542..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.client;
-
-/**
- * This package contains classes for the client of the storage container
- * protocol.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java
deleted file mode 100644
index f51336f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.scm.container.common.helpers;
-
-/**
- * Allocated block wraps the result returned from SCM#allocateBlock which
- * contains a Pipeline and the key.
- */
-public final class AllocatedBlock {
-  private Pipeline pipeline;
-  private String key;
-  // Indicates whether the client should create container before writing block.
-  private boolean shouldCreateContainer;
-
-  /**
-   * Builder for AllocatedBlock.
-   */
-  public static class Builder {
-    private Pipeline pipeline;
-    private String key;
-    private boolean shouldCreateContainer;
-
-    public Builder setPipeline(Pipeline p) {
-      this.pipeline = p;
-      return this;
-    }
-
-    public Builder setKey(String k) {
-      this.key = k;
-      return this;
-    }
-
-    public Builder setShouldCreateContainer(boolean shouldCreate) {
-      this.shouldCreateContainer = shouldCreate;
-      return this;
-    }
-
-    public AllocatedBlock build() {
-      return new AllocatedBlock(pipeline, key, shouldCreateContainer);
-    }
-  }
-
-  private AllocatedBlock(Pipeline pipeline, String key,
-      boolean shouldCreateContainer) {
-    this.pipeline = pipeline;
-    this.key = key;
-    this.shouldCreateContainer = shouldCreateContainer;
-  }
-
-  public Pipeline getPipeline() {
-    return pipeline;
-  }
-
-  public String getKey() {
-    return key;
-  }
-
-  public boolean getCreateContainer() {
-    return shouldCreateContainer;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java
deleted file mode 100644
index 11d4438..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java
+++ /dev/null
@@ -1,333 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.scm.container.common.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.scm.container.ContainerStates.ContainerID;
-import org.apache.hadoop.util.Time;
-
-import java.util.Comparator;
-
-/**
- * Class wraps ozone container info.
- */
-public class ContainerInfo
-    implements Comparator<ContainerInfo>, Comparable<ContainerInfo> {
-  private OzoneProtos.LifeCycleState state;
-  private Pipeline pipeline;
-  // Bytes allocated by SCM for clients.
-  private long allocatedBytes;
-  // Actual container usage, updated through heartbeat.
-  private long usedBytes;
-  private long numberOfKeys;
-  private long lastUsed;
-  // The wall-clock ms since the epoch at which the current state enters.
-  private long stateEnterTime;
-  private String owner;
-  private String containerName;
-  private long containerID;
-  ContainerInfo(
-      long containerID,
-      final String containerName,
-      OzoneProtos.LifeCycleState state,
-      Pipeline pipeline,
-      long allocatedBytes,
-      long usedBytes,
-      long numberOfKeys,
-      long stateEnterTime,
-      String owner) {
-    this.containerID = containerID;
-    this.containerName = containerName;
-    this.pipeline = pipeline;
-    this.allocatedBytes = allocatedBytes;
-    this.usedBytes = usedBytes;
-    this.numberOfKeys = numberOfKeys;
-    this.lastUsed = Time.monotonicNow();
-    this.state = state;
-    this.stateEnterTime = stateEnterTime;
-    this.owner = owner;
-  }
-
-  /**
-   * Needed for serialization findbugs.
-   */
-  public ContainerInfo() {
-  }
-
-  public static ContainerInfo fromProtobuf(OzoneProtos.SCMContainerInfo info) {
-    ContainerInfo.Builder builder = new ContainerInfo.Builder();
-    builder.setPipeline(Pipeline.getFromProtoBuf(info.getPipeline()));
-    builder.setAllocatedBytes(info.getAllocatedBytes());
-    builder.setUsedBytes(info.getUsedBytes());
-    builder.setNumberOfKeys(info.getNumberOfKeys());
-    builder.setState(info.getState());
-    builder.setStateEnterTime(info.getStateEnterTime());
-    builder.setOwner(info.getOwner());
-    builder.setContainerName(info.getContainerName());
-    builder.setContainerID(info.getContainerID());
-    return builder.build();
-  }
-
-  public long getContainerID() {
-    return containerID;
-  }
-
-  public String getContainerName() {
-    return containerName;
-  }
-
-  public OzoneProtos.LifeCycleState getState() {
-    return state;
-  }
-
-  public void setState(OzoneProtos.LifeCycleState state) {
-    this.state = state;
-  }
-
-  public long getStateEnterTime() {
-    return stateEnterTime;
-  }
-
-  public Pipeline getPipeline() {
-    return pipeline;
-  }
-
-  public long getAllocatedBytes() {
-    return allocatedBytes;
-  }
-
-  /**
-   * Set Allocated bytes.
-   *
-   * @param size - newly allocated bytes -- negative size is case of deletes
-   * can be used.
-   */
-  public void updateAllocatedBytes(long size) {
-    this.allocatedBytes += size;
-  }
-
-  public long getUsedBytes() {
-    return usedBytes;
-  }
-
-  public long getNumberOfKeys() {
-    return numberOfKeys;
-  }
-
-  public ContainerID containerID() {
-    return new ContainerID(getContainerID());
-  }
-
-  /**
-   * Gets the last used time from SCM's perspective.
-   *
-   * @return time in milliseconds.
-   */
-  public long getLastUsed() {
-    return lastUsed;
-  }
-
-  public void updateLastUsedTime() {
-    lastUsed = Time.monotonicNow();
-  }
-
-  public void allocate(long size) {
-    // should we also have total container size in ContainerInfo
-    // and check before allocating?
-    allocatedBytes += size;
-  }
-
-  public OzoneProtos.SCMContainerInfo getProtobuf() {
-    OzoneProtos.SCMContainerInfo.Builder builder =
-        OzoneProtos.SCMContainerInfo.newBuilder();
-    builder.setPipeline(getPipeline().getProtobufMessage());
-    builder.setAllocatedBytes(getAllocatedBytes());
-    builder.setUsedBytes(getUsedBytes());
-    builder.setNumberOfKeys(getNumberOfKeys());
-    builder.setState(state);
-    builder.setStateEnterTime(stateEnterTime);
-    builder.setContainerID(getContainerID());
-
-    if (getOwner() != null) {
-      builder.setOwner(getOwner());
-    }
-    builder.setContainerName(getContainerName());
-    return builder.build();
-  }
-
-  public String getOwner() {
-    return owner;
-  }
-
-  public void setOwner(String owner) {
-    this.owner = owner;
-  }
-
-  @Override
-  public String toString() {
-    return "ContainerInfo{"
-        + "state=" + state
-        + ", pipeline=" + pipeline
-        + ", stateEnterTime=" + stateEnterTime
-        + ", owner=" + owner
-        + ", containerName='" + containerName
-        + '}';
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-
-    ContainerInfo that = (ContainerInfo) o;
-
-    return new EqualsBuilder()
-        .append(pipeline.getContainerName(), that.pipeline.getContainerName())
-
-        // TODO : Fix this later. If we add these factors some tests fail.
-        // So Commenting this to continue and will enforce this with
-        // Changes in pipeline where we remove Container Name to
-        // SCMContainerinfo from Pipeline.
-        // .append(pipeline.getFactor(), that.pipeline.getFactor())
-        // .append(pipeline.getType(), that.pipeline.getType())
-        .append(owner, that.owner)
-        .isEquals();
-  }
-
-  @Override
-  public int hashCode() {
-    return new HashCodeBuilder(11, 811)
-        .append(pipeline.getContainerName())
-        .append(pipeline.getFactor())
-        .append(pipeline.getType())
-        .append(owner)
-        .toHashCode();
-  }
-
-  /**
-   * Compares its two arguments for order.  Returns a negative integer, zero, or
-   * a positive integer as the first argument is less than, equal to, or greater
-   * than the second.<p>
-   *
-   * @param o1 the first object to be compared.
-   * @param o2 the second object to be compared.
-   * @return a negative integer, zero, or a positive integer as the first
-   * argument is less than, equal to, or greater than the second.
-   * @throws NullPointerException if an argument is null and this comparator
-   *                              does not permit null arguments
-   * @throws ClassCastException   if the arguments' types prevent them from
-   *                              being compared by this comparator.
-   */
-  @Override
-  public int compare(ContainerInfo o1, ContainerInfo o2) {
-    return Long.compare(o1.getLastUsed(), o2.getLastUsed());
-  }
-
-  /**
-   * Compares this object with the specified object for order.  Returns a
-   * negative integer, zero, or a positive integer as this object is less than,
-   * equal to, or greater than the specified object.
-   *
-   * @param o the object to be compared.
-   * @return a negative integer, zero, or a positive integer as this object is
-   * less than, equal to, or greater than the specified object.
-   * @throws NullPointerException if the specified object is null
-   * @throws ClassCastException   if the specified object's type prevents it
-   *                              from being compared to this object.
-   */
-  @Override
-  public int compareTo(ContainerInfo o) {
-    return this.compare(this, o);
-  }
-
-  /**
-   * Builder class for ContainerInfo.
-   */
-  public static class Builder {
-    private OzoneProtos.LifeCycleState state;
-    private Pipeline pipeline;
-    private long allocated;
-    private long used;
-    private long keys;
-    private long stateEnterTime;
-    private String owner;
-    private String containerName;
-    private long containerID;
-
-    public Builder setContainerID(long id) {
-      Preconditions.checkState(id >= 0);
-      this.containerID = id;
-      return this;
-    }
-
-    public Builder setState(OzoneProtos.LifeCycleState lifeCycleState) {
-      this.state = lifeCycleState;
-      return this;
-    }
-
-    public Builder setPipeline(Pipeline containerPipeline) {
-      this.pipeline = containerPipeline;
-      return this;
-    }
-
-    public Builder setAllocatedBytes(long bytesAllocated) {
-      this.allocated = bytesAllocated;
-      return this;
-    }
-
-    public Builder setUsedBytes(long bytesUsed) {
-      this.used = bytesUsed;
-      return this;
-    }
-
-    public Builder setNumberOfKeys(long keyCount) {
-      this.keys = keyCount;
-      return this;
-    }
-
-    public Builder setStateEnterTime(long time) {
-      this.stateEnterTime = time;
-      return this;
-    }
-
-    public Builder setOwner(String containerOwner) {
-      this.owner = containerOwner;
-      return this;
-    }
-
-    public Builder setContainerName(String container) {
-      this.containerName = container;
-      return this;
-    }
-
-    public ContainerInfo build() {
-      return new
-          ContainerInfo(containerID, containerName, state, pipeline,
-          allocated, used, keys, stateEnterTime, owner);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java
deleted file mode 100644
index 5dbb853..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.scm.container.common.helpers;
-
-
-import static org.apache.hadoop.ozone.protocol.proto
-    .ScmBlockLocationProtocolProtos.DeleteScmBlockResult;
-
-/**
- * Class wraps storage container manager block deletion results.
- */
-public class DeleteBlockResult {
-  private String key;
-  private DeleteScmBlockResult.Result result;
-
-  public DeleteBlockResult(final String key,
-      final DeleteScmBlockResult.Result result) {
-    this.key = key;
-    this.result = result;
-  }
-
-  /**
-   * Get key deleted.
-   * @return key name.
-   */
-  public String getKey() {
-    return key;
-  }
-
-  /**
-   * Get key deletion result.
-   * @return key deletion result.
-   */
-  public DeleteScmBlockResult.Result getResult() {
-    return result;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java
deleted file mode 100644
index 9d47646..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.scm.container.common.helpers;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonFilter;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.ser.FilterProvider;
-import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
-import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * A pipeline represents the group of machines over which a container lives.
- */
-public class Pipeline {
-  static final String PIPELINE_INFO = "PIPELINE_INFO_FILTER";
-  private static final ObjectWriter WRITER;
-
-  static {
-    ObjectMapper mapper = new ObjectMapper();
-    String[] ignorableFieldNames = {"data"};
-    FilterProvider filters = new SimpleFilterProvider()
-        .addFilter(PIPELINE_INFO, SimpleBeanPropertyFilter
-            .serializeAllExcept(ignorableFieldNames));
-    mapper.setVisibility(PropertyAccessor.FIELD,
-        JsonAutoDetect.Visibility.ANY);
-    mapper.addMixIn(Object.class, MixIn.class);
-
-    WRITER = mapper.writer(filters);
-  }
-
-  private String containerName;
-  private PipelineChannel pipelineChannel;
-  /**
-   * Allows you to maintain private data on pipelines. This is not serialized
-   * via protobuf, just allows us to maintain some private data.
-   */
-  @JsonIgnore
-  private byte[] data;
-  /**
-   * Constructs a new pipeline data structure.
-   *
-   * @param containerName - Container
-   * @param pipelineChannel - transport information for this container
-   */
-  public Pipeline(String containerName, PipelineChannel pipelineChannel) {
-    this.containerName = containerName;
-    this.pipelineChannel = pipelineChannel;
-    data = null;
-  }
-
-  /**
-   * Gets pipeline object from protobuf.
-   *
-   * @param pipeline - ProtoBuf definition for the pipeline.
-   * @return Pipeline Object
-   */
-  public static Pipeline getFromProtoBuf(OzoneProtos.Pipeline pipeline) {
-    Preconditions.checkNotNull(pipeline);
-    PipelineChannel pipelineChannel =
-        PipelineChannel.getFromProtoBuf(pipeline.getPipelineChannel());
-    return new Pipeline(pipeline.getContainerName(), pipelineChannel);
-  }
-
-  public OzoneProtos.ReplicationFactor getFactor() {
-    return pipelineChannel.getFactor();
-  }
-
-  /**
-   * Returns the first machine in the set of datanodes.
-   *
-   * @return First Machine.
-   */
-  @JsonIgnore
-  public DatanodeID getLeader() {
-    return pipelineChannel.getDatanodes().get(pipelineChannel.getLeaderID());
-  }
-
-  /**
-   * Returns the leader host.
-   *
-   * @return First Machine.
-   */
-  public String getLeaderHost() {
-    return pipelineChannel.getDatanodes()
-        .get(pipelineChannel.getLeaderID()).getHostName();
-  }
-
-  /**
-   * Returns all machines that make up this pipeline.
-   *
-   * @return List of Machines.
-   */
-  @JsonIgnore
-  public List<DatanodeID> getMachines() {
-    return new ArrayList<>(pipelineChannel.getDatanodes().values());
-  }
-
-  /**
-   * Returns all machines that make up this pipeline.
-   *
-   * @return List of Machines.
-   */
-  public List<String> getDatanodeHosts() {
-    List<String> dataHosts = new ArrayList<>();
-    for (DatanodeID id : pipelineChannel.getDatanodes().values()) {
-      dataHosts.add(id.getHostName());
-    }
-    return dataHosts;
-  }
-
-  /**
-   * Return a Protobuf Pipeline message from pipeline.
-   *
-   * @return Protobuf message
-   */
-  @JsonIgnore
-  public OzoneProtos.Pipeline getProtobufMessage() {
-    OzoneProtos.Pipeline.Builder builder =
-        OzoneProtos.Pipeline.newBuilder();
-    builder.setContainerName(this.containerName);
-    builder.setPipelineChannel(this.pipelineChannel.getProtobufMessage());
-    return builder.build();
-  }
-
-  /**
-   * Returns containerName if available.
-   *
-   * @return String.
-   */
-  public String getContainerName() {
-    return containerName;
-  }
-
-  /**
-   * Returns private data that is set on this pipeline.
-   *
-   * @return blob, the user can interpret it any way they like.
-   */
-  public byte[] getData() {
-    if (this.data != null) {
-      return Arrays.copyOf(this.data, this.data.length);
-    } else {
-      return null;
-    }
-  }
-
-  @VisibleForTesting
-  public PipelineChannel getPipelineChannel() {
-    return pipelineChannel;
-  }
-
-  /**
-   * Set private data on pipeline.
-   *
-   * @param data -- private data.
-   */
-  public void setData(byte[] data) {
-    if (data != null) {
-      this.data = Arrays.copyOf(data, data.length);
-    }
-  }
-
-  /**
-   * Gets the State of the pipeline.
-   *
-   * @return - LifeCycleStates.
-   */
-  public OzoneProtos.LifeCycleState getLifeCycleState() {
-    return pipelineChannel.getLifeCycleState();
-  }
-
-  /**
-   * Gets the pipeline Name.
-   *
-   * @return - Name of the pipeline
-   */
-  public String getPipelineName() {
-    return pipelineChannel.getName();
-  }
-
-  /**
-   * Returns the type.
-   *
-   * @return type - Standalone, Ratis, Chained.
-   */
-  public OzoneProtos.ReplicationType getType() {
-    return pipelineChannel.getType();
-  }
-
-  @Override
-  public String toString() {
-    final StringBuilder b = new StringBuilder(getClass().getSimpleName())
-        .append("[");
-    pipelineChannel.getDatanodes().keySet().stream()
-        .forEach(id -> b.
-            append(id.endsWith(pipelineChannel.getLeaderID()) ? "*" + id : id));
-    b.append("] container:").append(containerName);
-    b.append(" name:").append(getPipelineName());
-    if (getType() != null) {
-      b.append(" type:").append(getType().toString());
-    }
-    if (getFactor() != null) {
-      b.append(" factor:").append(getFactor().toString());
-    }
-    if (getLifeCycleState() != null) {
-      b.append(" State:").append(getLifeCycleState().toString());
-    }
-    return b.toString();
-  }
-
-  /**
-   * Returns a JSON string of this object.
-   *
-   * @return String - json string
-   * @throws IOException
-   */
-  public String toJsonString() throws IOException {
-    return WRITER.writeValueAsString(this);
-  }
-
-  @JsonFilter(PIPELINE_INFO)
-  class MixIn {
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java
deleted file mode 100644
index 0af0534..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.container.common.helpers;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-
-import java.util.Map;
-import java.util.TreeMap;
-
-/**
- * PipelineChannel information for a {@link Pipeline}.
- */
-public class PipelineChannel {
-  @JsonIgnore
-  private String leaderID;
-  @JsonIgnore
-  private Map<String, DatanodeID> datanodes;
-  private LifeCycleState lifeCycleState;
-  private ReplicationType type;
-  private ReplicationFactor factor;
-  private String name;
-
-  public PipelineChannel(String leaderID, LifeCycleState lifeCycleState,
-      ReplicationType replicationType, ReplicationFactor replicationFactor,
-      String name) {
-    this.leaderID = leaderID;
-    this.lifeCycleState = lifeCycleState;
-    this.type = replicationType;
-    this.factor = replicationFactor;
-    this.name = name;
-    datanodes = new TreeMap<>();
-  }
-
-  public String getLeaderID() {
-    return leaderID;
-  }
-
-  public Map<String, DatanodeID> getDatanodes() {
-    return datanodes;
-  }
-
-  public LifeCycleState getLifeCycleState() {
-    return lifeCycleState;
-  }
-
-  public ReplicationType getType() {
-    return type;
-  }
-
-  public ReplicationFactor getFactor() {
-    return factor;
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public void addMember(DatanodeID dataNodeId) {
-    datanodes.put(dataNodeId.getDatanodeUuid(), dataNodeId);
-  }
-
-  @JsonIgnore
-  public OzoneProtos.PipelineChannel getProtobufMessage() {
-    OzoneProtos.PipelineChannel.Builder builder =
-        OzoneProtos.PipelineChannel.newBuilder();
-    for (DatanodeID datanode : datanodes.values()) {
-      builder.addMembers(datanode.getProtoBufMessage());
-    }
-    builder.setLeaderID(leaderID);
-
-    if (this.getLifeCycleState() != null) {
-      builder.setState(this.getLifeCycleState());
-    }
-    if (this.getType() != null) {
-      builder.setType(this.getType());
-    }
-
-    if (this.getFactor() != null) {
-      builder.setFactor(this.getFactor());
-    }
-    return builder.build();
-  }
-
-  public static PipelineChannel getFromProtoBuf(
-      OzoneProtos.PipelineChannel transportProtos) {
-    Preconditions.checkNotNull(transportProtos);
-    PipelineChannel pipelineChannel =
-        new PipelineChannel(transportProtos.getLeaderID(),
-            transportProtos.getState(),
-            transportProtos.getType(),
-            transportProtos.getFactor(),
-            transportProtos.getName());
-
-    for (HdfsProtos.DatanodeIDProto dataID : transportProtos.getMembersList()) {
-      pipelineChannel.addMember(DatanodeID.getFromProtoBuf(dataID));
-    }
-    return pipelineChannel;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java
deleted file mode 100644
index 549fced..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.scm.container.common.helpers;
-
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-
-import java.io.IOException;
-
-/**
- * Exceptions thrown from the Storage Container.
- */
-public class StorageContainerException extends IOException {
-  private ContainerProtos.Result result;
-
-  /**
-   * Constructs an {@code IOException} with {@code null}
-   * as its error detail message.
-   */
-  public StorageContainerException(ContainerProtos.Result result) {
-    this.result = result;
-  }
-
-  /**
-   * Constructs an {@code IOException} with the specified detail message.
-   *
-   * @param message The detail message (which is saved for later retrieval by
-   * the {@link #getMessage()} method)
-   * @param result - The result code
-   */
-  public StorageContainerException(String message,
-      ContainerProtos.Result result) {
-    super(message);
-    this.result = result;
-  }
-
-  /**
-   * Constructs an {@code IOException} with the specified detail message
-   * and cause.
-   * <p>
-   * <p> Note that the detail message associated with {@code cause} is
-   * <i>not</i> automatically incorporated into this exception's detail
-   * message.
-   *
-   * @param message The detail message (which is saved for later retrieval by
-   * the {@link #getMessage()} method)
-   *
-   * @param cause The cause (which is saved for later retrieval by the {@link
-   * #getCause()} method).  (A null value is permitted, and indicates that the
-   * cause is nonexistent or unknown.)
-   *
-   * @param result - The result code
-   * @since 1.6
-   */
-  public StorageContainerException(String message, Throwable cause,
-      ContainerProtos.Result result) {
-    super(message, cause);
-    this.result = result;
-  }
-
-  /**
-   * Constructs an {@code IOException} with the specified cause and a
-   * detail message of {@code (cause==null ? null : cause.toString())}
-   * (which typically contains the class and detail message of {@code cause}).
-   * This constructor is useful for IO exceptions that are little more
-   * than wrappers for other throwables.
-   *
-   * @param cause The cause (which is saved for later retrieval by the {@link
-   * #getCause()} method).  (A null value is permitted, and indicates that the
-   * cause is nonexistent or unknown.)
-   * @param result - The result code
-   * @since 1.6
-   */
-  public StorageContainerException(Throwable cause, ContainerProtos.Result
-      result) {
-    super(cause);
-    this.result = result;
-  }
-
-  /**
-   * Returns Result.
-   *
-   * @return Result.
-   */
-  public ContainerProtos.Result getResult() {
-    return result;
-  }
-
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java
deleted file mode 100644
index 3fa9663..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.container.common.helpers;
-/**
- Contains protocol buffer helper classes and utilites used in
- impl.
- **/
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/package-info.java
deleted file mode 100644
index ad24f98..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm;
-
-/**
- * This package contains classes for the client of the storage container
- * protocol.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java
deleted file mode 100644
index 469dab5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.protocol;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-
-import java.util.Set;
-
-/**
- * Holds the nodes that currently host the container for an object key hash.
- */
-@InterfaceAudience.Private
-public final class LocatedContainer {
-  private final String key;
-  private final String matchedKeyPrefix;
-  private final String containerName;
-  private final Set<DatanodeInfo> locations;
-  private final DatanodeInfo leader;
-
-  /**
-   * Creates a LocatedContainer.
-   *
-   * @param key object key
-   * @param matchedKeyPrefix prefix of key that was used to find the location
-   * @param containerName container name
-   * @param locations nodes that currently host the container
-   * @param leader node that currently acts as pipeline leader
-   */
-  public LocatedContainer(String key, String matchedKeyPrefix,
-      String containerName, Set<DatanodeInfo> locations, DatanodeInfo leader) {
-    this.key = key;
-    this.matchedKeyPrefix = matchedKeyPrefix;
-    this.containerName = containerName;
-    this.locations = locations;
-    this.leader = leader;
-  }
-
-  /**
-   * Returns the container name.
-   *
-   * @return container name
-   */
-  public String getContainerName() {
-    return this.containerName;
-  }
-
-  /**
-   * Returns the object key.
-   *
-   * @return object key
-   */
-  public String getKey() {
-    return this.key;
-  }
-
-  /**
-   * Returns the node that currently acts as pipeline leader.
-   *
-   * @return node that currently acts as pipeline leader
-   */
-  public DatanodeInfo getLeader() {
-    return this.leader;
-  }
-
-  /**
-   * Returns the nodes that currently host the container.
-   *
-   * @return Set<DatanodeInfo> nodes that currently host the container
-   */
-  public Set<DatanodeInfo> getLocations() {
-    return this.locations;
-  }
-
-  /**
-   * Returns the prefix of the key that was used to find the location.
-   *
-   * @return prefix of the key that was used to find the location
-   */
-  public String getMatchedKeyPrefix() {
-    return this.matchedKeyPrefix;
-  }
-
-  @Override
-  public boolean equals(Object otherObj) {
-    if (otherObj == null) {
-      return false;
-    }
-    if (!(otherObj instanceof LocatedContainer)) {
-      return false;
-    }
-    LocatedContainer other = (LocatedContainer)otherObj;
-    return this.key == null ? other.key == null : this.key.equals(other.key);
-  }
-
-  @Override
-  public int hashCode() {
-    return key.hashCode();
-  }
-
-  @Override
-  public String toString() {
-    return getClass().getSimpleName()
-        + "{key=" + key
-        + "; matchedKeyPrefix=" + matchedKeyPrefix
-        + "; containerName=" + containerName
-        + "; locations=" + locations
-        + "; leader=" + leader
-        + "}";
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java
deleted file mode 100644
index 9728554..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.protocol;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.scm.ScmInfo;
-
-/**
- * ScmBlockLocationProtocol is used by an HDFS node to find the set of nodes
- * to read/write a block.
- */
-public interface ScmBlockLocationProtocol {
-
-  /**
-   * Find the set of nodes to read/write a block, as
-   * identified by the block key.  This method supports batch lookup by
-   * passing multiple keys.
-   *
-   * @param keys batch of block keys to find
-   * @return allocated blocks for each block key
-   * @throws IOException if there is any failure
-   */
-  Set<AllocatedBlock> getBlockLocations(Set<String> keys) throws IOException;
-
-  /**
-   * Asks SCM where a block should be allocated. SCM responds with the
-   * set of datanodes that should be used creating this block.
-   * @param size - size of the block.
-   * @return allocated block accessing info (key, pipeline).
-   * @throws IOException
-   */
-  AllocatedBlock allocateBlock(long size, ReplicationType type,
-      ReplicationFactor factor, String owner) throws IOException;
-
-  /**
-   * Delete blocks for a set of object keys.
-   *
-   * @param keyBlocksInfoList Map of object key and its blocks.
-   * @return list of block deletion results.
-   * @throws IOException if there is any failure.
-   */
-  List<DeleteBlockGroupResult>
-      deleteKeyBlocks(List<BlockGroup> keyBlocksInfoList) throws IOException;
-
-  /**
-   * Gets the Clusterid and SCM Id from SCM.
-   */
-  ScmInfo getScmInfo() throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java
deleted file mode 100644
index 4e4b3d6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.protocol;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-
-/**
- * Holds the nodes that currently host the block for a block key.
- */
-@InterfaceAudience.Private
-public final class ScmLocatedBlock {
-  private final String key;
-  private final List<DatanodeInfo> locations;
-  private final DatanodeInfo leader;
-
-  /**
-   * Creates a ScmLocatedBlock.
-   *
-   * @param key object key
-   * @param locations nodes that currently host the block
-   * @param leader node that currently acts as pipeline leader
-   */
-  public ScmLocatedBlock(final String key, final List<DatanodeInfo> locations,
-      final DatanodeInfo leader) {
-    this.key = key;
-    this.locations = locations;
-    this.leader = leader;
-  }
-
-  /**
-   * Returns the object key.
-   *
-   * @return object key
-   */
-  public String getKey() {
-    return this.key;
-  }
-
-  /**
-   * Returns the node that currently acts as pipeline leader.
-   *
-   * @return node that currently acts as pipeline leader
-   */
-  public DatanodeInfo getLeader() {
-    return this.leader;
-  }
-
-  /**
-   * Returns the nodes that currently host the block.
-   *
-   * @return List<DatanodeInfo> nodes that currently host the block
-   */
-  public List<DatanodeInfo> getLocations() {
-    return this.locations;
-  }
-
-  @Override
-  public boolean equals(Object otherObj) {
-    if (otherObj == null) {
-      return false;
-    }
-    if (!(otherObj instanceof ScmLocatedBlock)) {
-      return false;
-    }
-    ScmLocatedBlock other = (ScmLocatedBlock)otherObj;
-    return this.key == null ? other.key == null : this.key.equals(other.key);
-  }
-
-  @Override
-  public int hashCode() {
-    return key.hashCode();
-  }
-
-  @Override
-  public String toString() {
-    return getClass().getSimpleName() + "{key=" + key + "; locations="
-        + locations.stream().map(loc -> loc.toString()).collect(Collectors
-            .joining(",")) + "; leader=" + leader + "}";
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java
deleted file mode 100644
index ad78bd8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.scm.protocol;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.List;
-
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
-import org.apache.hadoop.scm.ScmInfo;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-/**
- * ContainerLocationProtocol is used by an HDFS node to find the set of nodes
- * that currently host a container.
- */
-public interface StorageContainerLocationProtocol {
-  /**
-   * Asks SCM where a container should be allocated. SCM responds with the
-   * set of datanodes that should be used creating this container.
-   *
-   */
-  Pipeline allocateContainer(OzoneProtos.ReplicationType replicationType,
-      OzoneProtos.ReplicationFactor factor, String containerName, String owner)
-      throws IOException;
-
-  /**
-   * Ask SCM the location of the container. SCM responds with a group of
-   * nodes where this container and its replicas are located.
-   *
-   * @param containerName - Name of the container.
-   * @return Pipeline - the pipeline where container locates.
-   * @throws IOException
-   */
-  Pipeline getContainer(String containerName) throws IOException;
-
-  /**
-   * Ask SCM a list of containers with a range of container names
-   * and the limit of count.
-   * Search container names between start name(exclusive), and
-   * use prefix name to filter the result. the max size of the
-   * searching range cannot exceed the value of count.
-   *
-   * @param startName start name, if null, start searching at the head.
-   * @param prefixName prefix name, if null, then filter is disabled.
-   * @param count count, if count < 0, the max size is unlimited.(
-   *              Usually the count will be replace with a very big
-   *              value instead of being unlimited in case the db is very big)
-   *
-   * @return a list of container.
-   * @throws IOException
-   */
-  List<ContainerInfo> listContainer(String startName, String prefixName,
-      int count) throws IOException;
-
-  /**
-   * Deletes a container in SCM.
-   *
-   * @param containerName
-   * @throws IOException
-   *   if failed to delete the container mapping from db store
-   *   or container doesn't exist.
-   */
-  void deleteContainer(String containerName) throws IOException;
-
-  /**
-   *  Queries a list of Node Statuses.
-   * @param nodeStatuses
-   * @return List of Datanodes.
-   */
-  OzoneProtos.NodePool queryNode(EnumSet<OzoneProtos.NodeState> nodeStatuses,
-      OzoneProtos.QueryScope queryScope, String poolName) throws IOException;
-
-  /**
-   * Notify from client when begin or finish creating objects like pipeline
-   * or containers on datanodes.
-   * Container will be in Operational state after that.
-   * @param type object type
-   * @param name object name
-   * @param op operation type (e.g., create, close, delete)
-   * @param stage creation stage
-   */
-  void notifyObjectStageChange(
-      ObjectStageChangeRequestProto.Type type, String name,
-      ObjectStageChangeRequestProto.Op op,
-      ObjectStageChangeRequestProto.Stage stage) throws IOException;
-
-  /**
-   * Creates a replication pipeline of a specified type.
-   * @param type - replication type
-   * @param factor - factor 1 or 3
-   * @param nodePool - optional machine list to build a pipeline.
-   * @throws IOException
-   */
-  Pipeline createReplicationPipeline(OzoneProtos.ReplicationType type,
-      OzoneProtos.ReplicationFactor factor, OzoneProtos.NodePool nodePool)
-      throws IOException;
-
-  /**
-   * Returns information about SCM.
-   *
-   * @return {@link ScmInfo}
-   * @throws IOException
-   */
-  ScmInfo getScmInfo() throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java
deleted file mode 100644
index 274f859..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.protocol;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java
deleted file mode 100644
index 02d973a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.scm.protocolPB;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Sets;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolTranslator;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.DeleteScmKeyBlocksRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.AllocateScmBlockRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.AllocateScmBlockResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.DeleteScmKeyBlocksResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.GetScmBlockLocationsRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.GetScmBlockLocationsResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.ScmLocatedBlockProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.KeyBlocks;
-import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.scm.ScmInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/**
- * This class is the client-side translator to translate the requests made on
- * the {@link ScmBlockLocationProtocol} interface to the RPC server
- * implementing {@link ScmBlockLocationProtocolPB}.
- */
-@InterfaceAudience.Private
-public final class ScmBlockLocationProtocolClientSideTranslatorPB
-    implements ScmBlockLocationProtocol, ProtocolTranslator, Closeable {
-
-  /**
-   * RpcController is not used and hence is set to null.
-   */
-  private static final RpcController NULL_RPC_CONTROLLER = null;
-
-  private final ScmBlockLocationProtocolPB rpcProxy;
-
-  /**
-   * Creates a new StorageContainerLocationProtocolClientSideTranslatorPB.
-   *
-   * @param rpcProxy {@link StorageContainerLocationProtocolPB} RPC proxy
-   */
-  public ScmBlockLocationProtocolClientSideTranslatorPB(
-      ScmBlockLocationProtocolPB rpcProxy) {
-    this.rpcProxy = rpcProxy;
-  }
-
-  /**
-   * Find the set of nodes to read/write a block, as
-   * identified by the block key.  This method supports batch lookup by
-   * passing multiple keys.
-   *
-   * @param keys batch of block keys to find
-   * @return allocated blocks for each block key
-   * @throws IOException if there is any failure
-   */
-  @Override
-  public Set<AllocatedBlock> getBlockLocations(Set<String> keys)
-      throws IOException {
-    GetScmBlockLocationsRequestProto.Builder req =
-        GetScmBlockLocationsRequestProto.newBuilder();
-    for (String key : keys) {
-      req.addKeys(key);
-    }
-    final GetScmBlockLocationsResponseProto resp;
-    try {
-      resp = rpcProxy.getScmBlockLocations(NULL_RPC_CONTROLLER,
-          req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    Set<AllocatedBlock> locatedBlocks =
-        Sets.newLinkedHashSetWithExpectedSize(resp.getLocatedBlocksCount());
-    for (ScmLocatedBlockProto locatedBlock : resp.getLocatedBlocksList()) {
-      locatedBlocks.add(new AllocatedBlock.Builder()
-          .setKey(locatedBlock.getKey())
-          .setPipeline(Pipeline.getFromProtoBuf(locatedBlock.getPipeline()))
-          .build());
-    }
-    return locatedBlocks;
-  }
-
-  /**
-   * Asks SCM where a block should be allocated. SCM responds with the
-   * set of datanodes that should be used creating this block.
-   * @param size - size of the block.
-   * @return allocated block accessing info (key, pipeline).
-   * @throws IOException
-   */
-  @Override
-  public AllocatedBlock allocateBlock(long size,
-      OzoneProtos.ReplicationType type, OzoneProtos.ReplicationFactor factor,
-      String owner) throws IOException {
-    Preconditions.checkArgument(size > 0, "block size must be greater than 0");
-
-    AllocateScmBlockRequestProto request =
-        AllocateScmBlockRequestProto.newBuilder().setSize(size).setType(type)
-            .setFactor(factor).setOwner(owner).build();
-    final AllocateScmBlockResponseProto response;
-    try {
-      response = rpcProxy.allocateScmBlock(NULL_RPC_CONTROLLER, request);
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    if (response.getErrorCode() !=
-        AllocateScmBlockResponseProto.Error.success) {
-      throw new IOException(response.hasErrorMessage() ?
-          response.getErrorMessage() : "Allocate block failed.");
-    }
-    AllocatedBlock.Builder builder = new AllocatedBlock.Builder()
-        .setKey(response.getKey())
-        .setPipeline(Pipeline.getFromProtoBuf(response.getPipeline()))
-        .setShouldCreateContainer(response.getCreateContainer());
-    return builder.build();
-  }
-
-  /**
-   * Delete the set of keys specified.
-   *
-   * @param keyBlocksInfoList batch of block keys to delete.
-   * @return list of block deletion results.
-   * @throws IOException if there is any failure.
-   *
-   */
-  @Override
-  public List<DeleteBlockGroupResult> deleteKeyBlocks(
-      List<BlockGroup> keyBlocksInfoList) throws IOException {
-    List<KeyBlocks> keyBlocksProto = keyBlocksInfoList.stream()
-        .map(BlockGroup::getProto).collect(Collectors.toList());
-    DeleteScmKeyBlocksRequestProto request = DeleteScmKeyBlocksRequestProto
-        .newBuilder().addAllKeyBlocks(keyBlocksProto).build();
-
-    final DeleteScmKeyBlocksResponseProto resp;
-    try {
-      resp = rpcProxy.deleteScmKeyBlocks(NULL_RPC_CONTROLLER, request);
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    List<DeleteBlockGroupResult> results =
-        new ArrayList<>(resp.getResultsCount());
-    results.addAll(resp.getResultsList().stream().map(
-        result -> new DeleteBlockGroupResult(result.getObjectKey(),
-            DeleteBlockGroupResult
-                .convertBlockResultProto(result.getBlockResultsList())))
-        .collect(Collectors.toList()));
-    return results;
-  }
-
-  /**
-   * Gets the cluster Id and Scm Id from SCM.
-   * @return ScmInfo
-   * @throws IOException
-   */
-  @Override
-  public ScmInfo getScmInfo() throws IOException {
-    OzoneProtos.GetScmInfoRequestProto request =
-        OzoneProtos.GetScmInfoRequestProto.getDefaultInstance();
-    OzoneProtos.GetScmInfoRespsonseProto resp;
-    try {
-      resp = rpcProxy.getScmInfo(NULL_RPC_CONTROLLER, request);
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    ScmInfo.Builder builder = new ScmInfo.Builder()
-        .setClusterId(resp.getClusterId())
-        .setScmId(resp.getScmId());
-    return builder.build();
-  }
-
-  @Override
-  public Object getUnderlyingProxyObject() {
-    return rpcProxy;
-  }
-
-  @Override
-  public void close() {
-    RPC.stopProxy(rpcProxy);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolPB.java
deleted file mode 100644
index 256e735..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolPB.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.protocolPB;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos
-    .ScmBlockLocationProtocolService;
-
-/**
- * Protocol used from an HDFS node to StorageContainerManager.  This extends the
- * Protocol Buffers service interface to add Hadoop-specific annotations.
- */
-@ProtocolInfo(protocolName =
-    "org.apache.hadoop.ozone.protocol.ScmBlockLocationProtocol",
-    protocolVersion = 1)
-@InterfaceAudience.Private
-public interface ScmBlockLocationProtocolPB
-    extends ScmBlockLocationProtocolService.BlockingInterface {
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[03/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/ozone.css
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/ozone.css b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/ozone.css
deleted file mode 100644
index 271ac74..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/ozone.css
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- *   Licensed to the Apache Software Foundation (ASF) under one or more
- *  contributor license agreements.  See the NOTICE file distributed with
- *  this work for additional information regarding copyright ownership.
- *  The ASF licenses this file to You under the Apache License, Version 2.0
- *  (the "License"); you may not use this file except in compliance with
- *  the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
-*/
-body {
-    padding: 40px;
-    padding-top: 60px;
-}
-.starter-template {
-    padding: 40px 15px;
-    text-align: center;
-}
-
-
-.btn {
-    border: 0 none;
-    font-weight: 700;
-    letter-spacing: 1px;
-    text-transform: uppercase;
-}
-
-.btn:focus, .btn:active:focus, .btn.active:focus {
-    outline: 0 none;
-}
-
-.table-striped > tbody > tr:nth-child(2n+1).selectedtag > td:hover {
-    background-color: #3276b1;
-}
-.table-striped > tbody > tr:nth-child(2n+1).selectedtag > td {
-    background-color: #3276b1;
-}
-.tagPanel tr.selectedtag td {
-    background-color: #3276b1;
-}
-.top-buffer { margin-top:4px; }
-
-
-.sortorder:after {
-    content: '\25b2';   // BLACK UP-POINTING TRIANGLE
-}
-.sortorder.reverse:after {
-    content: '\25bc';   // BLACK DOWN-POINTING TRIANGLE
-}
-
-.wrap-table{
-    word-wrap: break-word;
-    table-layout: fixed;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/ozone.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/ozone.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/ozone.js
deleted file mode 100644
index 37cafef..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/ozone.js
+++ /dev/null
@@ -1,355 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-(function () {
-    "use strict";
-
-    var isIgnoredJmxKeys = function (key) {
-        return key == 'name' || key == 'modelerType' || key == "$$hashKey" ||
-            key.match(/tag.*/);
-    };
-    angular.module('ozone', ['nvd3', 'ngRoute']);
-    angular.module('ozone').config(function ($routeProvider) {
-        $routeProvider
-            .when("/", {
-                templateUrl: "main.html"
-            })
-            .when("/metrics/rpc", {
-                template: "<rpc-metrics></rpc-metrics>"
-            })
-            .when("/config", {
-                template: "<config></config>"
-            })
-    });
-    angular.module('ozone').component('overview', {
-        templateUrl: 'static/templates/overview.html',
-        transclude: true,
-        controller: function ($http) {
-            var ctrl = this;
-            $http.get("jmx?qry=Hadoop:service=*,name=*,component=ServerRuntime")
-                .then(function (result) {
-                    ctrl.jmx = result.data.beans[0]
-                })
-        }
-    });
-    angular.module('ozone').component('jvmParameters', {
-        templateUrl: 'static/templates/jvm.html',
-        controller: function ($http) {
-            var ctrl = this;
-            $http.get("jmx?qry=java.lang:type=Runtime")
-                .then(function (result) {
-                    ctrl.jmx = result.data.beans[0];
-
-                    //convert array to a map
-                    var systemProperties = {};
-                    for (var idx in ctrl.jmx.SystemProperties) {
-                        var item = ctrl.jmx.SystemProperties[idx];
-                        systemProperties[item.key.replace(/\./g, "_")] = item.value;
-                    }
-                    ctrl.jmx.SystemProperties = systemProperties;
-                })
-        }
-    });
-
-    angular.module('ozone').component('rpcMetrics', {
-        template: '<h1>Rpc metrics</h1><tabs>' +
-        '<pane ng-repeat="metric in $ctrl.metrics" ' +
-        'title="{{metric[\'tag.serverName\']}} ({{metric[\'tag.port\']}})">' +
-        '<rpc-metric jmxdata="metric"></rpc-metric></pane>' +
-        '</tabs>',
-        controller: function ($http) {
-            var ctrl = this;
-            $http.get("jmx?qry=Hadoop:service=*,name=RpcActivityForPort*")
-                .then(function (result) {
-                    ctrl.metrics = result.data.beans;
-                })
-        }
-    });
-    angular.module('ozone').component('rpcMetric', {
-        bindings: {
-            jmxdata: '<'
-        },
-        templateUrl: 'static/templates/rpc-metrics.html',
-        controller: function () {
-            var ctrl = this;
-
-
-            ctrl.percentileGraphOptions = {
-                chart: {
-                    type: 'discreteBarChart',
-                    height: 450,
-                    margin: {
-                        top: 20,
-                        right: 20,
-                        bottom: 50,
-                        left: 55
-                    },
-                    x: function (d) {
-                        return d.label;
-                    },
-                    y: function (d) {
-                        return d.value;
-                    },
-                    showValues: true,
-                    valueFormat: function (d) {
-                        return d3.format(',.1f')(d);
-                    },
-                    duration: 500,
-                    xAxis: {
-                        axisLabel: 'Percentage'
-                    },
-                    yAxis: {
-                        axisLabel: 'Latency (ms)',
-                        axisLabelDistance: -10
-                    }
-                }
-            };
-
-            ctrl.$onChanges = function (data) {
-                var groupedMetrics = {}
-
-                var createPercentageMetrics = function (metricName, window) {
-                    groupedMetrics.percentiles = groupedMetrics['percentiles'] || {}
-                    groupedMetrics.percentiles[window] = groupedMetrics.percentiles[window] || {};
-                    groupedMetrics.percentiles[window][metricName] = groupedMetrics.percentiles[window][metricName] || {
-                            graphdata: [{
-                                key: window,
-                                values: []
-                            }], numOps: 0
-                        };
-
-                };
-                var metrics = ctrl.jmxdata;
-                for (var key in metrics) {
-                    var percentile = key.match(/(.*Time)(\d+s)(\d+th)PercentileLatency/);
-                    var percentileNumOps = key.match(/(.*Time)(\d+s)NumOps/);
-                    var successFailures = key.match(/(.*)(Success|Failures)/);
-                    var numAverages = key.match(/(.*Time)(NumOps|AvgTime)/);
-                    if (percentile) {
-                        var metricName = percentile[1];
-                        var window = percentile[2];
-                        var percentage = percentile[3]
-                        createPercentageMetrics(metricName, window);
-
-
-                        groupedMetrics.percentiles[window][metricName].graphdata[0]
-                            .values.push({
-                            label: percentage,
-                            value: metrics[key]
-                        })
-                    } else if (successFailures) {
-                        var metricName = successFailures[1];
-                        groupedMetrics.successfailures = groupedMetrics['successfailures'] || {}
-                        groupedMetrics.successfailures[metricName] = groupedMetrics.successfailures[metricName] || {
-                                success: 0,
-                                failures: 0
-                            };
-                        if (successFailures[2] == 'Success') {
-                            groupedMetrics.successfailures[metricName].success = metrics[key];
-                        } else {
-                            groupedMetrics.successfailures[metricName].failures = metrics[key];
-                        }
-
-                    } else if (numAverages) {
-                        var metricName = numAverages[1];
-                        groupedMetrics.numavgs = groupedMetrics['numavgs'] || {}
-                        groupedMetrics.numavgs[metricName] = groupedMetrics.numavgs[metricName] || {
-                                numOps: 0,
-                                avgTime: 0
-                            };
-                        if (numAverages[2] == 'NumOps') {
-                            groupedMetrics.numavgs[metricName].numOps = metrics[key];
-                        } else {
-                            groupedMetrics.numavgs[metricName].avgTime = metrics[key];
-                        }
-
-                    } else if (percentileNumOps) {
-                        var metricName = percentileNumOps[1];
-                        var window = percentileNumOps[2];
-                        createPercentageMetrics(metricName, window);
-                        groupedMetrics.percentiles[window][metricName].numOps = metrics[key];
-                    } else if (isIgnoredJmxKeys(key)) {
-                        //ignore
-                    } else {
-                        groupedMetrics.others = groupedMetrics.others || [];
-                        groupedMetrics.others.push({
-                            'key': key,
-                            'value': metrics[key]
-                        });
-                    }
-
-                }
-                ctrl.metrics = groupedMetrics;
-            };
-
-        }
-    });
-    angular.module('ozone')
-        .component('tabs', {
-            transclude: true,
-            controller: function ($scope) {
-                var ctrl = this;
-                var panes = this.panes = [];
-                this.select = function (pane) {
-                    angular.forEach(panes, function (pane) {
-                        pane.selected = false;
-                    });
-                    pane.selected = true;
-                };
-                this.addPane = function (pane) {
-                    if (panes.length === 0) {
-                        this.select(pane);
-                    }
-                    panes.push(pane);
-                };
-                this.click = function(pane) {
-                    ctrl.select(pane);
-                }
-            },
-            template: '<div class="nav navtabs"><div class="row"><ul' +
-            ' class="nav nav-pills">' +
-            '<li ng-repeat="pane in $ctrl.panes" ng-class="{active:pane.selected}">' +
-            '<a href="" ng-click="$ctrl.click(pane)">{{pane.title}}</a> ' +
-            '</li> </ul></div><br/><div class="tab-content" ng-transclude></div> </div>'
-        })
-        .component('pane', {
-            transclude: true,
-            require: {
-                tabsCtrl: '^tabs'
-            },
-            bindings: {
-                title: '@'
-            },
-            controller: function () {
-                this.$onInit = function () {
-                    this.tabsCtrl.addPane(this);
-                };
-            },
-            template: '<div class="tab-pane" ng-if="$ctrl.selected" ng-transclude></div>'
-        });
-
-    angular.module('ozone').component('navmenu', {
-        bindings: {
-            metrics: '<'
-        },
-        templateUrl: 'static/templates/menu.html',
-        controller: function ($http) {
-            var ctrl = this;
-            ctrl.docs = false;
-            $http.head("docs/index.html")
-                .then(function (result) {
-                    ctrl.docs = true;
-                },function(){
-                    ctrl.docs = false;
-                });
-        }
-    });
-
-    angular.module('ozone').component('config', {
-        templateUrl: 'static/templates/config.html',
-        controller: function ($scope, $http) {
-            var ctrl = this;
-            ctrl.selectedTags = [];
-
-            $http.get("conf?cmd=getOzoneTags&group=ozone")
-                .then(function (response) {
-                    ctrl.tags = response.data;
-
-                    var excludedTags = ['CBLOCK', 'KSM', 'SCM'];
-                    for (var i = 0; i < excludedTags.length; i++) {
-                        var idx = ctrl.tags.indexOf(excludedTags[i]);
-                        // Remove CBLOCK related properties
-                        if (idx > -1) {
-                            ctrl.tags.splice(idx, 1);
-                        }
-                    }
-                    ctrl.loadAll();
-                });
-
-
-
-            ctrl.loadAll = function () {
-                console.log("Displaying all configs");
-                $http.get("conf?cmd=getPropertyByTag&tags=" + ctrl.tags + "&group=ozone").then(function (response) {
-                    ctrl.configs = response.data;
-                    console.log(ctrl.configs)
-                    for (var idx in ctrl.configs) {
-                        var tags = []
-                        var parsedTags = ctrl.configs[idx].tag.split(",");
-                        for (var t in parsedTags) {
-                            tags.push(parsedTags[t].trim())
-                        }
-                        ctrl.configs[idx].tag = tags;
-
-                    };
-                    ctrl.sortBy('name');
-                });
-            };
-
-            ctrl.tagFilter = function (value, index, array) {
-                if (!ctrl.selectedTags) {
-                    return true;
-                }
-                var selected = true;
-                for (var idx in ctrl.selectedTags) {
-                    selected = selected && (value.tag.indexOf(ctrl.selectedTags[idx]) > -1);
-                }
-                return selected;
-            };
-            ctrl.configFilter = function (config) {
-                return false;
-            };
-            ctrl.selected = function (tag) {
-                return ctrl.selectedTags.includes(tag);
-            };
-
-            ctrl.allSelected = function () {
-                return ctrl.selectedTags.indexOf('SCM') == -1
-                    && ctrl.selectedTags.indexOf('KSM') == -1
-            };
-
-            ctrl.switchto = function (tag) {
-                var tags = ctrl.selectedTags.filter(function (item) {
-                    return item != 'KSM' && item != 'SCM';
-                });
-                if (tag) {
-                    tags.push(tag);
-                }
-                ctrl.selectedTags = tags;
-            };
-
-            ctrl.select = function (tag) {
-                var tagIdx = ctrl.selectedTags.indexOf(tag);
-                if (tagIdx > -1) {
-                    ctrl.selectedTags = ctrl.selectedTags.filter(function (item) {
-                        return item != tag;
-                    });
-                } else {
-                    ctrl.selectedTags.push(tag);
-                }
-                console.log("Tags selected:" + ctrl.selectedTags);
-            };
-
-            ctrl.sortBy = function (propertyName) {
-                ctrl.reverse = (ctrl.propertyName === propertyName) ? !ctrl.reverse : false;
-                ctrl.propertyName = propertyName;
-            };
-
-        }
-    });
-
-})();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/config.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/config.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/config.html
deleted file mode 100644
index 0a273c3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/config.html
+++ /dev/null
@@ -1,92 +0,0 @@
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-  -->
-
-<div class="row top-buffer">
-  <div class="col-md-2">
-    <input type="text" class="form-control" placeholder="Search Properties"
-           name="search" ng-model="search.$">
-  </div>
-  <div class="col-md-10">
-    <div class="btn-group btn-group-justified">
-      <a class="btn"
-         ng-class="$ctrl.allSelected() ? 'btn-primary' :'btn-secondary'"
-         ng-click="$ctrl.switchto('')">All
-      </a>
-      <a class="btn"
-         ng-class="$ctrl.selected('KSM') ? 'btn-primary' :'btn-secondary'"
-         ng-click="$ctrl.switchto('KSM')">KSM</a>
-      <a class="btn"
-         ng-class="$ctrl.selected('SCM') ? 'btn-primary' :'btn-secondary'"
-         ng-click="$ctrl.switchto('SCM')">SCM</a>
-    </div>
-  </div>
-</div>
-<div class="row">
-  <div class="col-md-2">
-
-    <table class="table table-striped table-condensed tagPanel">
-      <colgroup>
-        <col class="col-md-12">
-      </colgroup>
-      <thead>
-      <tr>
-        <th>Tag</th>
-      </tr>
-      </thead>
-      <tbody>
-      <tr ng-click="$ctrl.select(tag)"
-          ng-class="$ctrl.selected(tag) ? 'selectedtag':''"
-          ng-repeat="tag in $ctrl.tags">
-        <td>{{tag}}</td>
-      </tr>
-      </tbody>
-    </table>
-  </div>
-  <div class="col-md-10">
-    <table class="table table-striped table-condensed table-hover wrap-table">
-      <thead>
-      <tr>
-        <th class="col-md-3" >
-          <a href="#" ng-click="$ctrl.sortBy('name')">Property</a>
-          <span class="sortorder" ng-show="propertyName === 'name'"
-                ng-class="{reverse: reverse}">
-
-              </span>
-        </th>
-        <th class="col-md-2" style="word-wrap: break-word;">
-          <a ng-click="$ctrl.sortBy('value')">Value</a>
-          <span class="sortorder" ng-show="propertyName === 'value'"
-                ng-class="{reverse: reverse}"></span>
-        </th>
-        <th class="col-md-7">
-          <a href="#" ng-click="$ctrl.sortBy('description')">Description</a>
-          <span class="sortorder" ng-show="propertyName === 'description'"
-                ng-class="{reverse: reverse}"></span>
-        </th>
-      </tr>
-      </thead>
-      <tbody>
-      <tr
-              ng-repeat="config in $ctrl.configs | filter:$ctrl.tagFilter | filter:search | orderBy:propertyName:reverse">
-        <td style="word-wrap: break-word;">{{config.name}}</td>
-        <td style="word-wrap: break-word;">{{config.value}}</td>
-        <td style="word-wrap: break-word;">{{config.description}}</td>
-      </tr>
-      </tbody>
-    </table>
-  </div>
-</div>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/jvm.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/jvm.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/jvm.html
deleted file mode 100644
index c1f7d16..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/jvm.html
+++ /dev/null
@@ -1,26 +0,0 @@
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<table class="table table-bordered table-striped">
-    <tr>
-        <th>JVM:</th>
-        <td>{{$ctrl.jmx.SystemProperties.java_vm_name}} {{$ctrl.jmx.SystemProperties.java_vm_version}}</td>
-    </tr>
-    <tr>
-        <th>Input arguments:</th>
-        <td>{{$ctrl.jmx.InputArguments}}</td>
-    </tr>
-</table>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/menu.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/menu.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/menu.html
deleted file mode 100644
index 95f1b48..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/menu.html
+++ /dev/null
@@ -1,60 +0,0 @@
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<div id="navbar" class="collapse navbar-collapse">
-    <ul class="nav navbar-nav" id="ui-tabs">
-        <li>
-            <a class="dropdown-toggle"
-               id="metricsMenu"
-               data-toggle="dropdown"
-               aria-haspopup="true"
-               aria-expanded="true">
-                Metrics
-                <span class="caret"></span>
-            </a>
-            <ul
-                class="dropdown-menu"
-                aria-labelledby="metricsMenu">
-                <li ng-repeat="(name, url) in $ctrl.metrics">
-                    <a ng-href="{{url}}">{{name}}<span
-                        aria-hidden="true"></span></a></li>
-            </ul>
-        </li>
-        <li><a href="#!/config">Configuration</a></li>
-        <li ng-show="$ctrl.docs"><a href="/docs">Documentation</a></li>
-        <li>
-            <a class="dropdown-toggle"
-               id="toolsMenu"
-               data-toggle="dropdown"
-               aria-haspopup="true"
-               aria-expanded="true"
-               >
-                Common tools
-                <span class="caret"></span>
-            </a>
-            <ul class="dropdown-menu" aria-labelledby="toolsMenu">
-                <li><a href="jmx">JMX <span
-                        aria-hidden="true"></span></a></li>
-                <li><a href="conf">Config <span
-                        aria-hidden="true"></a></li>
-                <li><a href="stacks">Stacks <span
-                        aria-hidden="true"></a></li>
-                <li><a href="logLevel">Log levels <span
-                        aria-hidden="true"></a></li>
-            </ul>
-        </li>
-    </ul>
-</div><!--/.nav-collapse -->

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/overview.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/overview.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/overview.html
deleted file mode 100644
index 30e2d26..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/overview.html
+++ /dev/null
@@ -1,39 +0,0 @@
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<h1>Overview</h1>
-<table class="table table-bordered table-striped">
-    <tbody>
-    <tr>
-        <th>Started:</th>
-        <td>{{$ctrl.jmx.StartedTimeInMillis | date : 'medium'}}</td>
-    </tr>
-    <tr>
-        <th>Version:</th>
-        <td>{{$ctrl.jmx.Version}}</td>
-    </tr>
-    <tr>
-        <th>Compiled:</th>
-        <td>{{$ctrl.jmx.CompileInfo}}</td>
-    </tr>
-    </tbody>
-</table>
-
-<h2>JVM parameters</h2>
-
-<jvm-parameters></jvm-parameters>
-
-<div ng-transclude></div>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/rpc-metrics.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/rpc-metrics.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/rpc-metrics.html
deleted file mode 100644
index facb152..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/rpc-metrics.html
+++ /dev/null
@@ -1,87 +0,0 @@
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<div ng-hide="$ctrl.metrics.percentiles" class="alert alert-info">
-    Please set <b>rpc.metrics.quantile.enable</b> to <b>true</b> and define the
-    intervals in seconds with setting <b>rpc.metrics.percentiles.intervals</b>
-    (eg. set to <b>60,300</b>) in your hdfs-site.xml
-    to display Hadoop RPC related graphs.
-</div>
-<div ng-repeat="(window,windowed) in $ctrl.metrics.percentiles">
-    <h2>{{window}} window</h2>
-    <p>Quantiles based on a fixed {{window}} window. Calculated once at every
-        {{window}}</p>
-
-    <div class="row">
-        <div class="col-md-6 col-lg-4"
-             ng-repeat="(metric,percentiles) in windowed">
-            <h3>{{metric}}</h3>
-            <p>{{percentiles.numOps}} sample</p>
-            <nvd3 options="$ctrl.percentileGraphOptions"
-                  data="percentiles.graphdata"></nvd3>
-        </div>
-    </div>
-
-</div>
-<div class="row">
-    <div ng-show="$ctrl.metrics.numavgs" class="col-md-6">
-        <h2>Number of ops / Averages</h2>
-
-        <table class="table table-bordered table-striped">
-            <thead>
-            <tr>
-                <th>Metric name</th>
-                <th>Number of ops</th>
-                <th>Average time (ms)</th>
-            </tr>
-            </thead>
-            <tr ng-repeat="(key,metric) in $ctrl.metrics.numavgs">
-                <td>{{key}}</td>
-                <td>{{metric.numOps | number}}</td>
-                <td>{{metric.avgTime | number:2}}</td>
-            </tr>
-        </table>
-    </div>
-    <div ng-show="$ctrl.metrics.successfailures" class="col-md-6">
-        <h2>Success / Failures</h2>
-
-        <table class="table table-bordered table-striped">
-            <thead>
-            <tr>
-                <th>Metric name</th>
-                <th>Success</th>
-                <th>Failures</th>
-            </tr>
-            </thead>
-
-            <tr ng-repeat="(key,metric) in $ctrl.metrics.successfailures">
-                <td>{{key}}</td>
-                <td>{{metric.success}}</td>
-                <td>{{metric.failures}}</td>
-            </tr>
-        </table>
-    </div>
-</div>
-<div ng-show="$ctrl.metrics.others">
-    <h2>Other JMX Metrics</h2>
-
-    <table class="table">
-        <tr ng-repeat="metric in $ctrl.metrics.others">
-            <td>{{metric.key}}</td>
-            <td>{{metric.value}}</td>
-        </tr>
-    </table>
-</div>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneCommandShell.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneCommandShell.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneCommandShell.md
deleted file mode 100644
index 9df974f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneCommandShell.md
+++ /dev/null
@@ -1,150 +0,0 @@
-<!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-
-Ozone Command Shell
-===================
-
-Ozone command shell gives a command shell interface to work against ozone.
-Please note that this  document assumes that cluster is deployed
-with simple authentication.
-
-The Ozone commands take the following format.
-
-* `hdfs oz --command_ http://hostname:port/volume/bucket/key -user
-<name> -root`
-
-The *port* specified in command should match the port mentioned in the config
-property `dfs.datanode.http.address`. This property can be set in `hdfs-site.xml`.
-The default value for the port is `9864` and is used in below commands.
-
-The *--root* option is a command line short cut that allows *hdfs oz*
-commands to be run as the user that started the cluster. This is useful to
-indicate that you want the commands to be run as some admin user. The only
-reason for this option is that it makes the life of a lazy developer more
-easier.
-
-Ozone Volume Commands
---------------------
-
-The volume commands allow users to create, delete and list the volumes in the
-ozone cluster.
-
-### Create Volume
-
-Volumes can be created only by Admins. Here is an example of creating a volume.
-
-* `hdfs oz -createVolume http://localhost:9864/hive -user bilbo -quota
-100TB -root`
-
-The above command creates a volume called `hive` owned by user `bilbo`. The
-`--root` option allows the command to be executed as user `hdfs` which is an
-admin in the cluster.
-
-### Update Volume
-
-Updates information like ownership and quota on an existing volume.
-
-* `hdfs oz  -updateVolume  http://localhost:9864/hive -quota 500TB -root`
-
-The above command changes the volume quota of hive from 100TB to 500TB.
-
-### Delete Volume
-Deletes a Volume if it is empty.
-
-* `hdfs oz -deleteVolume http://localhost:9864/hive -root`
-
-
-### Info Volume
-Info volume command allows the owner or the administrator of the cluster to read meta-data about a specific volume.
-
-* `hdfs oz -infoVolume http://localhost:9864/hive -root`
-
-### List Volumes
-
-List volume command can be used by administrator to list volumes of any user. It can also be used by a user to list volumes owned by him.
-
-* `hdfs oz -listVolume http://localhost:9864/ -user bilbo -root`
-
-The above command lists all volumes owned by user bilbo.
-
-Ozone Bucket Commands
---------------------
-
-Bucket commands follow a similar pattern as volume commands. However bucket commands are designed to be run by the owner of the volume.
-Following examples assume that these commands are run by the owner of the volume or bucket.
-
-
-### Create Bucket
-
-Create bucket call allows the owner of a volume to create a bucket.
-
-* `hdfs oz -createBucket http://localhost:9864/hive/january`
-
-This call creates a bucket called `january` in the volume called `hive`. If
-the volume does not exist, then this call will fail.
-
-
-### Update Bucket
-Updates bucket meta-data, like ACLs.
-
-* `hdfs oz -updateBucket http://localhost:9864/hive/january  -addAcl
-user:spark:rw`
-
-### Delete Bucket
-Deletes a bucket if it is empty.
-
-* `hdfs oz -deleteBucket http://localhost:9864/hive/january`
-
-### Info Bucket
-Returns information about a given bucket.
-
-* `hdfs oz -infoBucket http://localhost:9864/hive/january`
-
-### List Buckets
-List buckets on a given volume.
-
-* `hdfs oz -listBucket http://localhost:9864/hive`
-
-Ozone Key Commands
-------------------
-
-Ozone key commands allows users to put, delete and get keys from ozone buckets.
-
-### Put Key
-Creates or overwrites a key in ozone store, -file points to the file you want
-to upload.
-
-* `hdfs oz -putKey  http://localhost:9864/hive/january/processed.orc  -file
-processed.orc`
-
-### Get Key
-Downloads a file from the ozone bucket.
-
-* `hdfs oz -getKey  http://localhost:9864/hive/january/processed.orc  -file
-  processed.orc.copy`
-
-### Delete Key
-Deletes a key  from the ozone store.
-
-* `hdfs oz -deleteKey http://localhost:9864/hive/january/processed.orc`
-
-### Info Key
-Reads  key metadata from the ozone store.
-
-* `hdfs oz -infoKey http://localhost:9864/hive/january/processed.orc`
-
-### List Keys
-List all keys in an ozone bucket.
-
-* `hdfs oz -listKey  http://localhost:9864/hive/january`

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm
deleted file mode 100644
index 3395a8e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm
+++ /dev/null
@@ -1,320 +0,0 @@
-<!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-Ozone - Object store for Hadoop
-==============================
-
-Introduction
-------------
-Ozone is an object store for Hadoop. It  is a redundant, distributed object
-store build by leveraging primitives present in HDFS. Ozone supports REST
-API for accessing the store.
-
-Getting Started
----------------
-Ozone is a work in progress and  currently lives in its own branch. To
-use it, you have to build a package by yourself and deploy a cluster.
-
-### Building Ozone
-
-To build Ozone, please checkout the hadoop sources from github. Then
-checkout the ozone branch, HDFS-7240 and build it.
-
-- `git checkout HDFS-7240`
-- `mvn clean package -DskipTests=true -Dmaven.javadoc.skip=true -Pdist -Dtar -DskipShade`
-
-skipShade is just to make compilation faster and not really required.
-
-This will give you a tarball in your distribution directory. This is the
-tarball that can be used for deploying your hadoop cluster. Here is an
-example of the tarball that will be generated.
-
-* `~/apache/hadoop/hadoop-dist/target/${project.version}.tar.gz`
-
-At this point we have an option to setup a physical cluster or run ozone via
-docker.
-
-Running Ozone via Docker
-------------------------
-
-This assumes that you have a running docker setup on the machine. Please run
-these following commands to see ozone in action.
-
- Go to the directory where the docker compose files exist.
-
-
- - `cd dev-support/compose/ozone`
-
-Tell docker to start ozone, this will start a KSM, SCM and a single datanode in
-the background.
-
-
- - `docker-compose up -d`
-
-Now let us run some work load against ozone, to do that we will run freon.
-
-This will log into the datanode and run bash.
-
- - `docker-compose exec datanode bash`
- - `cd hadoop/bin`
-
-Now you can run the oz command shell or freon, the ozone load generator.
-
-This is the command to run freon.
-
- - `./hdfs freon -mode offline -validateWrites -numOfVolumes 1 -numOfBuckets 10 -numOfKeys 100`
-
-You can checkout the KSM UI to see the requests information.
-
- - `http://localhost:9874/`
-
-If you need more datanode you can scale up:
-
- - `docker-compose scale datanode=3`
-
-Running Ozone using a real cluster
-----------------------------------
-
-Please proceed to setup a hadoop cluster by creating the hdfs-site.xml and
-other configuration files that are needed for your cluster.
-
-
-### Ozone Configuration
-
-Ozone relies on its own configuration file called `ozone-site.xml`. It is
-just for convenience and ease of management --  you can add these settings
-to `hdfs-site.xml`, if you don't want to keep ozone settings separate.
-This document refers to `ozone-site.xml` so that ozone settings are in one
-place  and not mingled with HDFS settings.
-
- * _*ozone.enabled*_  This is the most important setting for ozone.
- Currently, Ozone is an opt-in subsystem of HDFS. By default, Ozone is
- disabled. Setting this flag to `true` enables ozone in the HDFS cluster.
- Here is an example,
-
-```
-    <property>
-       <name>ozone.enabled</name>
-       <value>True</value>
-    </property>
-```
- *  _*ozone.metadata.dirs*_ Ozone is designed with modern hardware
- in mind. It tries to use SSDs effectively. So users can specify where the
- metadata must reside. Usually you pick your fastest disk (SSD if
- you have them on your nodes). KSM, SCM and datanode will write the metadata
- to these disks. This is a required setting, if this is missing Ozone will
- fail to come up. Here is an example,
-
-```
-   <property>
-      <name>ozone.metadata.dirs</name>
-      <value>/data/disk1/meta</value>
-   </property>
-```
-
-* _*ozone.scm.names*_ Ozone is build on top of container framework (See Ozone
- Architecture TODO). Storage container manager(SCM) is a distributed block
- service which is used by ozone and other storage services.
- This property allows datanodes to discover where SCM is, so that
- datanodes can send heartbeat to SCM. SCM is designed to be highly available
- and datanodes assume there are multiple instances of SCM which form a highly
- available ring. The HA feature of SCM is a work in progress. So we
- configure ozone.scm.names to be a single machine. Here is an example,
-
-```
-    <property>
-      <name>ozone.scm.names</name>
-      <value>scm.hadoop.apache.org</value>
-    </property>
-```
-
-* _*ozone.scm.datanode.id*_ Each datanode that speaks to SCM generates an ID
-just like HDFS.  This is an optional setting. Please note:
-This path will be created by datanodes if it doesn't exist already. Here is an
- example,
-
-```
-   <property>
-      <name>ozone.scm.datanode.id</name>
-      <value>/data/disk1/scm/meta/node/datanode.id</value>
-   </property>
-```
-
-* _*ozone.scm.block.client.address*_ Storage Container Manager(SCM) offers a
- set of services that can be used to build a distributed storage system. One
- of the services offered is the block services. KSM and HDFS would use this
- service. This property describes where KSM can discover SCM's block service
- endpoint. There is corresponding ports etc, but assuming that we are using
- default ports, the server address is the only required field. Here is an
- example,
-
-```
-    <property>
-      <name>ozone.scm.block.client.address</name>
-      <value>scm.hadoop.apache.org</value>
-    </property>
-```
-
-* _*ozone.ksm.address*_ KSM server address. This is used by Ozonehandler and
-Ozone File System.
-
-```
-    <property>
-       <name>ozone.ksm.address</name>
-       <value>ksm.hadoop.apache.org</value>
-    </property>
-```
-
-Here is a quick summary of settings needed by Ozone.
-
-| Setting                        | Value                        | Comment |
-|--------------------------------|------------------------------|------------------------------------------------------------------|
-| ozone.enabled                  | True                         | This enables SCM and  containers in HDFS cluster.                |
-| ozone.metadata.dirs            | file path                    | The metadata will be stored here.                                |
-| ozone.scm.names                | SCM server name              | Hostname:port or or IP:port address of SCM.                      |
-| ozone.scm.block.client.address | SCM server name and port     | Used by services like KSM                                        |
-| ozone.scm.client.address       | SCM server name and port     | Used by client side                                              |
-| ozone.scm.datanode.address     | SCM server name and port     | Used by datanode to talk to SCM                                  |
-| ozone.ksm.address              | KSM server name              | Used by Ozone handler and Ozone file system.                     |
-
- Here is a working example of`ozone-site.xml`.
-
-```
-    <?xml version="1.0" encoding="UTF-8"?>
-    <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-    <configuration>
-      <property>
-          <name>ozone.enabled</name>
-          <value>True</value>
-        </property>
-
-        <property>
-          <name>ozone.metadata.dirs</name>
-          <value>/data/disk1/ozone/meta</value>
-        </property>
-
-        <property>
-          <name>ozone.scm.names</name>
-          <value>127.0.0.1</value>
-        </property>
-
-        <property>
-           <name>ozone.scm.client.address</name>
-           <value>127.0.0.1:9860</value>
-        </property>
-
-         <property>
-           <name>ozone.scm.block.client.address</name>
-           <value>127.0.0.1:9863</value>
-         </property>
-
-         <property>
-           <name>ozone.scm.datanode.address</name>
-           <value>127.0.0.1:9861</value>
-         </property>
-
-         <property>
-           <name>ozone.ksm.address</name>
-           <value>127.0.0.1:9874</value>
-         </property>
-    </configuration>
-```
-
-### Starting Ozone
-
-Ozone is designed to run concurrently with HDFS. The simplest way to [start
-HDFS](../hadoop-common/ClusterSetup.html) is to run `start-dfs.sh` from the
-`$HADOOP/sbin/start-dfs.sh`. Once HDFS
-is running, please verify it is fully functional by running some commands like
-
-   - *./hdfs dfs -mkdir /usr*
-   - *./hdfs dfs -ls /*
-
- Once you are sure that HDFS is running, start Ozone. To start  ozone, you
- need to start SCM and KSM. Currently we assume that both KSM and SCM
-  is running on the same node, this will change in future.
-
- The first time you bring up Ozone, SCM must be initialized.
-
-   - `./hdfs scm -init`
-
- Start SCM.
-
-   - `./hdfs --daemon start scm`
-
- Once SCM gets started, KSM must be initialized.
-
-   - `./hdfs ksm -createObjectStore`
-
- Start KSM.
-
-   - `./hdfs --daemon start ksm`
-
-if you would like to start HDFS and Ozone together, you can do that by running
- a single command.
- - `$HADOOP/sbin/start-ozone.sh`
-
- This command will start HDFS and then start the ozone components.
-
- Once you have ozone running you can use these ozone [shell](./OzoneCommandShell.html)
- commands to  create a  volume, bucket and keys.
-
-### Diagnosing issues
-
-Ozone tries not to pollute the existing HDFS streams of configuration and
-logging. So ozone logs are by default configured to be written to a file
-called `ozone.log`. This is controlled by the settings in `log4j.properties`
-file in the hadoop configuration directory.
-
-Here is the log4j properties that are added by ozone.
-
-
-```
-   #
-   # Add a logger for ozone that is separate from the Datanode.
-   #
-   #log4j.debug=true
-   log4j.logger.org.apache.hadoop.ozone=DEBUG,OZONE,FILE
-
-   # Do not log into datanode logs. Remove this line to have single log.
-   log4j.additivity.org.apache.hadoop.ozone=false
-
-   # For development purposes, log both to console and log file.
-   log4j.appender.OZONE=org.apache.log4j.ConsoleAppender
-   log4j.appender.OZONE.Threshold=info
-   log4j.appender.OZONE.layout=org.apache.log4j.PatternLayout
-   log4j.appender.OZONE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p \
-    %X{component} %X{function} %X{resource} %X{user} %X{request} - %m%n
-
-   # Real ozone logger that writes to ozone.log
-   log4j.appender.FILE=org.apache.log4j.DailyRollingFileAppender
-   log4j.appender.FILE.File=${hadoop.log.dir}/ozone.log
-   log4j.appender.FILE.Threshold=debug
-   log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
-   log4j.appender.FILE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p \
-     (%F:%L) %X{function} %X{resource} %X{user} %X{request} - \
-      %m%n
-```
-
-If you would like to have a single datanode log instead of ozone stuff
-getting written to ozone.log, please remove this line or set this to true.
-
- ` log4j.additivity.org.apache.hadoop.ozone=false`
-
-On the SCM/KSM side, you will be able to see
-
-  - `hadoop-hdfs-ksm-hostname.log`
-  - `hadoop-hdfs-scm-hostname.log`
-
-Please file any issues you see under [Object store in HDFS (HDFS-7240)](https://issues.apache.org/jira/browse/HDFS-7240)
-as this is still a work in progress.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneMetrics.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneMetrics.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneMetrics.md
deleted file mode 100644
index f5eccf6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneMetrics.md
+++ /dev/null
@@ -1,166 +0,0 @@
-<!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-
-
-
-HDFS Ozone Metrics
-===============
-
-<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
-
-Overview
---------
-
-The container metrics that is used in HDFS Ozone.
-
-### Storage Container Metrics
-
-The metrics for various storage container operations in HDFS Ozone.
-
-Storage container is an optional service that can be enabled by setting
-'ozone.enabled' to true.
-These metrics are only available when ozone is enabled.
-
-Storage Container Metrics maintains a set of generic metrics for all
-container RPC calls that can be made to a datandoe/container.
-
-Along with the total number of RPC calls containers maintain a set of metrics
-for each RPC call. Following is the set of counters maintained for each RPC
-operation.
-
-*Total number of operation* - We maintain an array which counts how
-many times a specific operation has been performed.
-Eg.`NumCreateContainer` tells us how many times create container has been
-invoked on this datanode.
-
-*Total number of pending operation* - This is an array which counts how
-many times a specific operation is waitting to be processed from the client
-point of view.
-Eg.`NumPendingCreateContainer` tells us how many create container requests that
-waitting to be processed.
-
-*Average latency of each pending operation in nanoseconds* - The average latency
-of the operation from the client point of view.
-Eg. `CreateContainerLatencyAvgTime` - This tells us the average latency of
-Create Container from the client point of view.
-
-*Number of bytes involved in a specific command* - This is an array that is
-maintained for all operations, but makes sense only for read and write
-operations.
-
-While it is possible to read the bytes in update container, it really makes
-no sense, since no data stream involved. Users are advised to use this
-metric only when it makes sense. Eg. `BytesReadChunk` -- Tells us how
-many bytes have been read from this data using Read Chunk operation.
-
-*Average Latency of each operation* - The average latency of the operation.
-Eg. `LatencyCreateContainerAvgTime` - This tells us the average latency of
-Create Container.
-
-*Quantiles for each of these operations* - The 50/75/90/95/99th percentile
-of these operations. Eg. `CreateContainerNanos60s50thPercentileLatency` --
-gives latency of the create container operations at the 50th percentile latency
-(1 minute granularity). We report 50th, 75th, 90th, 95th and 99th percentile
-for all RPCs.
-
-So this leads to the containers reporting these counters for each of these
-RPC operations.
-
-| Name | Description |
-|:---- |:---- |
-| `NumOps` | Total number of container operations |
-| `CreateContainer` | Create container operation |
-| `ReadContainer` | Read container operation |
-| `UpdateContainer` | Update container operations |
-| `DeleteContainer` | Delete container operations |
-| `ListContainer` | List container operations |
-| `PutKey` | Put key operations |
-| `GetKey` | Get key operations |
-| `DeleteKey` | Delete key operations |
-| `ListKey` | List key operations |
-| `ReadChunk` | Read chunk operations |
-| `DeleteChunk` | Delete chunk operations |
-| `WriteChunk` | Write chunk operations|
-| `ListChunk` | List chunk operations |
-| `CompactChunk` | Compact chunk operations |
-| `PutSmallFile` | Put small file operations |
-| `GetSmallFile` | Get small file operations |
-| `CloseContainer` | Close container operations |
-
-### Storage Container Manager Metrics
-
-The metrics for containers that managed by Storage Container Manager.
-
-Storage Container Manager (SCM) is a master service which keeps track of
-replicas of storage containers. It also manages all data nodes and their
-states, dealing with container reports and dispatching commands for execution.
-
-Following are the counters for containers:
-
-| Name | Description |
-|:---- |:---- |
-| `LastContainerReportSize` | Total size in bytes of all containers in latest container report that SCM received from datanode |
-| `LastContainerReportUsed` | Total number of bytes used by all containers in latest container report that SCM received from datanode |
-| `LastContainerReportKeyCount` | Total number of keys in all containers in latest container report that SCM received from datanode |
-| `LastContainerReportReadBytes` | Total number of bytes have been read from all containers in latest container report that SCM received from datanode |
-| `LastContainerReportWriteBytes` | Total number of bytes have been written into all containers in latest container report that SCM received from datanode |
-| `LastContainerReportReadCount` | Total number of times containers have been read from in latest container report that SCM received from datanode |
-| `LastContainerReportWriteCount` | Total number of times containers have been written to in latest container report that SCM received from datanode |
-| `ContainerReportSize` | Total size in bytes of all containers over whole cluster |
-| `ContainerReportUsed` | Total number of bytes used by all containers over whole cluster |
-| `ContainerReportKeyCount` | Total number of keys in all containers over whole cluster |
-| `ContainerReportReadBytes` | Total number of bytes have been read from all containers over whole cluster |
-| `ContainerReportWriteBytes` | Total number of bytes have been written into all containers over whole cluster |
-| `ContainerReportReadCount` | Total number of times containers have been read from over whole cluster |
-| `ContainerReportWriteCount` | Total number of times containers have been written to over whole cluster |
-
-### Key Space Metrics
-
-The metrics for various key space manager operations in HDFS Ozone.
-
-key space manager (KSM) is a service that similar to the Namenode in HDFS.
-In the current design of KSM, it maintains metadata of all volumes, buckets and keys.
-These metrics are only available when ozone is enabled.
-
-Following is the set of counters maintained for each key space operation.
-
-*Total number of operation* - We maintain an array which counts how
-many times a specific operation has been performed.
-Eg.`NumVolumeCreate` tells us how many times create volume has been
-invoked in KSM.
-
-*Total number of failed operation* - This type operation is opposite to the above
-operation.
-Eg.`NumVolumeCreateFails` tells us how many times create volume has been invoked
-failed in KSM.
-
-Following are the counters for each of key space operations.
-
-| Name | Description |
-|:---- |:---- |
-| `VolumeCreate` | Create volume operation |
-| `VolumeUpdates` | Update volume property operation |
-| `VolumeInfos` | Get volume information operation |
-| `VolumeCheckAccesses` | Check volume access operation |
-| `VolumeDeletes` | Delete volume operation |
-| `VolumeLists` | List volume operation |
-| `BucketCreates` | Create bucket operation |
-| `BucketInfos` | Get bucket information operation |
-| `BucketUpdates` | Update bucket property operation |
-| `BucketDeletes` | Delete bucket operation |
-| `BucketLists` | List bucket operation |
-| `KeyAllocate` | Allocate key operation |
-| `KeyLookup` | Look up key operation |
-| `KeyDeletes` | Delete key operation |
-| `KeyLists` | List key operation |
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneOverview.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneOverview.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneOverview.md
deleted file mode 100644
index 41d7dbd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneOverview.md
+++ /dev/null
@@ -1,88 +0,0 @@
-<!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-Ozone Overview
-==============
-
-
Ozone is an Object store for Apache Hadoop. It aims to scale to billions of
-keys. 
The following is a high-level overview of the core components of Ozone.


-
-![Ozone Architecture Overview](images/ozoneoverview.png) 


-
-The main elements of Ozone are
:
-
-### Clients
-Ozone ships with a set of ready-made clients. They are 
Ozone CLI and Freon.

-
-    * [Ozone CLI](./OzoneCommandShell.html) is the command line interface like 'hdfs' command.

-
-    * Freon is a  load generation tool for Ozone.

-
-### REST Handler
-Ozone provides both an RPC (Remote Procedure Call) as well as a  REST
-(Representational State Transfer) style interface. This allows clients to be
-written in many languages quickly. Ozone strives to maintain a similar
-interface between REST and RPC. The Rest handler offers the REST protocol
-services of Ozone.
-
-For most purposes, a client can make one line change to switch from REST to
-RPC or vice versa.  

-
-### Ozone File System
-Ozone file system (TODO: Add documentation) is a Hadoop compatible file system.
-This is the important user-visible component of ozone.
-This allows Hadoop services and applications like Hive/Spark to run against
-Ozone without any change.
-
-### Ozone Client
-This is like DFSClient in HDFS. This acts as the standard client to talk to
-Ozone. All other components that we have discussed so far rely on Ozone client
-(TODO: Add Ozone client documentation).

-
-### Key Space Manager

-Key Space Manager(KSM) takes care of the Ozone's namespace.
-All ozone entities like volumes, buckets and keys are managed by KSM
-(TODO: Add KSM documentation). In Short, KSM is the metadata manager for Ozone.
-KSM talks to blockManager(SCM) to get blocks and passes it on to the Ozone
-client.  Ozone client writes data to these blocks.
-KSM will eventually be replicated via Apache Ratis for High Availability.

-
-### Storage Container Manager
-Storage Container Manager (SCM) is the block and cluster manager for Ozone.
-SCM along with data nodes offer a service called 'containers'.
-A container is a group unrelated of blocks that are managed together
-as a single entity.
-
-SCM offers the following abstractions.


-
-![SCM Abstractions](images/scmservices.png)
-#### Blocks
-Blocks are like blocks in HDFS. They are replicated store of data.
-
-#### Containers
-A collection of blocks replicated and managed together.
-
-#### Pipelines
-SCM allows each container to choose its method of replication.
-For example, a container might decide that it needs only one copy of a  block
-and might choose a stand-alone pipeline. Another container might want to have
-a very high level of reliability and pick a RATIS based pipeline. In other
-words, SCM allows different kinds of replication strategies to co-exist.
-
-#### Pools
-A group of data nodes is called a pool. For scaling purposes,
-we define a pool as a set of machines. This makes management of datanodes
-easier.
-
-#### Nodes
-The data node where data is stored.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneRest.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneRest.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneRest.md
deleted file mode 100644
index 0294a53..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneRest.md
+++ /dev/null
@@ -1,549 +0,0 @@
-<!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-
-Ozone REST API's.
-===================
-
-<!-- MACRO{toc|fromDepth=0|toDepth=1} -->
-
-Overview
---------
-
-The Ozone REST API's allows user to access ozone via  REST protocol.
-
-Authentication and Authorization
---------------------
-
-For time being, The default authentication mode of REST API is insecure access
-mode, which is *Simple* mode. Under this mode, ozone server trusts the user
-name specified by client and it does not perform any authentication.
-
-User name can be specified in HTTP header by
-
-* `x-ozone-user: {USER_NAME}`
-
-for example if add following header *x-ozone-user: bilbo* in the HTTP request,
-then operation will be executed as *bilbo* user.
-In *Simple* mode, there is no real authorization either. Client can be
-authorized to obtain administrator privilege by using HTTP header
-
-* `Authorization: {AUTH_METHOD} {SIGNATURE}`
-
-for example set following header *Authorization: OZONE root* in the HTTP request,
-then ozone will authorize the client with administrator privilege.
-
-Common REST Headers
---------------------
-
-The following HTTP headers must be set for each REST call.
-
-| Property | Description |
-|:---- |:----
-| Authorization | The authorization field determines which authentication method is used by ozone. Currently only *simple* mode is supported, the corresponding value is *OZONE*. Optionally an user name can be set as *OZONE {USER_NAME}* to authorize as a particular user. |
-| Date | Standard HTTP header that represents dates. The format is - day of the week, month, day, year and time (military time format) in GMT. Any other time zone will be rejected by ozone server. Eg. *Date : Mon, Apr 4, 2016 06:22:00 GMT*. This field is required. |
-| x-ozone-version | A required HTTP header to indicate which version of API this call will be communicating to. E.g *x-ozone-version: v1*. Currently ozone only publishes v1 version API. |
-
-Common Reply Headers
---------------------
-
-The common reply headers are part of all Ozone server replies.
-
-| Property | Description |
-|:---- |:----
-| Date | This is the HTTP date header and it is set to server’s local time expressed in GMT. |
-| x-ozone-request-id | This is a UUID string that represents an unique request ID. This ID is used to track the request through the ozone system and is useful for debugging purposes. |
-| x-ozone-server-name | Fully qualified domain name of the sever which handled the request. |
-
-Volume APIs
---------------------
-
-### Create a Volume
-
-This API allows admins to create a new storage volume.
-
-Schema:
-
-- `POST /{volume}?quota=<VOLUME_QUOTA>`
-
-Query Parameter:
-
-| Query Parameter | Value | Description |
-|:---- |:---- |:----
-| quota | long<BYTES \| MB \| GB \| TB> | Optional. Quota size in BYTEs, MBs, GBs or TBs |
-
-Sample HTTP POST request:
-
-    curl -i -X POST -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root" "http://localhost:9864/volume-to-create"
-
-this request creates a volume as user *bilbo*, the authorization field is set to *OZONE root* because this call requires administration privilege. The client receives a response with zero content length.
-
-    HTTP/1.1 201 Created
-    x-ozone-server-name: localhost
-    x-ozone-request-id: 2173deb5-bbb7-4f0a-8236-f354784e3bae
-    Date: Tue, 27 Jun 2017 07:42:04 GMT
-    Content-Type: application/octet-stream
-    Content-Length: 0
-    Connection: keep-alive
-
-### Update Volume
-
-This API allows administrators to update volume info such as ownership and quota. This API requires administration privilege.
-
-Schema:
-
-- `PUT /{volume}?quota=<VOLUME_QUOTA>`
-
-Query Parameter:
-
-| Query Parameter | Value | Description |
-|:---- |:---- |:----
-| quota | long<BYTES \| MB \| GB \| TB>  \| remove | Optional. Quota size in BYTEs, MBs, GBs or TBs. Or use string value *remove* to remove an existing quota for a volume. |
-
-Sample HTTP PUT request:
-
-    curl -X PUT -H "Authorization:OZONE root" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "x-ozone-version: v1" -H "x-ozone-user: john"  http://localhost:9864/volume-to-update
-
-this request modifies the owner of */volume-to-update* to *john*.
-
-### Delete Volume
-
-This API allows user to delete a volume owned by themselves if the volume is not empty. Administrators can delete volumes owned by any user.
-
-Schema:
-
-- `DELETE /{volume}`
-
-Sample HTTP DELETE request:
-
-    curl -i -X DELETE -H "Authorization:OZONE root" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "x-ozone-version: v1" -H "x-ozone-user: bilbo"  http://localhost:9864/volume-to-delete
-
-this request deletes an empty volume */volume-to-delete*. The client receives a zero length content.
-
-    HTTP/1.1 200 OK
-    x-ozone-server-name: localhost
-    x-ozone-request-id: 6af14c64-e3a9-40fe-9634-df60b7cbbc6a
-    Date: Tue, 27 Jun 2017 08:49:52 GMT
-    Content-Type: application/octet-stream
-    Content-Length: 0
-    Connection: keep-alive
-
-### Info Volume
-
-This API allows user to read the info of a volume owned by themselves. Administrators can read volume info owned by any user.
-
-Schema:
-
-- `GET /{volume}?info=volume`
-
-Query Parameter:
-
-| Query Parameter | Value | Description |
-|:---- |:---- |:----
-| info | "volume" | Required and enforced with this value. |
-
-Sample HTTP GET request:
-
-    curl -i -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" "http://localhost:9864/volume-of-bilbo?info=volume"
-
-this request gets the info of volume */volume-of-bilbo*, the client receives a response with a JSON object of volume info.
-
-    HTTP/1.1 200 OK
-    x-ozone-server-name: localhost
-    x-ozone-request-id: a2224806-beaf-42dd-a68e-533cd7508f74
-    Date: Tue, 27 Jun 2017 07:55:35 GMT
-    Content-Type: application/octet-stream
-    Content-Length: 171
-    Connection: keep-alive
-
-    {
-      "owner" : { "name" : "bilbo" },
-      "quota" : { "unit" : "TB", "size" : 1048576 },
-      "volumeName" : "volume-of-bilbo",
-      "createdOn" : "Tue, 27 Jun 2017 07:42:04 GMT",
-      "createdBy" : "root"
-    }
-
-### List Volumes
-
-This API allows user to list all volumes owned by themselves. Administrators can list all volumes owned by any user.
-
-Schema:
-
-- `GET /?prefix=<PREFIX>&max-keys=<MAX_RESULT_SIZE>&prev-key=<PREVIOUS_VOLUME_KEY>`
-
-Query Parameter:
-
-| Query Parameter | Value | Description |
-|:---- |:---- |:----
-| prefix | string | Optional. Only volumes with this prefix are included in the result. |
-| max-keys | int | Optional. Maximum number of volumes included in the result. Default is 1024 if not specified. |
-| prev-key | string | Optional. Volume name from where listing should start, this key is excluded in the result. It must be a valid volume name. |
-| root-scan | bool | Optional. List all volumes in the cluster if this is set to true. Default false. |
-
-Sample HTTP GET request:
-
-    curl -i -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" "http://localhost:9864/?max-keys=100&prefix=Jan"
-
-this request gets all volumes owned by *bilbo* and each volume's name contains prefix *Jan*, the result at most contains *100* entries. The client receives a list of SON objects, each of them describes the info of a volume.
-
-    HTTP/1.1 200 OK
-    x-ozone-server-name: localhost
-    x-ozone-request-id: 7fa0dce1-a8bd-4387-bc3c-1dac4b710bb1
-    Date: Tue, 27 Jun 2017 08:07:04 GMT
-    Content-Type: application/octet-stream
-    Content-Length: 602
-    Connection: keep-alive
-
-    {
-      "volumes" : [
-        {
-          "owner" : { "name" : "bilbo"},
-          "quota" : { "unit" : "TB", "size" : 2 },
-          "volumeName" : "Jan-vol1",
-          "createdOn" : "Tue, 27 Jun 2017 07:42:04 GMT",
-          "createdBy" : root
-      },
-      ...
-      ]
-    }
-
-Bucket APIs
---------------------
-
-### Create Bucket
-
-This API allows an user to create a bucket in a volume.
-
-Schema:
-
-- `POST /{volume}/{bucket}`
-
-Additional HTTP Headers:
-
-| HTTP Header | Value | Description |
-|:---- |:---- |:----
-| x-ozone-acl | ozone ACLs | Optional. Ozone acls. |
-| x-ozone-storage-class | <DEFAULT \| ARCHIVE \| DISK \| RAM_DISK \| SSD > | Optional. Storage type for a volume. |
-| x-ozone-bucket-versioning | enabled/disabled | Optional. Do enable bucket versioning or not. |
-
-Sample HTTP POST request:
-
-    curl -i -X POST -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" http://localhost:9864/volume-of-bilbo/bucket-0
-
-this request creates a bucket *bucket-0* under volume *volume-of-bilbo*.
-
-    HTTP/1.1 201 Created
-    x-ozone-server-name: localhost
-    x-ozone-request-id: 49acfeec-4c85-470a-872b-2eaebd8d751e
-    Date: Tue, 27 Jun 2017 08:55:25 GMT
-    Content-Type: application/octet-stream
-    Content-Length: 0
-    Connection: keep-alive
-
-### Update Bucket
-
-Updates bucket meta-data, like ACLs.
-
-Schema:
-
-- `PUT /{volume}/{bucket}`
-
-Additional HTTP Headers:
-
-| HTTP Header | Value | Description |
-|:---- |:---- |:----
-| x-ozone-acl | ozone ACLs | Optional. Ozone acls. |
-| x-ozone-bucket-versioning | enabled/disabled | Optional. Do enable bucket versioning or not. |
-
-Sample HTTP PUT request:
-
-    curl -i -X PUT -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" -H "x-ozone-acl: ADD user:peregrin:rw" http://localhost:9864/volume-of-bilbo/bucket-to-update
-
-this request adds an ACL policy specified by HTTP header *x-ozone-acl* to bucket */volume-of-bilbo/bucket-to-update*, the ACL field *ADD user:peregrin:rw* gives add additional read/write permission to user *peregrin* to this bucket.
-
-    HTTP/1.1 200 OK
-    x-ozone-server-name: localhost
-    x-ozone-request-id: b061a295-5faf-4b98-94b9-8b3e87c8eb5e
-    Date: Tue, 27 Jun 2017 09:02:37 GMT
-    Content-Type: application/octet-stream
-    Content-Length: 0
-    Connection: keep-alive
-
-### Delete Bucket
-
-Deletes a bucket if it is empty. An user can only delete bucket owned by themselves, and administrators can delete buckets owned by any user, as long as it is empty.
-
-Schema:
-
-- `DELETE /{volume}/{bucket}`
-
-Sample HTTP DELETE request:
-
-    curl -i -X DELETE -H "Authorization:OZONE root" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "x-ozone-version: v1" -H "x-ozone-user:bilbo" "http://localhost:9864/volume-of-bilbo/bucket-0"
-
-this request deletes bucket */volume-of-bilbo/bucket-0*. The client receives a zero length content response.
-
-    HTTP/1.1 200 OK
-    x-ozone-server-name: localhost
-    x-ozone-request-id: f57acd7a-2116-4c2f-aa2f-5a483db81c9c
-    Date: Tue, 27 Jun 2017 09:16:52 GMT
-    Content-Type: application/octet-stream
-    Content-Length: 0
-    Connection: keep-alive
-
-
-### Info Bucket
-
-This API returns information about a given bucket.
-
-Schema:
-
-- `GET /{volume}/{bucket}?info=bucket`
-
-Query Parameters:
-
-| Query Parameter | Value | Description |
-|:---- |:---- |:----
-| info | "bucket" | Required and enforced with this value. |
-
-Sample HTTP GET request:
-
-    curl -i -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" "http://localhost:9864/volume-of-bilbo/bucket-0?info=bucket"
-
-this request gets the info of bucket */volume-of-bilbo/bucket-0*. The client receives a response of JSON object contains bucket info.
-
-    HTTP/1.1 200 OK
-    x-ozone-server-name: localhost
-    x-ozone-request-id: f125485b-8cae-4c7f-a2d6-5b1fefd6f193
-    Date: Tue, 27 Jun 2017 09:08:31 GMT
-    Content-Type: application/json
-    Content-Length: 138
-    Connection: keep-alive
-
-    {
-      "volumeName" : "volume-of-bilbo",
-      "bucketName" : "bucket-0",
-      "createdOn" : "Tue, 27 Jun 2017 08:55:25 GMT",
-      "acls" : [ ],
-      "versioning" : "DISABLED",
-      "storageType" : "DISK"
-    }
-
-### List Buckets
-
-List buckets in a given volume.
-
-Schema:
-
-- `GET /{volume}?prefix=<PREFIX>&max-keys=<MAX_RESULT_SIZE>&prev-key=<PREVIOUS_BUCKET_KEY>`
-
-Query Parameters:
-
-| Query Parameter | Value | Description |
-|:---- |:---- |:----
-| prefix | string | Optional. Only buckets with this prefix are included in the result. |
-| max-keys | int | Optional. Maximum number of buckets included in the result. Default is 1024 if not specified. |
-| prev-key | string | Optional. Bucket name from where listing should start, this key is excluded in the result. It must be a valid bucket name. |
-
-Sample HTTP GET request:
-
-    curl -i -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" "http://localhost:9864/volume-of-bilbo?max-keys=10"
-
-this request lists all the buckets under volume *volume-of-bilbo*, and the result at most contains 10 entries. The client receives response of a array of JSON objects, each of them represents for a bucket info.
-
-    HTTP/1.1 200 OK
-    x-ozone-server-name: localhost
-    x-ozone-request-id: e048c3d5-169c-470f-9903-632d9f9e32d5
-    Date: Tue, 27 Jun 2017 09:12:18 GMT
-    Content-Type: application/octet-stream
-    Content-Length: 207
-    Connection: keep-alive
-
-    {
-      "buckets" : [ {
-        "volumeName" : "volume-of-bilbo",
-        "bucketName" : "bucket-0",
-        "createdOn" : "Tue, 27 Jun 2017 08:55:25 GMT",
-        "acls" : [ ],
-        "versioning" : null,
-        "storageType" : "DISK",
-        "bytesUsed" : 0,
-        "keyCount" : 0
-        },
-        ...
-      ]
-    }
-
-Key APIs
-------------------
-
-### Put Key
-
-This API allows user to create or overwrite keys inside of a bucket.
-
-Schema:
-
-- `PUT /{volume}/{bucket}/{key}`
-
-Additional HTTP headers:
-
-| HTTP Header | Value | Description |
-|:---- |:---- |:----
-| Content-MD5 | MD5 digest | Standard HTTP header, file hash. |
-
-Sample PUT HTTP request:
-
-    curl -X PUT -T /path/to/localfile -H "Authorization:OZONE" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "x-ozone-version: v1" -H "x-ozone-user:bilbo" "http://localhost:9864/volume-of-bilbo/bucket-0/file-0"
-
-this request uploads a local file */path/to/localfile* specified by option *-T* to ozone as user *bilbo*, mapped to ozone key */volume-of-bilbo/bucket-0/file-0*. The client receives a zero length content response.
-
-### Get Key
-
-This API allows user to get or download a key from an ozone bucket.
-
-Schema:
-
-- `GET /{volume}/{bucket}/{key}`
-
-Sample HTTP GET request:
-
-    curl -i -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" "http://localhost:9864/volume-of-bilbo/bucket-0/file-0"
-
-this request reads the content of key */volume-of-bilbo/bucket-0/file-0*. If the content of the file is plain text, it can be directly dumped onto stdout.
-
-    HTTP/1.1 200 OK
-    Content-Type: application/octet-stream
-    x-ozone-server-name: localhost
-    x-ozone-request-id: 1bcd7de7-d8e3-46bb-afee-bdc933d383b8
-    Date: Tue, 27 Jun 2017 09:35:29 GMT
-    Content-Length: 6
-    Connection: keep-alive
-
-    Hello Ozone!
-
-if the file is not plain text, specify *-O* option in curl command and the file *file-0* will be downloaded into current working directory, file name will be same as the key. A sample request like following:
-
-    curl -O -i -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" "http://localhost:9864/volume-of-bilbo/bucket-0/file-1"
-
-response looks like following:
-
-    % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
-                                 Dload  Upload   Total   Spent    Left  Speed
-    100 6148k  100 6148k    0     0  24.0M      0 --:--:-- --:--:-- --:--:-- 24.1M
-
-### Delete Key
-
-This API allows user to delete a key from a bucket.
-
-Schema:
-
-- `DELETE /{volume}/{bucket}/{key}`
-
-Sample HTTP DELETE request:
-
-    curl -i -X DELETE -H "Authorization:OZONE root" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "x-ozone-version: v1" -H "x-ozone-user:bilbo" "http://localhost:9864/volume-of-bilbo/bucket-0/file-0"
-
-this request deletes key */volume-of-bilbo/bucket-0/file-0*. The client receives a zero length content result:
-
-    HTTP/1.1 200 OK
-    x-ozone-server-name: localhost
-    x-ozone-request-id: f8c4a373-dd5f-4e3a-b6c4-ddf7e191fe91
-    Date: Tue, 27 Jun 2017 14:19:48 GMT
-    Content-Type: application/octet-stream
-    Content-Length: 0
-    Connection: keep-alive
-
-### Info Key
-
-This API returns information about a given key.
-
-Schema:
-
-- `GET /{volume}/{bucket}/{key}?info=key`
-
-Query Parameter:
-
-| Query Parameter | Value | Description |
-|:---- |:---- |:----
-| info | String, "key" | Required and enforced with this value. |
-
-Sample HTTP DELETE request:
-
-    curl -i -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" "http://localhost:9864/volume-of-bilbo/buket-0/file-0?info=key"
-
-this request returns information of the key */volume-of-bilbo/bucket-0/file-0*. The client receives a JSON object listed attributes of the key.
-
-    HTTP/1.1 200 OK
-    x-ozone-server-name: localhost
-    x-ozone-request-id: c674343c-a0f2-49e4-bbd6-daa73e7dc131
-    Date: Mon, 03 Jul 2017 14:28:45 GMT
-    Content-Type: application/octet-stream
-    Content-Length: 73
-    Connection: keep-alive
-
-    {
-      "version" : 0,
-      "md5hash" : null,
-      "createdOn" : "Mon, 26 Jun 2017 04:23:30 GMT",
-      "modifiedOn" : "Mon, 26 Jun 2017 04:23:30 GMT",
-      "size" : 0,
-      "keyName" : "file-0"
-    }
-
-### List Keys
-
-This API allows user to list keys in a bucket.
-
-Schema:
-
-- `GET /{volume}/{bucket}?prefix=<PREFIX>&max-keys=<MAX_RESULT_SIZE>&prev-key=<PREVIOUS_KEY>`
-
-Query Parameters:
-
-| Query Parameter | Value | Description |
-|:---- |:---- |:----
-| prefix | string | Optional. Only keys with this prefix are included in the result. |
-| max-keys | int | Optional. Maximum number of keys included in the result. Default is 1024 if not specified. |
-| prev-key | string | Optional. Key name from where listing should start, this key is excluded in the result. It must be a valid key name. |
-
-Sample HTTP GET request:
-
-    curl -i -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" "http:/localhost:9864/volume-of-bilbo/bucket-0/?max-keys=100&prefix=file"
-
-this request list keys under bucket */volume-of-bilbo/bucket-0*, the listing result is filtered by prefix *file*. The client receives an array of JSON objects, each of them represents the info of a matched key.
-
-    HTTP/1.1 200 OK
-    x-ozone-server-name: localhost
-    x-ozone-request-id: 7f9fc970-9904-4c56-b671-83a086c6f555
-    Date: Tue, 27 Jun 2017 09:48:59 GMT
-    Content-Type: application/json
-    Content-Length: 209
-    Connection: keep-alive
-
-    {
-      "name" : null,
-      "prefix" : file,
-      "maxKeys" : 0,
-      "truncated" : false,
-      "keyList" : [ {
-          "version" : 0,
-          "md5hash" : null,
-          "createdOn" : "Mon, 26 Jun 2017 04:23:30 GMT",
-          "modifiedOn" : "Mon, 26 Jun 2017 04:23:30 GMT",
-          "size" : 0,
-          "keyName" : "file-0"
-          },
-          ...
-       ]
-    }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[13/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
deleted file mode 100644
index 158b5bb..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
+++ /dev/null
@@ -1,415 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell;
-
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.bucket.UpdateBucketHandler;
-import org.apache.hadoop.ozone.web.ozShell.keys.DeleteKeyHandler;
-import org.apache.hadoop.ozone.web.ozShell.keys.GetKeyHandler;
-import org.apache.hadoop.ozone.web.ozShell.keys.InfoKeyHandler;
-import org.apache.hadoop.ozone.web.ozShell.keys.ListKeyHandler;
-import org.apache.hadoop.ozone.web.ozShell.keys.PutKeyHandler;
-import org.apache.hadoop.ozone.web.ozShell.volume.CreateVolumeHandler;
-import org.apache.hadoop.ozone.web.ozShell.volume.DeleteVolumeHandler;
-import org.apache.hadoop.ozone.web.ozShell.volume.InfoVolumeHandler;
-import org.apache.hadoop.ozone.web.ozShell.volume.ListVolumeHandler;
-import org.apache.hadoop.ozone.web.ozShell.volume.UpdateVolumeHandler;
-import org.apache.hadoop.ozone.web.ozShell.bucket.CreateBucketHandler;
-import org.apache.hadoop.ozone.web.ozShell.bucket.DeleteBucketHandler;
-import org.apache.hadoop.ozone.web.ozShell.bucket.InfoBucketHandler;
-import org.apache.hadoop.ozone.web.ozShell.bucket.ListBucketHandler;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-
-/**
- * Ozone user interface commands.
- *
- * This class uses dispatch method to make calls
- * to appropriate handlers that execute the ozone functions.
- */
-public class Shell extends Configured implements Tool {
-  private static final Logger LOG = LoggerFactory.getLogger(Shell.class);
-
-  // General options
-  public static final int DEFAULT_OZONE_PORT = 50070;
-  public static final String VERBOSE = "v";
-
-  // volume related command line arguments
-  public static final String RUNAS = "root";
-  public static final String USER = "user";
-  public static final String OWNER = "owner";
-  public static final String QUOTA = "quota";
-  public static final String CREATE_VOLUME = "createVolume";
-  public static final String UPDATE_VOLUME = "updateVolume";
-  public static final String DELETE_VOLUME = "deleteVolume";
-  public static final String LIST_VOLUME = "listVolume";
-  public static final String INFO_VOLUME = "infoVolume";
-
-  // bucket related command line arguments
-  public static final String CREATE_BUCKET = "createBucket";
-  public static final String UPDATE_BUCKET = "updateBucket";
-  public static final String DELETE_BUCKET = "deleteBucket";
-  public static final String LIST_BUCKET = "listBucket";
-  public static final String INFO_BUCKET = "infoBucket";
-  public static final String ADD_ACLS = "addAcl";
-  public static final String REMOVE_ACLS = "removeAcl";
-  // TODO : Support versioning and StorageType for buckets
-
-  //Object related command line arguments
-  public static final String PUT_KEY = "putKey";
-  public static final String GET_KEY = "getKey";
-  public static final String INFO_KEY = "infoKey";
-  public static final String DELETE_KEY = "deleteKey";
-  public static final String LIST_KEY = "listKey";
-  public static final String FILE = "file";
-
-  // Listing related command line arguments
-  public static final String LIST_LENGTH = "length";
-  public static final String START = "start";
-  public static final String PREFIX = "prefix";
-
-  /**
-   * Main for the ozShell Command handling.
-   *
-   * @param argv - System Args Strings[]
-   * @throws Exception
-   */
-  public static void main(String[] argv) throws Exception {
-    Shell shell = new Shell();
-    Configuration conf = new OzoneConfiguration();
-    conf.setQuietMode(false);
-    shell.setConf(conf);
-    int res = 0;
-    try {
-      res = ToolRunner.run(shell, argv);
-    } catch (Exception ex) {
-      System.err.println("ERROR: " + ex.getMessage());
-      System.exit(1);
-    }
-    System.exit(res);
-  }
-
-  /**
-   * Execute the command with the given arguments.
-   *
-   * @param args command specific arguments.
-   *
-   * @return exit code.
-   *
-   * @throws Exception
-   */
-  @Override
-  public int run(String[] args) throws Exception {
-    Options opts = getOpts();
-    CommandLine cmd = parseArgs(args, opts);
-    return dispatch(cmd, opts);
-  }
-
-  /**
-   * returns the Command Line Options.
-   *
-   * @return Options
-   */
-  private Options getOpts() {
-    Options opts = new Options();
-    addVolumeCommands(opts);
-    addBucketCommands(opts);
-    addKeyCommands(opts);
-    addListingCommands(opts);
-    return opts;
-  }
-
-  /**
-   * This function parses all command line arguments
-   * and returns the appropriate values.
-   *
-   * @param argv - Argv from main
-   *
-   * @return CommandLine
-   */
-  private CommandLine parseArgs(String[] argv, Options opts)
-      throws org.apache.commons.cli.ParseException {
-    try {
-      BasicParser parser = new BasicParser();
-      return parser.parse(opts, argv);
-    } catch (ParseException ex) {
-      System.out.printf(ex.getMessage());
-    }
-
-    return null;
-  }
-
-
-  /**
-   * All volume related commands are added in this function for the command
-   * parser.
-   *
-   * @param options - Command Options class.
-   */
-  private void addVolumeCommands(Options options) {
-    Option verbose = new Option(VERBOSE, false, "verbose information output.");
-    options.addOption(verbose);
-
-    Option runas = new Option(RUNAS, false, "Run the command as \"hdfs\" user");
-    options.addOption(runas);
-
-    Option userName = new Option(USER, true,
-                                 "Name of the user in volume management " +
-                                     "functions");
-    options.addOption(userName);
-
-    Option quota = new Option(QUOTA, true, "Quota for the volume. E.g. 10TB");
-    options.addOption(quota);
-
-
-    Option createVolume = new Option(CREATE_VOLUME, true, "creates a volume" +
-        "for the specified user.\n \t For example : hdfs oz  -createVolume " +
-        "<volumeURI> -root -user <userName>\n");
-    options.addOption(createVolume);
-
-    Option deleteVolume = new Option(DELETE_VOLUME, true, "deletes a volume" +
-        "if it is empty.\n \t For example : hdfs oz -deleteVolume <volumeURI>" +
-        " -root \n");
-    options.addOption(deleteVolume);
-
-    Option listVolume =
-        new Option(LIST_VOLUME, true, "List the volumes of a given user.\n" +
-            "For example : hdfs oz -listVolume <ozoneURI>" +
-            "-user <username> -root or hdfs oz " +
-            "-listVolume");
-    options.addOption(listVolume);
-
-    Option updateVolume =
-        new Option(UPDATE_VOLUME, true, "updates an existing volume.\n" +
-            "\t For example : hdfs oz " +
-            "-updateVolume <volumeURI> -quota " +
-            "100TB\n");
-    options.addOption(updateVolume);
-
-    Option infoVolume = new Option(INFO_VOLUME, true,
-                                   "returns information about a specific " +
-                                       "volume.");
-    options.addOption(infoVolume);
-  }
-
-  /**
-   * All bucket related commands for ozone.
-   *
-   * @param opts - Options
-   */
-  private void addBucketCommands(Options opts) {
-    Option createBucket = new Option(CREATE_BUCKET, true,
-        "creates a bucket in a given volume." +
-        "For example: hdfs oz -createBucket <bucketURI>");
-    opts.addOption(createBucket);
-
-    Option infoBucket =
-        new Option(INFO_BUCKET, true, "returns information about a bucket.");
-    opts.addOption(infoBucket);
-
-    Option deleteBucket =
-        new Option(DELETE_BUCKET, true, "deletes an empty bucket.");
-    opts.addOption(deleteBucket);
-
-    Option listBucket =
-        new Option(LIST_BUCKET, true, "lists the buckets in a volume.");
-    opts.addOption(listBucket);
-
-    Option updateBucket =
-        new Option(UPDATE_BUCKET, true, "allows changing bucket attributes.\n" +
-            " For example: hdfs oz -updateBucket <bucketURI> " +
-            "-addAcl user:frodo:rw");
-    opts.addOption(updateBucket);
-
-    Option addAcl =
-        new Option(ADD_ACLS, true, "allows user to add acls to a bucket.");
-    opts.addOption(addAcl);
-
-    Option removeAcl =
-        new Option(REMOVE_ACLS, true, "allows user to remove acls from a " +
-            "bucket.");
-    opts.addOption(removeAcl);
-  }
-
-  /**
-   * All key commands.
-   *
-   * @param opts - options
-   */
-  private void addKeyCommands(Options opts) {
-    Option putKey =
-        new Option(PUT_KEY, true, "creates or overwrites an existing key");
-    opts.addOption(putKey);
-
-    Option deleteKey =
-        new Option(DELETE_KEY, true, "deletes an existing key");
-    opts.addOption(deleteKey);
-
-    Option infoKey =
-        new Option(INFO_KEY, true, "returns information about an existing key");
-    opts.addOption(infoKey);
-
-    Option listKey =
-        new Option(LIST_KEY, true, "list all keys in a given bucket");
-    opts.addOption(listKey);
-
-    Option getKey =
-        new Option(GET_KEY, true, "Gets a specific key from ozone server.");
-    opts.addOption(getKey);
-
-    Option fileArgument =
-        new Option(FILE, true, "Data file path");
-    opts.addOption(fileArgument);
-
-  }
-
-  /**
-   * Sub commands for list command.
-   * @param opts
-   */
-  private void addListingCommands(Options opts) {
-    Option maxKeys = new Option(LIST_LENGTH, true,
-        "Specify the max length of listing result.");
-    opts.addOption(maxKeys);
-
-    Option prevKey = new Option(START, true,
-        "Specify the start key where to start listing from.");
-    opts.addOption(prevKey);
-
-    Option prefix = new Option(PREFIX, true,
-        "Specify the prefix to filter the listing result.");
-    opts.addOption(prefix);
-  }
-
-  /**
-   * Dispatches calls to the right command Handler classes.
-   *
-   * @param cmd - CommandLine
-   *
-   * @throws IOException
-   * @throws OzoneException
-   * @throws URISyntaxException
-   */
-  private int dispatch(CommandLine cmd, Options opts)
-      throws IOException, OzoneException, URISyntaxException {
-    Handler handler = null;
-    final int eightyColumn = 80;
-
-    try {
-
-      // volume functions
-      if (cmd.hasOption(Shell.CREATE_VOLUME)) {
-        handler = new CreateVolumeHandler();
-      }
-
-      if (cmd.hasOption(Shell.DELETE_VOLUME)) {
-        handler = new DeleteVolumeHandler();
-      }
-
-      if (cmd.hasOption(Shell.LIST_VOLUME)) {
-        handler = new ListVolumeHandler();
-      }
-
-      if (cmd.hasOption(Shell.UPDATE_VOLUME)) {
-        handler = new UpdateVolumeHandler();
-      }
-
-      if (cmd.hasOption(Shell.INFO_VOLUME)) {
-        handler = new InfoVolumeHandler();
-      }
-
-      // bucket functions
-      if (cmd.hasOption(Shell.CREATE_BUCKET)) {
-        handler = new CreateBucketHandler();
-      }
-
-      if (cmd.hasOption(Shell.DELETE_BUCKET)) {
-        handler = new DeleteBucketHandler();
-      }
-
-      if (cmd.hasOption(Shell.INFO_BUCKET)) {
-        handler = new InfoBucketHandler();
-      }
-
-      if (cmd.hasOption(Shell.LIST_BUCKET)) {
-        handler = new ListBucketHandler();
-      }
-
-      if(cmd.hasOption(Shell.UPDATE_BUCKET)){
-        handler = new UpdateBucketHandler();
-      }
-
-      //Key Functions
-
-      if(cmd.hasOption(Shell.PUT_KEY)) {
-        handler = new PutKeyHandler();
-      }
-
-      if(cmd.hasOption(Shell.DELETE_KEY)) {
-        handler = new DeleteKeyHandler();
-      }
-
-      if(cmd.hasOption(Shell.INFO_KEY)) {
-        handler = new InfoKeyHandler();
-      }
-
-      if(cmd.hasOption(Shell.LIST_KEY)) {
-        handler = new ListKeyHandler();
-      }
-
-      if(cmd.hasOption(Shell.GET_KEY)) {
-        handler = new GetKeyHandler();
-      }
-
-      if (handler != null) {
-        handler.execute(cmd);
-        return 0;
-      } else {
-        HelpFormatter helpFormatter = new HelpFormatter();
-        helpFormatter.printHelp(eightyColumn, "hdfs oz -command uri [args]",
-            "Ozone Commands",
-            opts, "Please correct your command and try again.");
-        return 1;
-      }
-    } catch (IOException | URISyntaxException ex) {
-      System.err.printf("Command Failed : %s%n", ex.getMessage());
-    } catch (OzoneException ex) {
-      System.err.printf("Command Failed : %s%n", ex.toJsonString());
-      LOG.debug("Command Failed.", ex);
-    } catch (IllegalArgumentException ex) {
-      System.err.printf("Illegal argument: %s%n", ex.getMessage());
-    }
-    return 1;
-  }
-}
-

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
deleted file mode 100644
index d1c46b5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneBucket;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-/**
- * create bucket handler.
- */
-public class CreateBucketHandler extends Handler {
-
-  private String volumeName;
-  private String bucketName;
-  private String rootName;
-
-  /**
-   * Executes create bucket.
-   *
-   * @param cmd - CommandLine
-   *
-   * @throws IOException
-   * @throws OzoneException
-   * @throws URISyntaxException
-   */
-  @Override
-  protected void execute(CommandLine cmd)
-      throws IOException, OzoneException, URISyntaxException {
-    if (!cmd.hasOption(Shell.CREATE_BUCKET)) {
-      throw new OzoneRestClientException(
-          "Incorrect call : createBucket is missing");
-    }
-
-    String ozoneURIString = cmd.getOptionValue(Shell.CREATE_BUCKET);
-    URI ozoneURI = verifyURI(ozoneURIString);
-    Path path = Paths.get(ozoneURI.getPath());
-    if (path.getNameCount() < 2) {
-      throw new OzoneRestClientException(
-          "volume and bucket name required in createBucket");
-    }
-
-    volumeName = path.getName(0).toString();
-    bucketName = path.getName(1).toString();
-
-    if (cmd.hasOption(Shell.VERBOSE)) {
-      System.out.printf("Volume Name : %s%n", volumeName);
-      System.out.printf("Bucket Name : %s%n", bucketName);
-    }
-
-    if (cmd.hasOption(Shell.RUNAS)) {
-      rootName = "hdfs";
-    } else {
-      rootName = System.getProperty("user.name");
-    }
-
-
-    client.setEndPointURI(ozoneURI);
-    client.setUserAuth(rootName);
-
-
-    OzoneVolume vol = client.getVolume(volumeName);
-    OzoneBucket bucket = vol.createBucket(bucketName);
-
-    if (cmd.hasOption(Shell.VERBOSE)) {
-      System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
-          bucket.getBucketInfo().toJsonString()));
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/DeleteBucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/DeleteBucketHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/DeleteBucketHandler.java
deleted file mode 100644
index 181f6cc..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/DeleteBucketHandler.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-/**
- * Delete bucket Handler.
- */
-public class DeleteBucketHandler extends Handler {
-  private String volumeName;
-  private String bucketName;
-  private String rootName;
-
-  /**
-   * Executes the Client Calls.
-   *
-   * @param cmd - CommandLine
-   *
-   * @throws IOException
-   * @throws OzoneException
-   * @throws URISyntaxException
-   */
-  @Override
-  protected void execute(CommandLine cmd)
-      throws IOException, OzoneException, URISyntaxException {
-    if (!cmd.hasOption(Shell.DELETE_BUCKET)) {
-      throw new OzoneRestClientException(
-          "Incorrect call : deleteBucket is missing");
-    }
-
-    String ozoneURIString = cmd.getOptionValue(Shell.DELETE_BUCKET);
-    URI ozoneURI = verifyURI(ozoneURIString);
-    Path path = Paths.get(ozoneURI.getPath());
-    if (path.getNameCount() < 2) {
-      throw new OzoneRestClientException(
-          "volume and bucket name required in delete Bucket");
-    }
-
-    volumeName = path.getName(0).toString();
-    bucketName = path.getName(1).toString();
-
-    if (cmd.hasOption(Shell.VERBOSE)) {
-      System.out.printf("Volume Name : %s%n", volumeName);
-      System.out.printf("Bucket Name : %s%n", bucketName);
-    }
-
-    if (cmd.hasOption(Shell.RUNAS)) {
-      rootName = "hdfs";
-    } else {
-      rootName = System.getProperty("user.name");
-    }
-
-    client.setEndPointURI(ozoneURI);
-    client.setUserAuth(rootName);
-
-    OzoneVolume vol = client.getVolume(volumeName);
-    vol.deleteBucket(bucketName);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/InfoBucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/InfoBucketHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/InfoBucketHandler.java
deleted file mode 100644
index 321c4c2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/InfoBucketHandler.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneBucket;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-/**
- * Executes Info bucket.
- */
-public class InfoBucketHandler extends Handler {
-  private String volumeName;
-  private String bucketName;
-  private String rootName;
-
-  /**
-   * Executes the Client Calls.
-   *
-   * @param cmd - CommandLine
-   *
-   * @throws IOException
-   * @throws OzoneException
-   * @throws URISyntaxException
-   */
-  @Override
-  protected void execute(CommandLine cmd)
-      throws IOException, OzoneException, URISyntaxException {
-    if (!cmd.hasOption(Shell.INFO_BUCKET)) {
-      throw new OzoneRestClientException(
-          "Incorrect call : infoBucket is missing");
-    }
-
-    String ozoneURIString = cmd.getOptionValue(Shell.INFO_BUCKET);
-    URI ozoneURI = verifyURI(ozoneURIString);
-    Path path = Paths.get(ozoneURI.getPath());
-
-    if (path.getNameCount() < 2) {
-      throw new OzoneRestClientException(
-          "volume and bucket name required in info Bucket");
-    }
-
-    volumeName = path.getName(0).toString();
-    bucketName = path.getName(1).toString();
-
-    if (cmd.hasOption(Shell.VERBOSE)) {
-      System.out.printf("Volume Name : %s%n", volumeName);
-      System.out.printf("Bucket Name : %s%n", bucketName);
-    }
-
-    if (cmd.hasOption(Shell.RUNAS)) {
-      rootName = "hdfs";
-    } else {
-      rootName = System.getProperty("user.name");
-    }
-
-    client.setEndPointURI(ozoneURI);
-    client.setUserAuth(rootName);
-
-    OzoneVolume vol = client.getVolume(volumeName);
-    OzoneBucket bucket = vol.getBucket(bucketName);
-
-    System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
-        bucket.getBucketInfo().toJsonString()));
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
deleted file mode 100644
index ea77cae..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneBucket;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Executes List Bucket.
- */
-public class ListBucketHandler extends Handler {
-  private String volumeName;
-  private String rootName;
-
-  /**
-   * Executes the Client Calls.
-   *
-   * @param cmd - CommandLine
-   *
-   * @throws IOException
-   * @throws OzoneException
-   * @throws URISyntaxException
-   */
-  @Override
-  protected void execute(CommandLine cmd)
-      throws IOException, OzoneException, URISyntaxException {
-    if (!cmd.hasOption(Shell.LIST_BUCKET)) {
-      throw new OzoneRestClientException(
-          "Incorrect call : listBucket is missing");
-    }
-
-    String ozoneURIString = cmd.getOptionValue(Shell.LIST_BUCKET);
-    URI ozoneURI = verifyURI(ozoneURIString);
-    Path path = Paths.get(ozoneURI.getPath());
-    if (path.getNameCount() < 1) {
-      throw new OzoneRestClientException("volume is required in listBucket");
-    }
-
-    volumeName = path.getName(0).toString();
-
-
-    if (cmd.hasOption(Shell.VERBOSE)) {
-      System.out.printf("Volume Name : %s%n", volumeName);
-    }
-
-    if (cmd.hasOption(Shell.RUNAS)) {
-      rootName = "hdfs";
-    } else {
-      rootName = System.getProperty("user.name");
-    }
-
-
-    client.setEndPointURI(ozoneURI);
-    client.setUserAuth(rootName);
-
-    String length = null;
-    if (cmd.hasOption(Shell.LIST_LENGTH)) {
-      length = cmd.getOptionValue(Shell.LIST_LENGTH);
-      OzoneUtils.verifyMaxKeyLength(length);
-    }
-
-    String startBucket = null;
-    if (cmd.hasOption(Shell.START)) {
-      startBucket = cmd.getOptionValue(Shell.START);
-    }
-
-    String prefix = null;
-    if (cmd.hasOption(Shell.PREFIX)) {
-      prefix = cmd.getOptionValue(Shell.PREFIX);
-    }
-
-    OzoneVolume vol = client.getVolume(volumeName);
-    List<OzoneBucket> bucketList = vol.listBuckets(length, startBucket, prefix);
-
-    List<BucketInfo> jsonData = bucketList.stream()
-        .map(OzoneBucket::getBucketInfo).collect(Collectors.toList());
-    System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(
-        JsonUtils.toJsonString(jsonData)));
-  }
-}
-

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/UpdateBucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/UpdateBucketHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/UpdateBucketHandler.java
deleted file mode 100644
index 781a00c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/UpdateBucketHandler.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneBucket;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-/**
- * Allows users to add and remove acls and from a bucket.
- */
-public class UpdateBucketHandler extends Handler {
-  private String volumeName;
-  private String bucketName;
-  private String rootName;
-
-  @Override
-  protected void execute(CommandLine cmd)
-      throws IOException, OzoneException, URISyntaxException {
-    if (!cmd.hasOption(Shell.UPDATE_BUCKET)) {
-      throw new OzoneRestClientException(
-          "Incorrect call : updateBucket is missing");
-    }
-
-    String ozoneURIString = cmd.getOptionValue(Shell.UPDATE_BUCKET);
-    URI ozoneURI = verifyURI(ozoneURIString);
-    Path path = Paths.get(ozoneURI.getPath());
-
-    if (path.getNameCount() < 2) {
-      throw new OzoneRestClientException(
-          "volume and bucket name required in update bucket");
-    }
-
-    volumeName = path.getName(0).toString();
-    bucketName = path.getName(1).toString();
-
-    if (cmd.hasOption(Shell.VERBOSE)) {
-      System.out.printf("Volume Name : %s%n", volumeName);
-      System.out.printf("Bucket Name : %s%n", bucketName);
-    }
-
-    if (cmd.hasOption(Shell.RUNAS)) {
-      rootName = "hdfs";
-    } else {
-      rootName = System.getProperty("user.name");
-    }
-
-    client.setEndPointURI(ozoneURI);
-    client.setUserAuth(rootName);
-
-    OzoneVolume vol = client.getVolume(volumeName);
-    if (cmd.hasOption(Shell.ADD_ACLS)) {
-      String aclString = cmd.getOptionValue(Shell.ADD_ACLS);
-      String[] aclArray = aclString.split(",");
-      vol.addAcls(bucketName, aclArray);
-    }
-
-    if (cmd.hasOption(Shell.REMOVE_ACLS)) {
-      String aclString = cmd.getOptionValue(Shell.REMOVE_ACLS);
-      String[] aclArray = aclString.split(",");
-      vol.removeAcls(bucketName, aclArray);
-    }
-
-    OzoneBucket bucket = vol.getBucket(bucketName);
-    System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
-        bucket.getBucketInfo().toJsonString()));
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java
deleted file mode 100644
index c344c35..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Netty-based HTTP server implementation for Ozone.
- */
-package org.apache.hadoop.ozone.web.ozShell.bucket;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/DeleteKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/DeleteKeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/DeleteKeyHandler.java
deleted file mode 100644
index 1ad2588..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/DeleteKeyHandler.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneBucket;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-/**
- * Executes Delete Key.
- */
-public class DeleteKeyHandler extends Handler {
-  private String userName;
-  private String volumeName;
-  private String bucketName;
-  private String keyName;
-
-  /**
-   * Executes the Client Calls.
-   *
-   * @param cmd - CommandLine
-   * @throws IOException
-   * @throws OzoneException
-   * @throws URISyntaxException
-   */
-  @Override
-  protected void execute(CommandLine cmd)
-      throws IOException, OzoneException, URISyntaxException {
-    if (!cmd.hasOption(Shell.DELETE_KEY)) {
-      throw new OzoneRestClientException(
-          "Incorrect call : deleteKey is missing");
-    }
-
-
-    if (cmd.hasOption(Shell.USER)) {
-      userName = cmd.getOptionValue(Shell.USER);
-    } else {
-      userName = System.getProperty("user.name");
-    }
-
-
-    String ozoneURIString = cmd.getOptionValue(Shell.DELETE_KEY);
-    URI ozoneURI = verifyURI(ozoneURIString);
-    Path path = Paths.get(ozoneURI.getPath());
-    if (path.getNameCount() < 3) {
-      throw new OzoneRestClientException(
-          "volume/bucket/key name required in deleteKey");
-    }
-
-    volumeName = path.getName(0).toString();
-    bucketName = path.getName(1).toString();
-    keyName = path.getName(2).toString();
-
-
-    if (cmd.hasOption(Shell.VERBOSE)) {
-      System.out.printf("Volume Name : %s%n", volumeName);
-      System.out.printf("Bucket Name : %s%n", bucketName);
-      System.out.printf("Key Name : %s%n", keyName);
-    }
-
-    client.setEndPointURI(ozoneURI);
-    client.setUserAuth(userName);
-
-
-    OzoneVolume vol = client.getVolume(volumeName);
-    OzoneBucket bucket = vol.getBucket(bucketName);
-    bucket.deleteKey(keyName);
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
deleted file mode 100644
index a56bbc0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-/**
- * Gets an existing key.
- */
-public class GetKeyHandler extends Handler {
-  private String userName;
-  private String volumeName;
-  private String bucketName;
-  private String keyName;
-
-
-  /**
-   * Executes the Client Calls.
-   *
-   * @param cmd - CommandLine
-   *
-   * @throws IOException
-   * @throws OzoneException
-   * @throws URISyntaxException
-   */
-  @Override
-  protected void execute(CommandLine cmd)
-      throws IOException, OzoneException, URISyntaxException {
-    if (!cmd.hasOption(Shell.GET_KEY)) {
-      throw new OzoneRestClientException("Incorrect call : getKey is missing");
-    }
-
-    if (!cmd.hasOption(Shell.FILE)) {
-      throw new OzoneRestClientException(
-          "get key needs a file path to download to");
-    }
-
-    if (cmd.hasOption(Shell.USER)) {
-      userName = cmd.getOptionValue(Shell.USER);
-    } else {
-      userName = System.getProperty("user.name");
-    }
-
-
-    String ozoneURIString = cmd.getOptionValue(Shell.GET_KEY);
-    URI ozoneURI = verifyURI(ozoneURIString);
-    Path path = Paths.get(ozoneURI.getPath());
-    if (path.getNameCount() < 3) {
-      throw new OzoneRestClientException(
-          "volume/bucket/key name required in putKey");
-    }
-
-    volumeName = path.getName(0).toString();
-    bucketName = path.getName(1).toString();
-    keyName = path.getName(2).toString();
-
-
-    if (cmd.hasOption(Shell.VERBOSE)) {
-      System.out.printf("Volume Name : %s%n", volumeName);
-      System.out.printf("Bucket Name : %s%n", bucketName);
-      System.out.printf("Key Name : %s%n", keyName);
-    }
-
-
-    String fileName = cmd.getOptionValue(Shell.FILE);
-    Path dataFilePath = Paths.get(fileName);
-    File dataFile = new File(fileName);
-
-
-    if (dataFile.exists()) {
-      throw new OzoneRestClientException(fileName +
-                                         "exists. Download will overwrite an " +
-                                         "existing file. Aborting.");
-    }
-
-    client.setEndPointURI(ozoneURI);
-    client.setUserAuth(userName);
-
-    client.getKey(volumeName, bucketName, keyName, dataFilePath);
-    if(cmd.hasOption(Shell.VERBOSE)) {
-      FileInputStream stream = new FileInputStream(dataFile);
-      String hash = DigestUtils.md5Hex(stream);
-      System.out.printf("Downloaded file hash : %s%n", hash);
-    }
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/InfoKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/InfoKeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/InfoKeyHandler.java
deleted file mode 100644
index 3e9b2da..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/InfoKeyHandler.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneBucket;
-import org.apache.hadoop.ozone.web.client.OzoneKey;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-/**
- * Executes Info Object.
- */
-public class InfoKeyHandler extends Handler {
-  private String userName;
-  private String volumeName;
-  private String bucketName;
-  private String keyName;
-
-  /**
-   * Executes the Client Calls.
-   *
-   * @param cmd - CommandLine
-   * @throws IOException
-   * @throws OzoneException
-   * @throws URISyntaxException
-   */
-  @Override
-  protected void execute(CommandLine cmd)
-      throws IOException, OzoneException, URISyntaxException {
-    if (!cmd.hasOption(Shell.INFO_KEY)) {
-      throw new OzoneRestClientException("Incorrect call : infoKey is missing");
-    }
-
-
-    if (cmd.hasOption(Shell.USER)) {
-      userName = cmd.getOptionValue(Shell.USER);
-    } else {
-      userName = System.getProperty("user.name");
-    }
-
-
-    String ozoneURIString = cmd.getOptionValue(Shell.INFO_KEY);
-    URI ozoneURI = verifyURI(ozoneURIString);
-    Path path = Paths.get(ozoneURI.getPath());
-    if (path.getNameCount() < 3) {
-      throw new OzoneRestClientException(
-          "volume/bucket/key name required in infoKey");
-    }
-
-    volumeName = path.getName(0).toString();
-    bucketName = path.getName(1).toString();
-    keyName = path.getName(2).toString();
-
-
-    if (cmd.hasOption(Shell.VERBOSE)) {
-      System.out.printf("Volume Name : %s%n", volumeName);
-      System.out.printf("Bucket Name : %s%n", bucketName);
-      System.out.printf("Key Name : %s%n", keyName);
-    }
-
-    client.setEndPointURI(ozoneURI);
-    client.setUserAuth(userName);
-
-    OzoneVolume vol = client.getVolume(volumeName);
-    OzoneBucket bucket = vol.getBucket(bucketName);
-    OzoneKey key = bucket.getKeyInfo(keyName);
-
-    System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
-        key.getObjectInfo().toJsonString()));
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
deleted file mode 100644
index ea563ad..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneKey;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Executes List Keys.
- */
-public class ListKeyHandler extends Handler {
-  private String userName;
-  private String volumeName;
-  private String bucketName;
-
-  /**
-   * Executes the Client Calls.
-   *
-   * @param cmd - CommandLine
-   * @throws IOException
-   * @throws OzoneException
-   * @throws URISyntaxException
-   */
-  @Override
-  protected void execute(CommandLine cmd)
-      throws IOException, OzoneException, URISyntaxException {
-
-    if (!cmd.hasOption(Shell.LIST_KEY)) {
-      throw new OzoneRestClientException(
-          "Incorrect call : listKey is missing");
-    }
-
-    String length = null;
-    if (cmd.hasOption(Shell.LIST_LENGTH)) {
-      length = cmd.getOptionValue(Shell.LIST_LENGTH);
-      OzoneUtils.verifyMaxKeyLength(length);
-    }
-
-    String startKey = null;
-    if (cmd.hasOption(Shell.START)) {
-      startKey = cmd.getOptionValue(Shell.START);
-    }
-
-    String prefix = null;
-    if (cmd.hasOption(Shell.PREFIX)) {
-      prefix = cmd.getOptionValue(Shell.PREFIX);
-    }
-
-    String ozoneURIString = cmd.getOptionValue(Shell.LIST_KEY);
-    URI ozoneURI = verifyURI(ozoneURIString);
-    Path path = Paths.get(ozoneURI.getPath());
-    if (path.getNameCount() < 2) {
-      throw new OzoneRestClientException(
-          "volume/bucket is required in listKey");
-    }
-
-    volumeName = path.getName(0).toString();
-    bucketName = path.getName(1).toString();
-
-
-    if (cmd.hasOption(Shell.VERBOSE)) {
-      System.out.printf("Volume Name : %s%n", volumeName);
-      System.out.printf("bucket Name : %s%n", bucketName);
-    }
-
-    if (cmd.hasOption(Shell.USER)) {
-      userName = cmd.getOptionValue(Shell.USER);
-    } else {
-      userName = System.getProperty("user.name");
-    }
-
-
-    client.setEndPointURI(ozoneURI);
-    client.setUserAuth(userName);
-
-    List<OzoneKey> keys = client.listKeys(volumeName, bucketName, length,
-        startKey, prefix);
-
-    List<KeyInfo> jsonData = keys.stream()
-        .map(OzoneKey::getObjectInfo).collect(Collectors.toList());
-    System.out.printf(JsonUtils.toJsonStringWithDefaultPrettyPrinter(
-        JsonUtils.toJsonString(jsonData)));
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
deleted file mode 100644
index 1f2c692..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-/**
- * Puts a file into an ozone bucket.
- */
-public class PutKeyHandler extends Handler {
-  private String userName;
-  private String volumeName;
-  private String bucketName;
-  private String keyName;
-
-  /**
-   * Executes the Client Calls.
-   *
-   * @param cmd - CommandLine
-   * @throws IOException
-   * @throws OzoneException
-   * @throws URISyntaxException
-   */
-  @Override
-  protected void execute(CommandLine cmd)
-      throws IOException, OzoneException, URISyntaxException {
-    if (!cmd.hasOption(Shell.PUT_KEY)) {
-      throw new OzoneRestClientException("Incorrect call : putKey is missing");
-    }
-
-    if (!cmd.hasOption(Shell.FILE)) {
-      throw new OzoneRestClientException("put key needs a file to put");
-    }
-
-    if (cmd.hasOption(Shell.USER)) {
-      userName = cmd.getOptionValue(Shell.USER);
-    } else {
-      userName = System.getProperty("user.name");
-    }
-
-    String ozoneURIString = cmd.getOptionValue(Shell.PUT_KEY);
-    URI ozoneURI = verifyURI(ozoneURIString);
-    Path path = Paths.get(ozoneURI.getPath());
-    if (path.getNameCount() < 3) {
-      throw new OzoneRestClientException(
-          "volume/bucket/key name required in putKey");
-    }
-
-    volumeName = path.getName(0).toString();
-    bucketName = path.getName(1).toString();
-    keyName = path.getName(2).toString();
-
-
-    if (cmd.hasOption(Shell.VERBOSE)) {
-      System.out.printf("Volume Name : %s%n", volumeName);
-      System.out.printf("Bucket Name : %s%n", bucketName);
-      System.out.printf("Key Name : %s%n", keyName);
-    }
-
-
-    String fileName = cmd.getOptionValue(Shell.FILE);
-    File dataFile = new File(fileName);
-
-    if (cmd.hasOption(Shell.VERBOSE)) {
-      FileInputStream stream = new FileInputStream(dataFile);
-      String hash = DigestUtils.md5Hex(stream);
-      System.out.printf("File Hash : %s%n", hash);
-      stream.close();
-    }
-
-    client.setEndPointURI(ozoneURI);
-    client.setUserAuth(userName);
-
-    client.putKey(volumeName, bucketName, keyName, dataFile);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/package-info.java
deleted file mode 100644
index 1deb7ad..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Netty-based HTTP server implementation for Ozone.
- */
-package org.apache.hadoop.ozone.web.ozShell.keys;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
deleted file mode 100644
index e33b6e7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/**
- *  ozShell Class acts as the command line interface to
- *  the ozone Rest Client.
- */
-package org.apache.hadoop.ozone.web.ozShell;
-
-/**
- A simple CLI to work against Ozone.
- **/

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java
deleted file mode 100644
index c0b0bb9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-/**
- * Executes the create volume call for the shell.
- */
-public class CreateVolumeHandler extends Handler {
-
-  private String rootName;
-  private String userName;
-  private String volumeName;
-  private String quota;
-
-  /**
-   * Executes the Create Volume.
-   *
-   * @param cmd - CommandLine
-   * @throws IOException
-   * @throws OzoneException
-   * @throws URISyntaxException
-   */
-  @Override
-  protected void execute(CommandLine cmd)
-      throws IOException, OzoneException, URISyntaxException {
-    if (!cmd.hasOption(Shell.CREATE_VOLUME)) {
-      throw new OzoneRestClientException(
-          "Incorrect call : createVolume is missing");
-    }
-
-    String ozoneURIString = cmd.getOptionValue(Shell.CREATE_VOLUME);
-    URI ozoneURI = verifyURI(ozoneURIString);
-    if (ozoneURI.getPath().isEmpty()) {
-      throw new OzoneRestClientException(
-          "Volume name is required to create a volume");
-    }
-
-    // we need to skip the slash in the URI path
-    // getPath returns /volumeName needs to remove the first slash.
-    volumeName = ozoneURI.getPath().substring(1);
-
-    if (cmd.hasOption(Shell.VERBOSE)) {
-      System.out.printf("Volume name : %s%n", volumeName);
-    }
-    if (cmd.hasOption(Shell.RUNAS)) {
-      rootName = "hdfs";
-    } else {
-      rootName = System.getProperty("user.name");
-    }
-
-    if (!cmd.hasOption(Shell.USER)) {
-      throw new OzoneRestClientException(
-          "User name is needed in createVolume call.");
-    }
-
-    if (cmd.hasOption(Shell.QUOTA)) {
-      quota = cmd.getOptionValue(Shell.QUOTA);
-    }
-
-    userName = cmd.getOptionValue(Shell.USER);
-    client.setEndPointURI(ozoneURI);
-    client.setUserAuth(rootName);
-
-    OzoneVolume vol = client.createVolume(volumeName, userName, quota);
-    if (cmd.hasOption(Shell.VERBOSE)) {
-      System.out.printf("%s%n",
-          JsonUtils.toJsonStringWithDefaultPrettyPrinter(vol.getJsonString()));
-    }
-  }
-}
-

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
deleted file mode 100644
index bed7db3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-/**
- * Executes deleteVolume call for the shell.
- */
-public class DeleteVolumeHandler extends Handler {
-
-  private String volumeName;
-  private String rootName;
-
-  /**
-   * Executes the delete volume call.
-   *
-   * @param cmd - CommandLine
-   * @throws IOException
-   * @throws OzoneException
-   * @throws URISyntaxException
-   */
-  @Override
-  protected void execute(CommandLine cmd)
-      throws IOException, OzoneException, URISyntaxException {
-
-    if (!cmd.hasOption(Shell.DELETE_VOLUME)) {
-      throw new OzoneRestClientException(
-          "Incorrect call : deleteVolume call is missing");
-    }
-
-    String ozoneURIString = cmd.getOptionValue(Shell.DELETE_VOLUME);
-    URI ozoneURI = verifyURI(ozoneURIString);
-    if (ozoneURI.getPath().isEmpty()) {
-      throw new OzoneRestClientException(
-          "Volume name is required to delete a volume");
-    }
-
-    // we need to skip the slash in the URI path
-    volumeName = ozoneURI.getPath().substring(1);
-
-    if (cmd.hasOption(Shell.VERBOSE)) {
-      System.out.printf("Volume name : %s%n", volumeName);
-    }
-
-    if (cmd.hasOption(Shell.RUNAS)) {
-      rootName = "hdfs";
-    } else {
-      rootName = System.getProperty("user.name");
-    }
-
-    client.setEndPointURI(ozoneURI);
-    client.setUserAuth(rootName);
-    client.deleteVolume(volumeName);
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
deleted file mode 100644
index 16de3d4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-/**
- * Executes volume Info calls.
- */
-public class InfoVolumeHandler extends Handler{
-
-  private String rootName;
-  private String volumeName;
-  private String userName;
-
-  /**
-   * Executes volume Info.
-   *
-   * @param cmd - CommandLine
-   *
-   * @throws IOException
-   * @throws OzoneException
-   * @throws URISyntaxException
-   */
-  @Override
-  protected void execute(CommandLine cmd)
-      throws IOException, OzoneException, URISyntaxException {
-
-    if (!cmd.hasOption(Shell.INFO_VOLUME)) {
-      throw new OzoneRestClientException(
-          "Incorrect call : infoVolume is missing");
-    }
-
-    String ozoneURIString = cmd.getOptionValue(Shell.INFO_VOLUME);
-    URI ozoneURI = verifyURI(ozoneURIString);
-    if (ozoneURI.getPath().isEmpty()) {
-      throw new OzoneRestClientException(
-          "Volume name is required to get info of a volume");
-    }
-
-    if (cmd.hasOption(Shell.RUNAS)) {
-      rootName = "hdfs";
-    }
-
-    // we need to skip the slash in the URI path
-    volumeName = ozoneURI.getPath().substring(1);
-
-    if (cmd.hasOption(Shell.USER)) {
-      userName = cmd.getOptionValue(Shell.USER);
-    } else {
-      userName = System.getProperty("user.name");
-    }
-
-    client.setEndPointURI(ozoneURI);
-
-    if (rootName != null) {
-      client.setUserAuth(rootName);
-    } else {
-      client.setUserAuth(userName);
-    }
-
-    client.setEndPointURI(ozoneURI);
-    client.setUserAuth(rootName);
-
-    OzoneVolume vol = client.getVolume(volumeName);
-    System.out.printf("%s%n",
-        JsonUtils.toJsonStringWithDefaultPrettyPrinter(vol.getJsonString()));
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java
deleted file mode 100644
index 189c891..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Executes List Volume call.
- */
-public class ListVolumeHandler extends Handler {
-  private String rootName;
-  private String userName;
-
-  /**
-   * Executes the Client Calls.
-   *
-   * @param cmd - CommandLine
-   * @throws IOException
-   * @throws OzoneException
-   * @throws URISyntaxException
-   */
-  @Override
-  protected void execute(CommandLine cmd)
-      throws IOException, OzoneException, URISyntaxException {
-
-    if (!cmd.hasOption(Shell.LIST_VOLUME)) {
-      throw new OzoneRestClientException(
-          "Incorrect call : listVolume is missing");
-    }
-
-    int maxKeys = 0;
-    if (cmd.hasOption(Shell.LIST_LENGTH)) {
-      String length = cmd.getOptionValue(Shell.LIST_LENGTH);
-      OzoneUtils.verifyMaxKeyLength(length);
-
-      maxKeys = Integer.parseInt(length);
-    }
-
-    String startVolume = null;
-    if (cmd.hasOption(Shell.START)) {
-      startVolume = cmd.getOptionValue(Shell.START);
-    }
-
-    String prefix = null;
-    if (cmd.hasOption(Shell.PREFIX)) {
-      prefix = cmd.getOptionValue(Shell.PREFIX);
-    }
-
-    String ozoneURIString = cmd.getOptionValue(Shell.LIST_VOLUME);
-    URI ozoneURI = verifyURI(ozoneURIString);
-
-    if (cmd.hasOption(Shell.RUNAS)) {
-      rootName = "hdfs";
-    }
-
-    if (cmd.hasOption(Shell.USER)) {
-      userName = cmd.getOptionValue(Shell.USER);
-    } else {
-      userName = System.getProperty("user.name");
-    }
-
-    client.setEndPointURI(ozoneURI);
-    if (rootName != null) {
-      client.setUserAuth(rootName);
-    } else {
-      client.setUserAuth(userName);
-    }
-
-    List<OzoneVolume> volumes = client.listVolumes(userName, prefix, maxKeys,
-        startVolume);
-    if (volumes != null) {
-      if (cmd.hasOption(Shell.VERBOSE)) {
-        System.out.printf("Found : %d volumes for user : %s %n", volumes.size(),
-            userName);
-      }
-
-      List<VolumeInfo> jsonData = volumes.stream()
-          .map(OzoneVolume::getVolumeInfo).collect(Collectors.toList());
-      System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(
-          JsonUtils.toJsonString(jsonData)));
-    }
-  }
-}
-

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
deleted file mode 100644
index 164fe34..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-/**
- * Executes update volume calls.
- */
-public class UpdateVolumeHandler extends Handler {
-  private String rootName;
-  private String ownerName;
-  private String volumeName;
-  private String quota;
-
-  /**
-   * Executes update volume calls.
-   *
-   * @param cmd - CommandLine
-   * @throws IOException
-   * @throws OzoneException
-   * @throws URISyntaxException
-   */
-  @Override
-  protected void execute(CommandLine cmd)
-      throws IOException, OzoneException, URISyntaxException {
-    if (!cmd.hasOption(Shell.UPDATE_VOLUME)) {
-      throw new OzoneRestClientException(
-          "Incorrect call : updateVolume is missing");
-    }
-
-    String ozoneURIString = cmd.getOptionValue(Shell.UPDATE_VOLUME);
-    URI ozoneURI = verifyURI(ozoneURIString);
-    if (ozoneURI.getPath().isEmpty()) {
-      throw new OzoneRestClientException(
-          "Volume name is required to update a volume");
-    }
-
-    // we need to skip the slash in the URI path
-    volumeName = ozoneURI.getPath().substring(1);
-
-    if (cmd.hasOption(Shell.RUNAS)) {
-      rootName = "hdfs";
-    } else {
-      rootName = System.getProperty("user.name");
-    }
-
-    if (cmd.hasOption(Shell.QUOTA)) {
-      quota = cmd.getOptionValue(Shell.QUOTA);
-    }
-
-    if (cmd.hasOption(Shell.USER)) {
-      ownerName = cmd.getOptionValue(Shell.USER);
-    }
-
-    client.setEndPointURI(ozoneURI);
-    client.setUserAuth(rootName);
-
-    if (quota != null && !quota.isEmpty()) {
-      client.setVolumeQuota(volumeName, quota);
-    }
-
-    if (ownerName != null && !ownerName.isEmpty()) {
-      client.setVolumeOwner(volumeName, ownerName);
-    }
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/package-info.java
deleted file mode 100644
index fc19274..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Netty-based HTTP server implementation for Ozone.
- */
-package org.apache.hadoop.ozone.web.ozShell.volume;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/package-info.java
deleted file mode 100644
index 1a7275c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web;
-
-/**
- * This package contains generic class for the internal http server
- * and REST interfaces.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/OzoneQuota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/OzoneQuota.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/OzoneQuota.java
deleted file mode 100644
index 9619ebd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/OzoneQuota.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.request;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-
-/**
- * represents an OzoneQuota Object that can be applied to
- * a storage volume.
- */
-@InterfaceAudience.Private
-public class OzoneQuota {
-
-  private Units unit;
-  private int size;
-
-  /** Quota Units.*/
-  public enum Units {UNDEFINED, BYTES, MB, GB, TB}
-
-  /**
-   * Returns size.
-   *
-   * @return int
-   */
-  public int getSize() {
-    return size;
-  }
-
-  /**
-   * Returns Units.
-   *
-   * @return Unit in MB, GB or TB
-   */
-  public Units getUnit() {
-    return unit;
-  }
-
-  /**
-   * Constructs a default Quota object.
-   */
-  public OzoneQuota() {
-    this.size = 0;
-    this.unit = Units.UNDEFINED;
-  }
-
-  /**
-   * Constructor for Ozone Quota.
-   *
-   * @param size - Integer Size
-   * @param unit MB, GB  or TB
-   */
-  public OzoneQuota(int size, Units unit) {
-    this.size = size;
-    this.unit = unit;
-  }
-
-  /**
-   * Formats a quota as a string.
-   *
-   * @param quota the quota to format
-   * @return string representation of quota
-   */
-  public static String formatQuota(OzoneQuota quota) {
-    return String.valueOf(quota.size) + quota.unit;
-  }
-
-  /**
-   * Parses a user provided string and returns the
-   * Quota Object.
-   *
-   * @param quotaString Quota String
-   *
-   * @return OzoneQuota object
-   *
-   * @throws IllegalArgumentException
-   */
-  public static OzoneQuota parseQuota(String quotaString)
-      throws IllegalArgumentException {
-
-    if ((quotaString == null) || (quotaString.isEmpty())) {
-      throw new IllegalArgumentException(
-          "Quota string cannot be null or empty.");
-    }
-
-    if (isRemove(quotaString)) {
-      throw new IllegalArgumentException("Remove is invalid in this context.");
-    }
-
-    String uppercase = quotaString.toUpperCase().replaceAll("\\s+", "");
-    String size = "";
-    int nSize;
-    Units currUnit = Units.MB;
-    Boolean found = false;
-    if (uppercase.endsWith(Header.OZONE_QUOTA_MB)) {
-      size = uppercase
-          .substring(0, uppercase.length() - Header.OZONE_QUOTA_MB.length());
-      currUnit = Units.MB;
-      found = true;
-    }
-
-    if (uppercase.endsWith(Header.OZONE_QUOTA_GB)) {
-      size = uppercase
-          .substring(0, uppercase.length() - Header.OZONE_QUOTA_GB.length());
-      currUnit = Units.GB;
-      found = true;
-    }
-
-    if (uppercase.endsWith(Header.OZONE_QUOTA_TB)) {
-      size = uppercase
-          .substring(0, uppercase.length() - Header.OZONE_QUOTA_TB.length());
-      currUnit = Units.TB;
-      found = true;
-    }
-
-    if (uppercase.endsWith(Header.OZONE_QUOTA_BYTES)) {
-      size = uppercase
-          .substring(0, uppercase.length() - Header.OZONE_QUOTA_BYTES.length());
-      currUnit = Units.BYTES;
-      found = true;
-    }
-
-    if (!found) {
-      throw new IllegalArgumentException(
-          "Quota unit not recognized. Supported values are BYTES, MB, GB and " +
-              "TB.");
-    }
-
-    nSize = Integer.parseInt(size);
-    if (nSize < 0) {
-      throw new IllegalArgumentException("Quota cannot be negative.");
-    }
-
-    return new OzoneQuota(nSize, currUnit);
-  }
-
-
-  /**
-   * Checks if Quota String is just as remove operation.
-   *
-   * @param quotaString User provided quota String
-   *
-   * @return True if it is Remove, false otherwise
-   */
-  public static boolean isRemove(String quotaString) {
-
-    return (quotaString != null) &&
-        (quotaString.compareToIgnoreCase(Header.OZONE_QUOTA_REMOVE) == 0);
-  }
-
-  /**
-   * Returns size in Bytes or -1 if there is no Quota.
-   */
-  @JsonIgnore
-  public long sizeInBytes() {
-    switch (this.unit) {
-    case BYTES:
-      return this.getSize();
-    case MB:
-      return this.getSize() * OzoneConsts.MB;
-    case GB:
-      return this.getSize() * OzoneConsts.GB;
-    case TB:
-      return this.getSize() * OzoneConsts.TB;
-    case UNDEFINED:
-    default:
-      return -1;
-    }
-  }
-
-  /**
-   * Returns OzoneQuota corresponding to size in bytes.
-   *
-   * @param sizeInBytes size in bytes to be converted
-   *
-   * @return OzoneQuota object
-   */
-  public static OzoneQuota getOzoneQuota(long sizeInBytes) {
-    long size;
-    Units unit;
-    if (sizeInBytes % OzoneConsts.TB == 0) {
-      size = sizeInBytes / OzoneConsts.TB;
-      unit = Units.TB;
-    } else if (sizeInBytes % OzoneConsts.GB == 0) {
-      size = sizeInBytes / OzoneConsts.GB;
-      unit = Units.GB;
-    } else if (sizeInBytes % OzoneConsts.MB == 0) {
-      size = sizeInBytes / OzoneConsts.MB;
-      unit = Units.MB;
-    } else {
-      size = sizeInBytes;
-      unit = Units.BYTES;
-    }
-    return new OzoneQuota((int)size, unit);
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[12/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/package-info.java
deleted file mode 100644
index 4fbc18f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Netty-based HTTP server implementation for Ozone.
- */
-package org.apache.hadoop.ozone.web.request;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java
deleted file mode 100644
index e66cd20..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java
+++ /dev/null
@@ -1,325 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.web.response;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonFilter;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.ser.FilterProvider;
-import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
-import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-import com.google.common.base.Preconditions;
-
-/**
- * BucketInfo class, this is used as response class to send
- * Json info about a bucket back to a client.
- */
-public class BucketInfo implements Comparable<BucketInfo> {
-  static final String BUCKET_INFO = "BUCKET_INFO_FILTER";
-  private static final ObjectReader READER =
-      new ObjectMapper().readerFor(BucketInfo.class);
-  private static final ObjectWriter WRITER;
-
-  static {
-    ObjectMapper mapper = new ObjectMapper();
-    String[] ignorableFieldNames = {"bytesUsed", "keyCount"};
-
-    FilterProvider filters = new SimpleFilterProvider().addFilter(BUCKET_INFO,
-        SimpleBeanPropertyFilter.serializeAllExcept(ignorableFieldNames));
-    mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
-    mapper.addMixIn(Object.class, MixIn.class);
-
-    mapper.setFilterProvider(filters);
-    WRITER = mapper.writerWithDefaultPrettyPrinter();
-  }
-
-  private String volumeName;
-  private String bucketName;
-  private String createdOn;
-  private List<OzoneAcl> acls;
-  private OzoneConsts.Versioning versioning;
-  private StorageType storageType;
-  private long bytesUsed;
-  private long keyCount;
-
-  /**
-   * Constructor for BucketInfo.
-   *
-   * @param volumeName
-   * @param bucketName
-   */
-  public BucketInfo(String volumeName, String bucketName) {
-    this.volumeName = volumeName;
-    this.bucketName = bucketName;
-  }
-
-
-  /**
-   * Default constructor for BucketInfo.
-   */
-  public BucketInfo() {
-    acls = new LinkedList<OzoneAcl>();
-  }
-
-  /**
-   * Parse a JSON string into BucketInfo Object.
-   *
-   * @param jsonString - Json String
-   *
-   * @return - BucketInfo
-   *
-   * @throws IOException
-   */
-  public static BucketInfo parse(String jsonString) throws IOException {
-    return READER.readValue(jsonString);
-  }
-
-  /**
-   * Returns a List of ACL on the Bucket.
-   *
-   * @return List of Acls
-   */
-  public List<OzoneAcl> getAcls() {
-    return acls;
-  }
-
-  /**
-   * Sets ACls.
-   *
-   * @param acls - Acls list
-   */
-  public void setAcls(List<OzoneAcl> acls) {
-    this.acls = acls;
-  }
-
-  /**
-   * Returns Storage Type info.
-   *
-   * @return Storage Type of the bucket
-   */
-  public StorageType getStorageType() {
-    return storageType;
-  }
-
-  /**
-   * Sets the Storage Type.
-   *
-   * @param storageType - Storage Type
-   */
-  public void setStorageType(StorageType storageType) {
-    this.storageType = storageType;
-  }
-
-  /**
-   * Returns versioning.
-   *
-   * @return versioning Enum
-   */
-  public OzoneConsts.Versioning getVersioning() {
-    return versioning;
-  }
-
-  /**
-   * Sets Versioning.
-   *
-   * @param versioning
-   */
-  public void setVersioning(OzoneConsts.Versioning versioning) {
-    this.versioning = versioning;
-  }
-
-
-  /**
-   * Gets bucket Name.
-   *
-   * @return String
-   */
-  public String getBucketName() {
-    return bucketName;
-  }
-
-  /**
-   * Sets bucket Name.
-   *
-   * @param bucketName - Name of the bucket
-   */
-  public void setBucketName(String bucketName) {
-    this.bucketName = bucketName;
-  }
-
-  /**
-   * Sets creation time of the bucket.
-   *
-   * @param creationTime - Date String
-   */
-  public void setCreatedOn(String creationTime) {
-    this.createdOn = creationTime;
-  }
-
-  /**
-   * Returns creation time.
-   *
-   * @return creation time of bucket.
-   */
-  public String getCreatedOn() {
-    return createdOn;
-  }
-
-  /**
-   * Returns a JSON string of this object.
-   * After stripping out bytesUsed and keyCount
-   *
-   * @return String
-   */
-  public String toJsonString() throws IOException {
-    return WRITER.writeValueAsString(this);
-  }
-
-  /**
-   * Returns the Object as a Json String.
-   *
-   * The reason why both toJSONString exists and toDBString exists
-   * is because toJSONString supports an external facing contract with
-   * REST clients. However server internally would want to add more
-   * fields to this class. The distinction helps in serializing all
-   * fields vs. only fields that are part of REST protocol.
-   */
-  public String toDBString() throws IOException {
-    return JsonUtils.toJsonString(this);
-  }
-
-  /**
-   * Returns Volume Name.
-   *
-   * @return String volume name
-   */
-  public String getVolumeName() {
-    return volumeName;
-  }
-
-  /**
-   * Sets the Volume Name of the bucket.
-   *
-   * @param volumeName - volumeName
-   */
-  public void setVolumeName(String volumeName) {
-    this.volumeName = volumeName;
-  }
-
-  /**
-   * Compares this object with the specified object for order.  Returns a
-   * negative integer, zero, or a positive integer as this object is less
-   * than, equal to, or greater than the specified object.
-   *
-   * Please note : BucketInfo compare functions are used only within the
-   * context of a volume, hence volume name is purposefully ignored in
-   * compareTo, equal and hashcode functions of this class.
-   */
-  @Override
-  public int compareTo(BucketInfo o) {
-    Preconditions.checkState(o.getVolumeName().equals(this.getVolumeName()));
-    return this.bucketName.compareTo(o.getBucketName());
-  }
-
-  /**
-   * Checks if two bucketInfo's are equal.
-   * @param o Object BucketInfo
-   * @return  True or False
-   */
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (!(o instanceof BucketInfo)) {
-      return false;
-    }
-
-    BucketInfo that = (BucketInfo) o;
-    Preconditions.checkState(that.getVolumeName().equals(this.getVolumeName()));
-    return bucketName.equals(that.bucketName);
-
-  }
-
-  /**
-   * Hash Code for this object.
-   * @return int
-   */
-  @Override
-  public int hashCode() {
-    return bucketName.hashCode();
-  }
-
-  /**
-   * Get the number of bytes used by this bucket.
-   *
-   * @return long
-   */
-  public long getBytesUsed() {
-    return bytesUsed;
-  }
-
-  /**
-   * Set bytes Used.
-   *
-   * @param bytesUsed - bytesUsed
-   */
-  public void setBytesUsed(long bytesUsed) {
-    this.bytesUsed = bytesUsed;
-  }
-
-  /**
-   * Get Key Count  inside this bucket.
-   *
-   * @return - KeyCount
-   */
-  public long getKeyCount() {
-    return keyCount;
-  }
-
-  /**
-   * Set Key Count inside this bucket.
-   *
-   * @param keyCount - Sets the Key Count
-   */
-  public void setKeyCount(long keyCount) {
-    this.keyCount = keyCount;
-  }
-
-  /**
-   * This class allows us to create custom filters
-   * for the Json serialization.
-   */
-  @JsonFilter(BUCKET_INFO)
-  class MixIn {
-
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
deleted file mode 100644
index 34885f6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.web.response;
-
-import java.io.IOException;
-
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonFilter;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.ser.FilterProvider;
-import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
-import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-
-/**
- * Represents an Ozone key Object.
- */
-public class KeyInfo implements Comparable<KeyInfo> {
-  static final String OBJECT_INFO = "OBJECT_INFO_FILTER";
-
-  private static final ObjectReader READER =
-      new ObjectMapper().readerFor(KeyInfo.class);
-  private static final ObjectWriter WRITER;
-
-  static {
-    ObjectMapper mapper = new ObjectMapper();
-    String[] ignorableFieldNames = {"dataFileName"};
-
-    FilterProvider filters = new SimpleFilterProvider()
-        .addFilter(OBJECT_INFO, SimpleBeanPropertyFilter
-            .serializeAllExcept(ignorableFieldNames));
-    mapper.setVisibility(PropertyAccessor.FIELD,
-        JsonAutoDetect.Visibility.ANY);
-    mapper.addMixIn(Object.class, MixIn.class);
-
-    mapper.setFilterProvider(filters);
-    WRITER = mapper.writerWithDefaultPrettyPrinter();
-  }
-
-  /**
-   * This class allows us to create custom filters
-   * for the Json serialization.
-   */
-  @JsonFilter(OBJECT_INFO)
-  class MixIn {
-
-  }
-  private long version;
-  private String md5hash;
-  private String createdOn;
-  private String modifiedOn;
-  private long size;
-  private String keyName;
-
-  private String dataFileName;
-
-  /**
-   * When this key was created.
-   *
-   * @return Date String
-   */
-  public String getCreatedOn() {
-    return createdOn;
-  }
-
-  /**
-   * When this key was modified.
-   *
-   * @return Date String
-   */
-  public String getModifiedOn() {
-    return modifiedOn;
-  }
-
-  /**
-   * When this key was created.
-   *
-   * @param createdOn - Date String
-   */
-  public void setCreatedOn(String createdOn) {
-    this.createdOn = createdOn;
-  }
-
-  /**
-   * When this key was modified.
-   *
-   * @param modifiedOn - Date String
-   */
-  public void setModifiedOn(String modifiedOn) {
-    this.modifiedOn = modifiedOn;
-  }
-
-  /**
-   * Full path to where the actual data for this key is stored.
-   *
-   * @return String
-   */
-  public String getDataFileName() {
-    return dataFileName;
-  }
-
-  /**
-   * Sets up where the file path is stored.
-   *
-   * @param dataFileName - Data File Name
-   */
-  public void setDataFileName(String dataFileName) {
-    this.dataFileName = dataFileName;
-  }
-
-  /**
-   * Gets the Keyname of this object.
-   *
-   * @return String
-   */
-  public String getKeyName() {
-    return keyName;
-  }
-
-  /**
-   * Sets the Key name of this object.
-   *
-   * @param keyName - String
-   */
-  public void setKeyName(String keyName) {
-    this.keyName = keyName;
-  }
-
-  /**
-   * Returns the MD5 Hash for the data of this key.
-   *
-   * @return String MD5
-   */
-  public String getMd5hash() {
-    return md5hash;
-  }
-
-  /**
-   * Sets the MD5 of this file.
-   *
-   * @param md5hash - Md5 of this file
-   */
-  public void setMd5hash(String md5hash) {
-    this.md5hash = md5hash;
-  }
-
-  /**
-   * Number of bytes stored in the data part of this key.
-   *
-   * @return long size of the data file
-   */
-  public long getSize() {
-    return size;
-  }
-
-  /**
-   * Sets the size of the Data part of this key.
-   *
-   * @param size - Size in long
-   */
-  public void setSize(long size) {
-    this.size = size;
-  }
-
-  /**
-   * Version of this key.
-   *
-   * @return - returns the version of this key.
-   */
-  public long getVersion() {
-    return version;
-  }
-
-  /**
-   * Sets the version of this key.
-   *
-   * @param version - Version String
-   */
-  public void setVersion(long version) {
-    this.version = version;
-  }
-
-  /**
-   * Compares this object with the specified object for order.  Returns a
-   * negative integer, zero, or a positive integer as this object is less
-   * than, equal to, or greater than the specified object.
-   *
-   * @param o the object to be compared.
-   *
-   * @return a negative integer, zero, or a positive integer as this object
-   * is less than, equal to, or greater than the specified object.
-   *
-   * @throws NullPointerException if the specified object is null
-   * @throws ClassCastException if the specified object's type prevents it
-   * from being compared to this object.
-   */
-  @Override
-  public int compareTo(KeyInfo o) {
-    if (this.keyName.compareTo(o.getKeyName()) != 0) {
-      return this.keyName.compareTo(o.getKeyName());
-    }
-
-    if (this.getVersion() == o.getVersion()) {
-      return 0;
-    }
-    if (this.getVersion() < o.getVersion()) {
-      return -1;
-    }
-    return 1;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-
-    KeyInfo keyInfo = (KeyInfo) o;
-
-    return new EqualsBuilder()
-        .append(version, keyInfo.version)
-        .append(keyName, keyInfo.keyName)
-        .isEquals();
-  }
-
-  @Override
-  public int hashCode() {
-    return new HashCodeBuilder(17, 37)
-        .append(version)
-        .append(keyName)
-        .toHashCode();
-  }
-
-  /**
-
-   * Parse a string to retuen BucketInfo Object.
-   *
-   * @param jsonString - Json String
-   *
-   * @return - BucketInfo
-   *
-   * @throws IOException
-   */
-  public static KeyInfo parse(String jsonString) throws IOException {
-    return READER.readValue(jsonString);
-  }
-
-
-  /**
-   * Returns a JSON string of this object.
-   * After stripping out bytesUsed and keyCount
-   *
-   * @return String
-   */
-  public String toJsonString() throws IOException {
-    return WRITER.writeValueAsString(this);
-  }
-
-  /**
-   * Returns the Object as a Json String.
-   */
-  public String toDBString() throws IOException {
-    return JsonUtils.toJsonString(this);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListBuckets.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListBuckets.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListBuckets.java
deleted file mode 100644
index bc4e65b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListBuckets.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.web.response;
-
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonFilter;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.ser.FilterProvider;
-import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
-import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-
-/**
- * List Bucket is the response for the ListBucket Query.
- */
-public class ListBuckets {
-  static final String BUCKET_LIST = "BUCKET_LIST_FILTER";
-  private static final ObjectReader READER =
-      new ObjectMapper().readerFor(ListBuckets.class);
-  private static final ObjectWriter WRITER;
-
-  static {
-    ObjectMapper mapper = new ObjectMapper();
-    String[] ignorableFieldNames = {"dataFileName"};
-
-    FilterProvider filters = new SimpleFilterProvider()
-        .addFilter(BUCKET_LIST, SimpleBeanPropertyFilter
-            .serializeAllExcept(ignorableFieldNames));
-    mapper.setVisibility(PropertyAccessor.FIELD,
-        JsonAutoDetect.Visibility.ANY);
-    mapper.addMixIn(Object.class, MixIn.class);
-
-    mapper.setFilterProvider(filters);
-    WRITER = mapper.writerWithDefaultPrettyPrinter();
-  }
-
-  private List<BucketInfo> buckets;
-
-  /**
-   * Constructor for ListBuckets.
-   * @param buckets - List of buckets owned by this user
-   */
-  public ListBuckets(List<BucketInfo> buckets) {
-    this.buckets = buckets;
-
-  }
-
-  /**
-   * Constructor for ListBuckets.
-  */
-  public ListBuckets() {
-    this.buckets = new LinkedList<BucketInfo>();
-  }
-
-  /**
-   * Parses a String to return ListBuckets object.
-   *
-   * @param data - Json String
-   *
-   * @return - ListBuckets
-   *
-   * @throws IOException
-   */
-  public static ListBuckets parse(String data) throws IOException {
-    return READER.readValue(data);
-  }
-
-  /**
-   * Returns a list of Buckets.
-   *
-   * @return Bucket list
-   */
-  public List<BucketInfo> getBuckets() {
-    return buckets;
-  }
-
-  /**
-   * Sets the list of buckets owned by this user.
-   *
-   * @param buckets - List of Buckets
-   */
-  public void setBuckets(List<BucketInfo> buckets) {
-    this.buckets = buckets;
-  }
-
-
-  /**
-   * Returns a JSON string of this object.
-   * After stripping out bytesUsed and keyCount
-   *
-   * @return String
-   */
-  public String toJsonString() throws IOException {
-    return WRITER.writeValueAsString(this);
-  }
-
-  /**
-   * Returns the Object as a Json String.
-   */
-  public String toDBString() throws IOException {
-    return JsonUtils.toJsonString(this);
-  }
-
-  /**
-   * Sorts the buckets based on bucketName.
-   * This is useful when we return the list of buckets
-   */
-  public void sort() {
-    Collections.sort(buckets);
-  }
-
-  /**
-   * Add a new bucket to the list of buckets.
-   * @param bucketInfo - bucket Info
-   */
-  public void addBucket(BucketInfo bucketInfo){
-    this.buckets.add(bucketInfo);
-  }
-
-  /**
-   * This class allows us to create custom filters
-   * for the Json serialization.
-   */
-  @JsonFilter(BUCKET_LIST)
-  class MixIn {
-
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListKeys.java
deleted file mode 100644
index 9dc77d2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListKeys.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.web.response;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.ListArgs;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonFilter;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.ser.FilterProvider;
-import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
-import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-import com.google.common.base.Preconditions;
-
-/**
- * This class the represents the list of keys (Objects) in a bucket.
- */
-public class ListKeys {
-  static final String OBJECT_LIST = "OBJECT_LIST_FILTER";
-
-  private static final ObjectReader READER =
-      new ObjectMapper().readerFor(ListKeys.class);
-  private static final ObjectWriter WRITER;
-
-  static {
-    ObjectMapper mapper = new ObjectMapper();
-    String[] ignorableFieldNames = {"dataFileName"};
-
-    FilterProvider filters = new SimpleFilterProvider()
-        .addFilter(OBJECT_LIST, SimpleBeanPropertyFilter
-            .serializeAllExcept(ignorableFieldNames));
-    mapper.setVisibility(PropertyAccessor.FIELD,
-        JsonAutoDetect.Visibility.ANY);
-    mapper.addMixIn(Object.class, MixIn.class);
-
-    mapper.setFilterProvider(filters);
-    WRITER = mapper.writerWithDefaultPrettyPrinter();
-  }
-
-  private String name;
-  private String prefix;
-  private long maxKeys;
-  private boolean truncated;
-  private List<KeyInfo> keyList;
-
-  /**
-   * Default constructor needed for json serialization.
-   */
-  public ListKeys() {
-    this.keyList = new LinkedList<>();
-  }
-
-  /**
-   * Constructor for ListKeys.
-   *
-   * @param args      ListArgs
-   * @param truncated is truncated
-   */
-  public ListKeys(ListArgs args, boolean truncated) {
-    Preconditions.checkState(args.getArgs() instanceof  BucketArgs);
-    this.name = ((BucketArgs) args.getArgs()).getBucketName();
-    this.prefix = args.getPrefix();
-    this.maxKeys = args.getMaxKeys();
-    this.truncated = truncated;
-  }
-
-  /**
-   * Converts a Json string to POJO.
-   * @param jsonString - json string.
-   * @return ListObject
-   * @throws IOException - Json conversion error.
-   */
-  public static ListKeys parse(String jsonString) throws IOException {
-    return READER.readValue(jsonString);
-  }
-
-  /**
-   * Returns a list of Objects.
-   *
-   * @return List of KeyInfo Objects.
-   */
-  public List<KeyInfo> getKeyList() {
-    return keyList;
-  }
-
-  /**
-   * Sets the list of Objects.
-   *
-   * @param objectList - List of Keys
-   */
-  public void setKeyList(List<KeyInfo> objectList) {
-    this.keyList = objectList;
-  }
-
-  /**
-   * Gets the Max Key Count.
-   *
-   * @return long
-   */
-  public long getMaxKeys() {
-    return maxKeys;
-  }
-
-  /**
-   * Gets bucket Name.
-   *
-   * @return String
-   */
-  public String getName() {
-    return name;
-  }
-
-  /**
-   * Gets Prefix.
-   *
-   * @return String
-   */
-  public String getPrefix() {
-    return prefix;
-  }
-
-  /**
-   * Gets truncated Status.
-   *
-   * @return Boolean
-   */
-  public boolean isTruncated() {
-    return truncated;
-  }
-
-  /**
-   * Sets the value of truncated.
-   *
-   * @param value - Boolean
-   */
-  public void setTruncated(boolean value) {
-    this.truncated = value;
-  }
-
-  /**
-   * Returns a JSON string of this object. After stripping out bytesUsed and
-   * keyCount.
-   *
-   * @return String
-   * @throws  IOException - On json Errors.
-   */
-  public String toJsonString() throws IOException {
-    return WRITER.writeValueAsString(this);
-  }
-
-  /**
-   * Returns the Object as a Json String.
-   *
-   * @return String
-   * @throws IOException - on json errors.
-   */
-  public String toDBString() throws IOException {
-    return JsonUtils.toJsonString(this);
-  }
-
-  /**
-   * Sorts the keys based on name and version. This is useful when we return the
-   * list of keys.
-   */
-  public void sort() {
-    Collections.sort(keyList);
-  }
-
-  /**
-   * Add a new key to the list of keys.
-   * @param keyInfo - key Info
-   */
-  public void addKey(KeyInfo keyInfo){
-    this.keyList.add(keyInfo);
-  }
-
-  /**
-   * This class allows us to create custom filters for the Json serialization.
-   */
-  @JsonFilter(OBJECT_LIST)
-  class MixIn {
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListVolumes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListVolumes.java
deleted file mode 100644
index b918349..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListVolumes.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.web.response;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonFilter;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.ser.FilterProvider;
-import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
-import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-
-/**
- * List Volume Class is the class that is returned in JSON format to
- * users when they call ListVolumes.
- */
-@InterfaceAudience.Private
-public class ListVolumes {
-  private List<VolumeInfo> volumes;
-
-  static final String VOLUME_LIST = "VOLUME_LIST_FILTER";
-  private static final ObjectReader READER =
-      new ObjectMapper().readerFor(ListVolumes.class);
-  private static final ObjectWriter WRITER;
-
-  static {
-    ObjectMapper mapper = new ObjectMapper();
-    String[] ignorableFieldNames = {"bytesUsed", "bucketCount"};
-
-    FilterProvider filters = new SimpleFilterProvider()
-        .addFilter(VOLUME_LIST, SimpleBeanPropertyFilter
-            .serializeAllExcept(ignorableFieldNames));
-    mapper.setVisibility(PropertyAccessor.FIELD,
-        JsonAutoDetect.Visibility.ANY);
-    mapper.addMixIn(Object.class, MixIn.class);
-
-    mapper.setFilterProvider(filters);
-    WRITER = mapper.writerWithDefaultPrettyPrinter();
-  }
-
-  /**
-   * Used for json filtering.
-   */
-  @JsonFilter(VOLUME_LIST)
-  class MixIn {
-  }
-
-  /**
-   * Constructs ListVolume objects.
-   */
-  public ListVolumes() {
-    this.volumes = new LinkedList<VolumeInfo>();
-  }
-
-  /**
-   * Gets the list of volumes.
-   *
-   * @return List of VolumeInfo Objects
-   */
-  public List<VolumeInfo> getVolumes() {
-    return volumes;
-  }
-
-
-  /**
-   * Sets volume info.
-   *
-   * @param volumes - List of Volumes
-   */
-  public void setVolumes(List<VolumeInfo> volumes) {
-    this.volumes = volumes;
-  }
-
-  /**
-   * Returns a JSON string of this object.
-   * After stripping out bytesUsed and bucketCount
-   *
-   * @return String
-   */
-  public String toJsonString() throws IOException {
-    return WRITER.writeValueAsString(this);
-  }
-
-  /**
-   * When we serialize a volumeInfo to our database
-   * we will use all fields. However the toJsonString
-   * will strip out bytesUsed and bucketCount from the
-   * volume Info
-   *
-   * @return Json String
-   *
-   * @throws IOException
-   */
-  public String toDBString() throws IOException {
-    return JsonUtils.toJsonString(this);
-  }
-
-  /**
-   * Parses a String to return ListVolumes object.
-   *
-   * @param data - Json String
-   *
-   * @return - ListVolumes
-   *
-   * @throws IOException
-   */
-  public static ListVolumes parse(String data) throws IOException {
-    return READER.readValue(data);
-  }
-
-  /**
-   * Adds a new volume info to the List.
-   *
-   * @param info - VolumeInfo
-   */
-  public void addVolume(VolumeInfo info) {
-    this.volumes.add(info);
-  }
-
-  /**
-   * Sorts the volume names based on volume name.
-   * This is useful when we return the list of volume names
-   */
-  public void sort() {
-    Collections.sort(volumes);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/VolumeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/VolumeInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/VolumeInfo.java
deleted file mode 100644
index 112b27e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/VolumeInfo.java
+++ /dev/null
@@ -1,308 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.response;
-
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.web.request.OzoneQuota;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonFilter;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.ser.FilterProvider;
-import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
-import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-
-/**
- * VolumeInfo Class is the Java class that represents
- * Json when VolumeInfo Call is made.
- */
-@InterfaceAudience.Private
-public class VolumeInfo implements Comparable<VolumeInfo> {
-
-  static final String VOLUME_INFO = "VOLUME_INFO_FILTER";
-  private static final ObjectReader READER =
-      new ObjectMapper().readerFor(VolumeInfo.class);
-  private static final ObjectWriter WRITER;
-
-  static {
-    ObjectMapper mapper = new ObjectMapper();
-    String[] ignorableFieldNames = {"bytesUsed", "bucketCount"};
-
-    FilterProvider filters = new SimpleFilterProvider()
-        .addFilter(VOLUME_INFO, SimpleBeanPropertyFilter
-            .serializeAllExcept(ignorableFieldNames));
-    mapper.setVisibility(PropertyAccessor.FIELD,
-        JsonAutoDetect.Visibility.ANY);
-    mapper.addMixIn(Object.class, MixIn.class);
-
-    mapper.setFilterProvider(filters);
-    WRITER = mapper.writerWithDefaultPrettyPrinter();
-  }
-
-  /**
-   * Custom Json Filter Class.
-   */
-  @JsonFilter(VOLUME_INFO)
-  class MixIn {
-  }
-  private VolumeOwner owner;
-  private OzoneQuota quota;
-  private String volumeName;
-  private String createdOn;
-  private String createdBy;
-
-  private long bytesUsed;
-  private long bucketCount;
-
-
-  /**
-   * Constructor for VolumeInfo.
-   *
-   * @param volumeName - Name of the Volume
-   * @param createdOn _ Date String
-   * @param createdBy - Person who created it
-   */
-  public VolumeInfo(String volumeName, String createdOn, String createdBy) {
-    this.createdOn = createdOn;
-    this.volumeName = volumeName;
-    this.createdBy = createdBy;
-  }
-
-  /**
-   * Constructor for VolumeInfo.
-   */
-  public VolumeInfo() {
-  }
-
-  /**
-   * Returns the name of the person who created this volume.
-   *
-   * @return Name of Admin who created this
-   */
-  public String getCreatedBy() {
-    return createdBy;
-  }
-
-  /**
-   * Sets the user name of the person who created this volume.
-   *
-   * @param createdBy - UserName
-   */
-  public void setCreatedBy(String createdBy) {
-    this.createdBy = createdBy;
-  }
-
-  /**
-   * Gets the date on which this volume was created.
-   *
-   * @return - Date String
-   */
-  public String getCreatedOn() {
-    return createdOn;
-  }
-
-  /**
-   * Sets the date string.
-   *
-   * @param createdOn - Date String
-   */
-  public void setCreatedOn(String createdOn) {
-    this.createdOn = createdOn;
-  }
-
-  /**
-   * Returns the owner info.
-   *
-   * @return - OwnerInfo
-   */
-  public VolumeOwner getOwner() {
-    return owner;
-  }
-
-  /**
-   * Sets the owner.
-   *
-   * @param owner - OwnerInfo
-   */
-  public void setOwner(VolumeOwner owner) {
-    this.owner = owner;
-  }
-
-  /**
-   * Returns the quota information on a volume.
-   *
-   * @return Quota
-   */
-  public OzoneQuota getQuota() {
-    return quota;
-  }
-
-  /**
-   * Sets the quota info.
-   *
-   * @param quota - Quota Info
-   */
-  public void setQuota(OzoneQuota quota) {
-    this.quota = quota;
-  }
-
-  /**
-   * gets the volume name.
-   *
-   * @return - Volume Name
-   */
-  public String getVolumeName() {
-    return volumeName;
-  }
-
-  /**
-   * Sets the volume name.
-   *
-   * @param volumeName - Volume Name
-   */
-  public void setVolumeName(String volumeName) {
-    this.volumeName = volumeName;
-  }
-
-  /**
-   * Returns a JSON string of this object.
-   * After stripping out bytesUsed and bucketCount
-   *
-   * @return String - json string
-   * @throws IOException
-   */
-  public String toJsonString() throws IOException {
-    return WRITER.writeValueAsString(this);
-  }
-
-  /**
-   * When we serialize a volumeInfo to our database
-   * we will use all fields. However the toJsonString
-   * will strip out bytesUsed and bucketCount from the
-   * volume Info
-   *
-   * @return Json String
-   *
-   * @throws IOException
-   */
-  public String toDBString() throws IOException {
-    return JsonUtils.toJsonString(this);
-  }
-
-
-  /**
-   * Comparable Interface.
-   * @param o VolumeInfo Object.
-   * @return Result of comparison
-   */
-  @Override
-  public int compareTo(VolumeInfo o) {
-    return this.volumeName.compareTo(o.getVolumeName());
-  }
-
-  /**
-   * Gets the number of bytesUsed by this volume.
-   *
-   * @return long - Bytes used
-   */
-  public long getBytesUsed() {
-    return bytesUsed;
-  }
-
-  /**
-   * Sets number of bytesUsed by this volume.
-   *
-   * @param bytesUsed - Number of bytesUsed
-   */
-  public void setBytesUsed(long bytesUsed) {
-    this.bytesUsed = bytesUsed;
-  }
-
-  /**
-   * Returns VolumeInfo class from json string.
-   *
-   * @param data - Json String
-   *
-   * @return VolumeInfo
-   *
-   * @throws IOException
-   */
-  public static VolumeInfo parse(String data) throws IOException {
-    return READER.readValue(data);
-  }
-
-  /**
-   * Indicates whether some other object is "equal to" this one.
-   *
-   * @param obj the reference object with which to compare.
-   *
-   * @return {@code true} if this object is the same as the obj
-   * argument; {@code false} otherwise.
-   */
-  @Override
-  public boolean equals(Object obj) {
-    if (obj == null) {
-      return false;
-    }
-    if (getClass() != obj.getClass()) {
-      return false;
-    }
-    VolumeInfo otherInfo = (VolumeInfo) obj;
-    return otherInfo.getVolumeName().equals(this.getVolumeName());
-  }
-
-  /**
-   * Returns a hash code value for the object. This method is
-   * supported for the benefit of hash tables such as those provided by
-   * HashMap.
-   * @return a hash code value for this object.
-   *
-   * @see Object#equals(Object)
-   * @see System#identityHashCode
-   */
-  @Override
-  public int hashCode() {
-    return getVolumeName().hashCode();
-  }
-
-  /**
-   * Total number of buckets under this volume.
-   *
-   * @return - bucketCount
-   */
-  public long getBucketCount() {
-    return bucketCount;
-  }
-
-  /**
-   * Sets the buckets count.
-   *
-   * @param bucketCount - Bucket Count
-   */
-  public void setBucketCount(long bucketCount) {
-    this.bucketCount = bucketCount;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/VolumeOwner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/VolumeOwner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/VolumeOwner.java
deleted file mode 100644
index afb0460..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/VolumeOwner.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.response;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * Volume Owner represents the owner of a volume.
- *
- * This is a class instead of a string since we might need to extend this class
- * to support other forms of authentication.
- */
-@InterfaceAudience.Private
-public class VolumeOwner {
-  @JsonInclude(JsonInclude.Include.NON_NULL)
-  private String name;
-
-  /**
-   * Constructor for VolumeOwner.
-   *
-   * @param name - name of the User
-   */
-  public VolumeOwner(String name) {
-    this.name = name;
-  }
-
-  /**
-   * Constructs Volume Owner.
-   */
-  public VolumeOwner() {
-    name = null;
-  }
-
-  /**
-   * Returns the user name.
-   *
-   * @return Name
-   */
-  public String getName() {
-    return name;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/package-info.java
deleted file mode 100644
index 3bf66c8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Netty-based HTTP server implementation for Ozone.
- */
-package org.apache.hadoop.ozone.web.response;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
deleted file mode 100644
index 1830c71..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
+++ /dev/null
@@ -1,539 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.storage;
-
-import com.google.common.base.Strings;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.client.io.LengthInputStream;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.ksm.protocolPB
-    .KeySpaceManagerProtocolClientSideTranslatorPB;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.OzoneConsts.Versioning;
-import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream;
-import org.apache.hadoop.ozone.client.io.ChunkGroupOutputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
-import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.web.request.OzoneQuota;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.protocolPB
-    .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.KeyArgs;
-import org.apache.hadoop.ozone.web.handlers.ListArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.response.ListVolumes;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-import org.apache.hadoop.ozone.web.response.VolumeOwner;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.ozone.web.response.ListBuckets;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-import org.apache.hadoop.ozone.web.response.ListKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.List;
-
-/**
- * A {@link StorageHandler} implementation that distributes object storage
- * across the nodes of an HDFS cluster.
- */
-public final class DistributedStorageHandler implements StorageHandler {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(DistributedStorageHandler.class);
-
-  private final StorageContainerLocationProtocolClientSideTranslatorPB
-      storageContainerLocationClient;
-  private final KeySpaceManagerProtocolClientSideTranslatorPB
-      keySpaceManagerClient;
-  private final XceiverClientManager xceiverClientManager;
-  private final OzoneAcl.OzoneACLRights userRights;
-  private final OzoneAcl.OzoneACLRights groupRights;
-  private int chunkSize;
-  private final boolean useRatis;
-  private final OzoneProtos.ReplicationType type;
-  private final OzoneProtos.ReplicationFactor factor;
-
-  /**
-   * Creates a new DistributedStorageHandler.
-   *
-   * @param conf configuration
-   * @param storageContainerLocation StorageContainerLocationProtocol proxy
-   * @param keySpaceManagerClient KeySpaceManager proxy
-   */
-  public DistributedStorageHandler(OzoneConfiguration conf,
-      StorageContainerLocationProtocolClientSideTranslatorPB
-          storageContainerLocation,
-      KeySpaceManagerProtocolClientSideTranslatorPB
-          keySpaceManagerClient) {
-    this.keySpaceManagerClient = keySpaceManagerClient;
-    this.storageContainerLocationClient = storageContainerLocation;
-    this.xceiverClientManager = new XceiverClientManager(conf);
-    this.useRatis = conf.getBoolean(
-        ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY,
-        ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT);
-
-    if(useRatis) {
-      type = OzoneProtos.ReplicationType.RATIS;
-      factor = OzoneProtos.ReplicationFactor.THREE;
-    } else {
-      type = OzoneProtos.ReplicationType.STAND_ALONE;
-      factor = OzoneProtos.ReplicationFactor.ONE;
-    }
-
-    chunkSize = conf.getInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
-        ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
-    userRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_USER_RIGHTS,
-        KSMConfigKeys.OZONE_KSM_USER_RIGHTS_DEFAULT);
-    groupRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_GROUP_RIGHTS,
-        KSMConfigKeys.OZONE_KSM_GROUP_RIGHTS_DEFAULT);
-    if(chunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
-      LOG.warn("The chunk size ({}) is not allowed to be more than"
-              + " the maximum size ({}),"
-              + " resetting to the maximum size.",
-          chunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE);
-      chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
-    }
-  }
-
-  @Override
-  public void createVolume(VolumeArgs args) throws IOException, OzoneException {
-    long quota = args.getQuota() == null ?
-        OzoneConsts.MAX_QUOTA_IN_BYTES : args.getQuota().sizeInBytes();
-    OzoneAcl userAcl =
-        new OzoneAcl(OzoneAcl.OzoneACLType.USER,
-            args.getUserName(), userRights);
-    KsmVolumeArgs.Builder builder = KsmVolumeArgs.newBuilder();
-    builder.setAdminName(args.getAdminName())
-        .setOwnerName(args.getUserName())
-        .setVolume(args.getVolumeName())
-        .setQuotaInBytes(quota)
-        .addOzoneAcls(KSMPBHelper.convertOzoneAcl(userAcl));
-    if (args.getGroups() != null) {
-      for (String group : args.getGroups()) {
-        OzoneAcl groupAcl =
-            new OzoneAcl(OzoneAcl.OzoneACLType.GROUP, group, groupRights);
-        builder.addOzoneAcls(KSMPBHelper.convertOzoneAcl(groupAcl));
-      }
-    }
-    keySpaceManagerClient.createVolume(builder.build());
-  }
-
-  @Override
-  public void setVolumeOwner(VolumeArgs args) throws
-      IOException, OzoneException {
-    keySpaceManagerClient.setOwner(args.getVolumeName(), args.getUserName());
-  }
-
-  @Override
-  public void setVolumeQuota(VolumeArgs args, boolean remove)
-      throws IOException, OzoneException {
-    long quota = remove ? OzoneConsts.MAX_QUOTA_IN_BYTES :
-        args.getQuota().sizeInBytes();
-    keySpaceManagerClient.setQuota(args.getVolumeName(), quota);
-  }
-
-  @Override
-  public boolean checkVolumeAccess(String volume, OzoneAcl acl)
-      throws IOException, OzoneException {
-    return keySpaceManagerClient
-        .checkVolumeAccess(volume, KSMPBHelper.convertOzoneAcl(acl));
-  }
-
-  @Override
-  public ListVolumes listVolumes(ListArgs args)
-      throws IOException, OzoneException {
-    int maxNumOfKeys = args.getMaxKeys();
-    if (maxNumOfKeys <= 0 ||
-        maxNumOfKeys > OzoneConsts.MAX_LISTVOLUMES_SIZE) {
-      throw new IllegalArgumentException(
-          String.format("Illegal max number of keys specified,"
-                  + " the value must be in range (0, %d], actual : %d.",
-              OzoneConsts.MAX_LISTVOLUMES_SIZE, maxNumOfKeys));
-    }
-
-    List<KsmVolumeArgs> listResult;
-    if (args.isRootScan()) {
-      listResult = keySpaceManagerClient.listAllVolumes(args.getPrefix(),
-          args.getPrevKey(), args.getMaxKeys());
-    } else {
-      UserArgs userArgs = args.getArgs();
-      if (userArgs == null || userArgs.getUserName() == null) {
-        throw new IllegalArgumentException("Illegal argument,"
-            + " missing user argument.");
-      }
-      listResult = keySpaceManagerClient.listVolumeByUser(
-          args.getArgs().getUserName(), args.getPrefix(), args.getPrevKey(),
-          args.getMaxKeys());
-    }
-
-    // TODO Add missing fields createdBy, bucketCount and bytesUsed
-    ListVolumes result = new ListVolumes();
-    for (KsmVolumeArgs volumeArgs : listResult) {
-      VolumeInfo info = new VolumeInfo();
-      KeySpaceManagerProtocolProtos.VolumeInfo
-          infoProto = volumeArgs.getProtobuf();
-      info.setOwner(new VolumeOwner(infoProto.getOwnerName()));
-      info.setQuota(OzoneQuota.getOzoneQuota(infoProto.getQuotaInBytes()));
-      info.setVolumeName(infoProto.getVolume());
-      info.setCreatedOn(OzoneUtils.formatTime(infoProto.getCreationTime()));
-      result.addVolume(info);
-    }
-
-    return result;
-  }
-
-  @Override
-  public void deleteVolume(VolumeArgs args)
-      throws IOException, OzoneException {
-    keySpaceManagerClient.deleteVolume(args.getVolumeName());
-  }
-
-  @Override
-  public VolumeInfo getVolumeInfo(VolumeArgs args)
-      throws IOException, OzoneException {
-    KsmVolumeArgs volumeArgs =
-        keySpaceManagerClient.getVolumeInfo(args.getVolumeName());
-    //TODO: add support for createdOn and other fields in getVolumeInfo
-    VolumeInfo volInfo =
-        new VolumeInfo(volumeArgs.getVolume(), null,
-            volumeArgs.getAdminName());
-    volInfo.setOwner(new VolumeOwner(volumeArgs.getOwnerName()));
-    volInfo.setQuota(OzoneQuota.getOzoneQuota(volumeArgs.getQuotaInBytes()));
-    volInfo.setCreatedOn(OzoneUtils.formatTime(volumeArgs.getCreationTime()));
-    return volInfo;
-  }
-
-  @Override
-  public void createBucket(final BucketArgs args)
-      throws IOException, OzoneException {
-    KsmBucketInfo.Builder builder = KsmBucketInfo.newBuilder();
-    builder.setVolumeName(args.getVolumeName())
-        .setBucketName(args.getBucketName());
-    if(args.getAddAcls() != null) {
-      builder.setAcls(args.getAddAcls());
-    }
-    if(args.getStorageType() != null) {
-      builder.setStorageType(args.getStorageType());
-    }
-    if(args.getVersioning() != null) {
-      builder.setIsVersionEnabled(getBucketVersioningProtobuf(
-          args.getVersioning()));
-    }
-    keySpaceManagerClient.createBucket(builder.build());
-  }
-
-  /**
-   * Converts OzoneConts.Versioning enum to boolean.
-   *
-   * @param version
-   * @return corresponding boolean value
-   */
-  private boolean getBucketVersioningProtobuf(
-      Versioning version) {
-    if(version != null) {
-      switch(version) {
-      case ENABLED:
-        return true;
-      case NOT_DEFINED:
-      case DISABLED:
-      default:
-        return false;
-      }
-    }
-    return false;
-  }
-
-  @Override
-  public void setBucketAcls(BucketArgs args)
-      throws IOException, OzoneException {
-    List<OzoneAcl> removeAcls = args.getRemoveAcls();
-    List<OzoneAcl> addAcls = args.getAddAcls();
-    if(removeAcls != null || addAcls != null) {
-      KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
-      builder.setVolumeName(args.getVolumeName())
-          .setBucketName(args.getBucketName());
-      if(removeAcls != null && !removeAcls.isEmpty()) {
-        builder.setRemoveAcls(args.getRemoveAcls());
-      }
-      if(addAcls != null && !addAcls.isEmpty()) {
-        builder.setAddAcls(args.getAddAcls());
-      }
-      keySpaceManagerClient.setBucketProperty(builder.build());
-    }
-  }
-
-  @Override
-  public void setBucketVersioning(BucketArgs args)
-      throws IOException, OzoneException {
-    KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
-    builder.setVolumeName(args.getVolumeName())
-        .setBucketName(args.getBucketName())
-        .setIsVersionEnabled(getBucketVersioningProtobuf(
-            args.getVersioning()));
-    keySpaceManagerClient.setBucketProperty(builder.build());
-  }
-
-  @Override
-  public void setBucketStorageClass(BucketArgs args)
-      throws IOException, OzoneException {
-    KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
-    builder.setVolumeName(args.getVolumeName())
-        .setBucketName(args.getBucketName())
-        .setStorageType(args.getStorageType());
-    keySpaceManagerClient.setBucketProperty(builder.build());
-  }
-
-  @Override
-  public void deleteBucket(BucketArgs args)
-      throws IOException, OzoneException {
-    keySpaceManagerClient.deleteBucket(args.getVolumeName(),
-        args.getBucketName());
-  }
-
-  @Override
-  public void checkBucketAccess(BucketArgs args)
-      throws IOException, OzoneException {
-    throw new UnsupportedOperationException(
-        "checkBucketAccess not implemented");
-  }
-
-  @Override
-  public ListBuckets listBuckets(ListArgs args)
-      throws IOException, OzoneException {
-    ListBuckets result = new ListBuckets();
-    UserArgs userArgs = args.getArgs();
-    if (userArgs instanceof VolumeArgs) {
-      VolumeArgs va = (VolumeArgs) userArgs;
-      if (Strings.isNullOrEmpty(va.getVolumeName())) {
-        throw new IllegalArgumentException("Illegal argument,"
-            + " volume name cannot be null or empty.");
-      }
-
-      int maxNumOfKeys = args.getMaxKeys();
-      if (maxNumOfKeys <= 0 ||
-          maxNumOfKeys > OzoneConsts.MAX_LISTBUCKETS_SIZE) {
-        throw new IllegalArgumentException(
-            String.format("Illegal max number of keys specified,"
-                + " the value must be in range (0, %d], actual : %d.",
-                OzoneConsts.MAX_LISTBUCKETS_SIZE, maxNumOfKeys));
-      }
-
-      List<KsmBucketInfo> buckets =
-          keySpaceManagerClient.listBuckets(va.getVolumeName(),
-              args.getPrevKey(), args.getPrefix(), args.getMaxKeys());
-
-      // Convert the result for the web layer.
-      for (KsmBucketInfo bucketInfo : buckets) {
-        BucketInfo bk = new BucketInfo();
-        bk.setVolumeName(bucketInfo.getVolumeName());
-        bk.setBucketName(bucketInfo.getBucketName());
-        bk.setStorageType(bucketInfo.getStorageType());
-        bk.setAcls(bucketInfo.getAcls());
-        bk.setCreatedOn(OzoneUtils.formatTime(bucketInfo.getCreationTime()));
-        result.addBucket(bk);
-      }
-      return result;
-    } else {
-      throw new IllegalArgumentException("Illegal argument provided,"
-          + " expecting VolumeArgs type but met "
-          + userArgs.getClass().getSimpleName());
-    }
-  }
-
-  @Override
-  public BucketInfo getBucketInfo(BucketArgs args)
-      throws IOException {
-    String volumeName = args.getVolumeName();
-    String bucketName = args.getBucketName();
-    KsmBucketInfo ksmBucketInfo = keySpaceManagerClient.getBucketInfo(
-        volumeName, bucketName);
-    BucketInfo bucketInfo = new BucketInfo(ksmBucketInfo.getVolumeName(),
-        ksmBucketInfo.getBucketName());
-    if(ksmBucketInfo.getIsVersionEnabled()) {
-      bucketInfo.setVersioning(Versioning.ENABLED);
-    } else {
-      bucketInfo.setVersioning(Versioning.DISABLED);
-    }
-    bucketInfo.setStorageType(ksmBucketInfo.getStorageType());
-    bucketInfo.setAcls(ksmBucketInfo.getAcls());
-    bucketInfo.setCreatedOn(
-        OzoneUtils.formatTime(ksmBucketInfo.getCreationTime()));
-    return bucketInfo;
-  }
-
-  @Override
-  public OutputStream newKeyWriter(KeyArgs args) throws IOException,
-      OzoneException {
-    KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
-        .setVolumeName(args.getVolumeName())
-        .setBucketName(args.getBucketName())
-        .setKeyName(args.getKeyName())
-        .setDataSize(args.getSize())
-        .setType(xceiverClientManager.getType())
-        .setFactor(xceiverClientManager.getFactor())
-        .build();
-    // contact KSM to allocate a block for key.
-    OpenKeySession openKey = keySpaceManagerClient.openKey(keyArgs);
-    ChunkGroupOutputStream groupOutputStream =
-        new ChunkGroupOutputStream.Builder()
-            .setHandler(openKey)
-            .setXceiverClientManager(xceiverClientManager)
-            .setScmClient(storageContainerLocationClient)
-            .setKsmClient(keySpaceManagerClient)
-            .setChunkSize(chunkSize)
-            .setRequestID(args.getRequestID())
-            .setType(xceiverClientManager.getType())
-            .setFactor(xceiverClientManager.getFactor())
-            .build();
-    groupOutputStream.addPreallocateBlocks(
-        openKey.getKeyInfo().getLatestVersionLocations(),
-        openKey.getOpenVersion());
-    return new OzoneOutputStream(groupOutputStream);
-  }
-
-  @Override
-  public void commitKey(KeyArgs args, OutputStream stream) throws
-      IOException, OzoneException {
-    stream.close();
-  }
-
-  @Override
-  public LengthInputStream newKeyReader(KeyArgs args) throws IOException,
-      OzoneException {
-    KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
-        .setVolumeName(args.getVolumeName())
-        .setBucketName(args.getBucketName())
-        .setKeyName(args.getKeyName())
-        .setDataSize(args.getSize())
-        .build();
-    KsmKeyInfo keyInfo = keySpaceManagerClient.lookupKey(keyArgs);
-    return ChunkGroupInputStream.getFromKsmKeyInfo(
-        keyInfo, xceiverClientManager, storageContainerLocationClient,
-        args.getRequestID());
-  }
-
-  @Override
-  public void deleteKey(KeyArgs args) throws IOException, OzoneException {
-    KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
-        .setVolumeName(args.getVolumeName())
-        .setBucketName(args.getBucketName())
-        .setKeyName(args.getKeyName())
-        .build();
-    keySpaceManagerClient.deleteKey(keyArgs);
-  }
-
-  @Override
-  public KeyInfo getKeyInfo(KeyArgs args) throws IOException, OzoneException {
-    KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
-        .setVolumeName(args.getVolumeName())
-        .setBucketName(args.getBucketName())
-        .setKeyName(args.getKeyName())
-        .build();
-
-    KsmKeyInfo ksmKeyInfo = keySpaceManagerClient.lookupKey(keyArgs);
-    KeyInfo keyInfo = new KeyInfo();
-    keyInfo.setVersion(0);
-    keyInfo.setKeyName(ksmKeyInfo.getKeyName());
-    keyInfo.setSize(ksmKeyInfo.getDataSize());
-    keyInfo.setCreatedOn(
-        OzoneUtils.formatTime(ksmKeyInfo.getCreationTime()));
-    keyInfo.setModifiedOn(
-        OzoneUtils.formatTime(ksmKeyInfo.getModificationTime()));
-    return keyInfo;
-  }
-
-  @Override
-  public ListKeys listKeys(ListArgs args) throws IOException, OzoneException {
-    ListKeys result = new ListKeys();
-    UserArgs userArgs = args.getArgs();
-    if (userArgs instanceof BucketArgs) {
-      BucketArgs bucketArgs = (BucketArgs) userArgs;
-      if (Strings.isNullOrEmpty(bucketArgs.getVolumeName())) {
-        throw new IllegalArgumentException("Illegal argument,"
-            + " volume name cannot be null or empty.");
-      }
-
-      if (Strings.isNullOrEmpty(bucketArgs.getBucketName())) {
-        throw new IllegalArgumentException("Illegal argument,"
-            + " bucket name cannot be null or empty.");
-      }
-
-      int maxNumOfKeys = args.getMaxKeys();
-      if (maxNumOfKeys <= 0 ||
-          maxNumOfKeys > OzoneConsts.MAX_LISTKEYS_SIZE) {
-        throw new IllegalArgumentException(
-            String.format("Illegal max number of keys specified,"
-                + " the value must be in range (0, %d], actual : %d.",
-                OzoneConsts.MAX_LISTKEYS_SIZE, maxNumOfKeys));
-      }
-
-      List<KsmKeyInfo> keys=
-          keySpaceManagerClient.listKeys(bucketArgs.getVolumeName(),
-              bucketArgs.getBucketName(),
-              args.getPrevKey(), args.getPrefix(), args.getMaxKeys());
-
-      // Convert the result for the web layer.
-      for (KsmKeyInfo info : keys) {
-        KeyInfo tempInfo = new KeyInfo();
-        tempInfo.setVersion(0);
-        tempInfo.setKeyName(info.getKeyName());
-        tempInfo.setSize(info.getDataSize());
-        tempInfo.setCreatedOn(
-            OzoneUtils.formatTime(info.getCreationTime()));
-        tempInfo.setModifiedOn(
-            OzoneUtils.formatTime(info.getModificationTime()));
-
-        result.addKey(tempInfo);
-      }
-      return result;
-    } else {
-      throw new IllegalArgumentException("Illegal argument provided,"
-          + " expecting BucketArgs type but met "
-          + userArgs.getClass().getSimpleName());
-    }
-  }
-
-  /**
-   * Closes DistributedStorageHandler.
-   */
-  @Override
-  public void close() {
-    IOUtils.cleanupWithLogger(LOG, xceiverClientManager);
-    IOUtils.cleanupWithLogger(LOG, keySpaceManagerClient);
-    IOUtils.cleanupWithLogger(LOG, storageContainerLocationClient);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/package-info.java
deleted file mode 100644
index f5499f5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/package-info.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Ozone storage handler implementation integrating REST interface front-end
- * with container data pipeline back-end.
- */
-@InterfaceAudience.Private
-package org.apache.hadoop.ozone.web.storage;
-
-import org.apache.hadoop.classification.InterfaceAudience;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/Simple.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/Simple.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/Simple.java
deleted file mode 100644
index 397c80f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/Simple.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.web.userauth;
-
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.interfaces.UserAuth;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.security.UserGroupInformation;
-
-import javax.ws.rs.core.HttpHeaders;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Simple is an UserAuth class that is used in the insecure
- * mode of ozone. This maps more or less to the simple user scheme in
- * HDFS.
- */
-@InterfaceAudience.Private
-public class Simple implements UserAuth {
-  /**
-   * Returns the x-ozone-user or the user on behalf of, This is
-   * used in volume creation path.
-   *
-   * @param userArgs - UserArgs
-   *
-   * @throws OzoneException
-   */
-  @Override
-  public String getOzoneUser(UserArgs userArgs) throws OzoneException {
-    assert userArgs != null : "userArgs cannot be null";
-
-    HttpHeaders headers = userArgs.getHeaders();
-    List<String> users = headers.getRequestHeader(Header.OZONE_USER);
-
-    if ((users == null) || (users.size() == 0)) {
-      return null;
-    }
-    if (users.size() > 1) {
-      throw ErrorTable.newError(ErrorTable.BAD_AUTHORIZATION, userArgs);
-    }
-    return users.get(0).toLowerCase().trim();
-  }
-
-  /**
-   * Returns the user name as a string from the URI and HTTP headers.
-   *
-   * @param userArgs - user args
-   *
-   * @throws OzoneException -- Allows the underlying system
-   * to throw, that error will get propagated to clients
-   */
-  @Override
-  public String getUser(UserArgs userArgs) throws OzoneException {
-    assert userArgs != null : "userArgs cannot be null";
-
-    HttpHeaders headers = userArgs.getHeaders();
-    List<String> users = headers.getRequestHeader(HttpHeaders.AUTHORIZATION);
-    if (users == null || users.size() > 1) {
-      throw ErrorTable.newError(ErrorTable.BAD_AUTHORIZATION, userArgs);
-    }
-
-    if (users.size() == 0) {
-      return null;
-    }
-
-    String user = users.get(0).trim();
-    if (user.startsWith(Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME)) {
-      user = user.replace(Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME, "");
-      return user.toLowerCase().trim();
-    } else {
-      throw ErrorTable.newError(ErrorTable.BAD_AUTHORIZATION, userArgs);
-    }
-  }
-
-
-  /**
-   * Returns true if a user is a Admin - {root and hdfs are treated as admins}.
-   *
-   * @param userArgs - User Args
-   *
-   * @throws OzoneException -- Allows the underlying system
-   * to throw, that error will get propagated to clients
-   */
-  @Override
-  public boolean isAdmin(UserArgs userArgs) throws OzoneException {
-    assert userArgs != null : "userArgs cannot be null";
-
-    String user;
-    String currentUser;
-    try {
-      user = getUser(userArgs);
-      currentUser = UserGroupInformation.getCurrentUser().getShortUserName();
-    } catch (IOException e) {
-      throw ErrorTable.newError(ErrorTable.BAD_AUTHORIZATION, userArgs);
-    }
-    return
-        (user.compareToIgnoreCase(OzoneConsts.OZONE_SIMPLE_ROOT_USER) == 0) ||
-            (user.compareToIgnoreCase(OzoneConsts.OZONE_SIMPLE_HDFS_USER) == 0)
-            || (user.compareToIgnoreCase(currentUser) == 0);
-  }
-
-  /**
-   * Returns true if the request is Anonymous.
-   *
-   * @param userArgs - user Args
-   *
-   * @throws OzoneException -- Allows the underlying system
-   * to throw, that error will get propagated to clients
-   */
-  @Override
-  public boolean isAnonymous(UserArgs userArgs) throws OzoneException {
-    assert userArgs != null : "userArgs cannot be null";
-
-    return getUser(userArgs) == null;
-  }
-
-  /**
-   * Returns true if the name is a recognizable user in the system.
-   *
-   * @param userName - Name of the user
-   * @param userArgs - user Args
-   *
-   * @throws OzoneException -- Allows the underlying system
-   * to throw, that error will get propagated to clients
-   */
-  @Override
-  public boolean isUser(String userName, UserArgs userArgs)
-      throws OzoneException {
-    // In the simple case, all non-null users names are users :)
-    return userName != null;
-  }
-
-  /**
-   * Returns all the Groups that user is a member of.
-   *
-   * @param userArgs - User Args
-   *
-   * @return String Array which contains 0 or more group names
-   *
-   * @throws OzoneException
-   */
-  @Override
-  public String[] getGroups(UserArgs userArgs) throws OzoneException {
-    // Not implemented
-    return null;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/package-info.java
deleted file mode 100644
index d498fc8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Netty-based HTTP server implementation for Ozone.
- */
-package org.apache.hadoop.ozone.web.userauth;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/JsonUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/JsonUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/JsonUtils.java
deleted file mode 100644
index 909873f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/JsonUtils.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.utils;
-
-import java.io.IOException;
-import java.util.List;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.type.CollectionType;
-
-/**
- * JSON Utility functions used in ozone.
- */
-public final class JsonUtils {
-
-  // Reuse ObjectMapper instance for improving performance.
-  // ObjectMapper is thread safe as long as we always configure instance
-  // before use.
-  private static final ObjectMapper MAPPER = new ObjectMapper();
-  private static final ObjectReader READER = MAPPER.readerFor(Object.class);
-  private static final ObjectWriter WRITTER =
-      MAPPER.writerWithDefaultPrettyPrinter();
-
-  private JsonUtils() {
-    // Never constructed
-  }
-
-  public static String toJsonStringWithDefaultPrettyPrinter(String jsonString)
-      throws IOException {
-    Object json = READER.readValue(jsonString);
-    return WRITTER.writeValueAsString(json);
-  }
-
-  public static String toJsonString(Object obj) throws IOException {
-    return MAPPER.writeValueAsString(obj);
-  }
-
-  /**
-   * Deserialize a list of elements from a given string,
-   * each element in the list is in the given type.
-   *
-   * @param str json string.
-   * @param elementType element type.
-   * @return List of elements of type elementType
-   * @throws IOException
-   */
-  public static List<?> toJsonList(String str, Class<?> elementType)
-      throws IOException {
-    CollectionType type = MAPPER.getTypeFactory()
-        .constructCollectionType(List.class, elementType);
-    return MAPPER.readValue(str, type);
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[09/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-1.6.4.min.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-1.6.4.min.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-1.6.4.min.js
deleted file mode 100644
index c4bf158..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-1.6.4.min.js
+++ /dev/null
@@ -1,332 +0,0 @@
-/*
- AngularJS v1.6.4
- (c) 2010-2017 Google, Inc. http://angularjs.org
- License: MIT
-*/
-(function(x){'use strict';function L(a,b){b=b||Error;return function(){var d=arguments[0],c;c="["+(a?a+":":"")+d+"] http://errors.angularjs.org/1.6.4/"+(a?a+"/":"")+d;for(d=1;d<arguments.length;d++){c=c+(1==d?"?":"&")+"p"+(d-1)+"=";var e=encodeURIComponent,f;f=arguments[d];f="function"==typeof f?f.toString().replace(/ \{[\s\S]*$/,""):"undefined"==typeof f?"undefined":"string"!=typeof f?JSON.stringify(f):f;c+=e(f)}return new b(c)}}function me(a){if(C(a))u(a.objectMaxDepth)&&(Ic.objectMaxDepth=Sb(a.objectMaxDepth)?
-a.objectMaxDepth:NaN);else return Ic}function Sb(a){return ba(a)&&0<a}function qa(a){if(null==a||Wa(a))return!1;if(H(a)||F(a)||B&&a instanceof B)return!0;var b="length"in Object(a)&&a.length;return ba(b)&&(0<=b&&(b-1 in a||a instanceof Array)||"function"===typeof a.item)}function q(a,b,d){var c,e;if(a)if(D(a))for(c in a)"prototype"!==c&&"length"!==c&&"name"!==c&&a.hasOwnProperty(c)&&b.call(d,a[c],c,a);else if(H(a)||qa(a)){var f="object"!==typeof a;c=0;for(e=a.length;c<e;c++)(f||c in a)&&b.call(d,
-a[c],c,a)}else if(a.forEach&&a.forEach!==q)a.forEach(b,d,a);else if(Jc(a))for(c in a)b.call(d,a[c],c,a);else if("function"===typeof a.hasOwnProperty)for(c in a)a.hasOwnProperty(c)&&b.call(d,a[c],c,a);else for(c in a)ua.call(a,c)&&b.call(d,a[c],c,a);return a}function Kc(a,b,d){for(var c=Object.keys(a).sort(),e=0;e<c.length;e++)b.call(d,a[c[e]],c[e]);return c}function Lc(a){return function(b,d){a(d,b)}}function ne(){return++qb}function Tb(a,b,d){for(var c=a.$$hashKey,e=0,f=b.length;e<f;++e){var g=b[e];
-if(C(g)||D(g))for(var h=Object.keys(g),k=0,l=h.length;k<l;k++){var m=h[k],n=g[m];d&&C(n)?ga(n)?a[m]=new Date(n.valueOf()):Xa(n)?a[m]=new RegExp(n):n.nodeName?a[m]=n.cloneNode(!0):Ub(n)?a[m]=n.clone():(C(a[m])||(a[m]=H(n)?[]:{}),Tb(a[m],[n],!0)):a[m]=n}}c?a.$$hashKey=c:delete a.$$hashKey;return a}function S(a){return Tb(a,va.call(arguments,1),!1)}function oe(a){return Tb(a,va.call(arguments,1),!0)}function Z(a){return parseInt(a,10)}function Vb(a,b){return S(Object.create(a),b)}function z(){}function Ya(a){return a}
-function la(a){return function(){return a}}function Wb(a){return D(a.toString)&&a.toString!==ma}function w(a){return"undefined"===typeof a}function u(a){return"undefined"!==typeof a}function C(a){return null!==a&&"object"===typeof a}function Jc(a){return null!==a&&"object"===typeof a&&!Mc(a)}function F(a){return"string"===typeof a}function ba(a){return"number"===typeof a}function ga(a){return"[object Date]"===ma.call(a)}function D(a){return"function"===typeof a}function Xa(a){return"[object RegExp]"===
-ma.call(a)}function Wa(a){return a&&a.window===a}function Za(a){return a&&a.$evalAsync&&a.$watch}function Ha(a){return"boolean"===typeof a}function pe(a){return a&&ba(a.length)&&qe.test(ma.call(a))}function Ub(a){return!(!a||!(a.nodeName||a.prop&&a.attr&&a.find))}function re(a){var b={};a=a.split(",");var d;for(d=0;d<a.length;d++)b[a[d]]=!0;return b}function wa(a){return Q(a.nodeName||a[0]&&a[0].nodeName)}function $a(a,b){var d=a.indexOf(b);0<=d&&a.splice(d,1);return d}function ra(a,b,d){function c(a,
-b,c){c--;if(0>c)return"...";var d=b.$$hashKey,f;if(H(a)){f=0;for(var g=a.length;f<g;f++)b.push(e(a[f],c))}else if(Jc(a))for(f in a)b[f]=e(a[f],c);else if(a&&"function"===typeof a.hasOwnProperty)for(f in a)a.hasOwnProperty(f)&&(b[f]=e(a[f],c));else for(f in a)ua.call(a,f)&&(b[f]=e(a[f],c));d?b.$$hashKey=d:delete b.$$hashKey;return b}function e(a,b){if(!C(a))return a;var d=g.indexOf(a);if(-1!==d)return h[d];if(Wa(a)||Za(a))throw Fa("cpws");var d=!1,e=f(a);void 0===e&&(e=H(a)?[]:Object.create(Mc(a)),
-d=!0);g.push(a);h.push(e);return d?c(a,e,b):e}function f(a){switch(ma.call(a)){case "[object Int8Array]":case "[object Int16Array]":case "[object Int32Array]":case "[object Float32Array]":case "[object Float64Array]":case "[object Uint8Array]":case "[object Uint8ClampedArray]":case "[object Uint16Array]":case "[object Uint32Array]":return new a.constructor(e(a.buffer),a.byteOffset,a.length);case "[object ArrayBuffer]":if(!a.slice){var b=new ArrayBuffer(a.byteLength);(new Uint8Array(b)).set(new Uint8Array(a));
-return b}return a.slice(0);case "[object Boolean]":case "[object Number]":case "[object String]":case "[object Date]":return new a.constructor(a.valueOf());case "[object RegExp]":return b=new RegExp(a.source,a.toString().match(/[^/]*$/)[0]),b.lastIndex=a.lastIndex,b;case "[object Blob]":return new a.constructor([a],{type:a.type})}if(D(a.cloneNode))return a.cloneNode(!0)}var g=[],h=[];d=Sb(d)?d:NaN;if(b){if(pe(b)||"[object ArrayBuffer]"===ma.call(b))throw Fa("cpta");if(a===b)throw Fa("cpi");H(b)?b.length=
-0:q(b,function(a,c){"$$hashKey"!==c&&delete b[c]});g.push(a);h.push(b);return c(a,b,d)}return e(a,d)}function Xb(a,b){return a===b||a!==a&&b!==b}function sa(a,b){if(a===b)return!0;if(null===a||null===b)return!1;if(a!==a&&b!==b)return!0;var d=typeof a,c;if(d===typeof b&&"object"===d)if(H(a)){if(!H(b))return!1;if((d=a.length)===b.length){for(c=0;c<d;c++)if(!sa(a[c],b[c]))return!1;return!0}}else{if(ga(a))return ga(b)?Xb(a.getTime(),b.getTime()):!1;if(Xa(a))return Xa(b)?a.toString()===b.toString():!1;
-if(Za(a)||Za(b)||Wa(a)||Wa(b)||H(b)||ga(b)||Xa(b))return!1;d=V();for(c in a)if("$"!==c.charAt(0)&&!D(a[c])){if(!sa(a[c],b[c]))return!1;d[c]=!0}for(c in b)if(!(c in d)&&"$"!==c.charAt(0)&&u(b[c])&&!D(b[c]))return!1;return!0}return!1}function ab(a,b,d){return a.concat(va.call(b,d))}function bb(a,b){var d=2<arguments.length?va.call(arguments,2):[];return!D(b)||b instanceof RegExp?b:d.length?function(){return arguments.length?b.apply(a,ab(d,arguments,0)):b.apply(a,d)}:function(){return arguments.length?
-b.apply(a,arguments):b.call(a)}}function Nc(a,b){var d=b;"string"===typeof a&&"$"===a.charAt(0)&&"$"===a.charAt(1)?d=void 0:Wa(b)?d="$WINDOW":b&&x.document===b?d="$DOCUMENT":Za(b)&&(d="$SCOPE");return d}function cb(a,b){if(!w(a))return ba(b)||(b=b?2:null),JSON.stringify(a,Nc,b)}function Oc(a){return F(a)?JSON.parse(a):a}function Pc(a,b){a=a.replace(se,"");var d=Date.parse("Jan 01, 1970 00:00:00 "+a)/6E4;return da(d)?b:d}function Yb(a,b,d){d=d?-1:1;var c=a.getTimezoneOffset();b=Pc(b,c);d*=b-c;a=new Date(a.getTime());
-a.setMinutes(a.getMinutes()+d);return a}function xa(a){a=B(a).clone();try{a.empty()}catch(b){}var d=B("<div>").append(a).html();try{return a[0].nodeType===Ia?Q(d):d.match(/^(<[^>]+>)/)[1].replace(/^<([\w-]+)/,function(a,b){return"<"+Q(b)})}catch(c){return Q(d)}}function Qc(a){try{return decodeURIComponent(a)}catch(b){}}function Rc(a){var b={};q((a||"").split("&"),function(a){var c,e,f;a&&(e=a=a.replace(/\+/g,"%20"),c=a.indexOf("="),-1!==c&&(e=a.substring(0,c),f=a.substring(c+1)),e=Qc(e),u(e)&&(f=
-u(f)?Qc(f):!0,ua.call(b,e)?H(b[e])?b[e].push(f):b[e]=[b[e],f]:b[e]=f))});return b}function Zb(a){var b=[];q(a,function(a,c){H(a)?q(a,function(a){b.push($(c,!0)+(!0===a?"":"="+$(a,!0)))}):b.push($(c,!0)+(!0===a?"":"="+$(a,!0)))});return b.length?b.join("&"):""}function db(a){return $(a,!0).replace(/%26/gi,"&").replace(/%3D/gi,"=").replace(/%2B/gi,"+")}function $(a,b){return encodeURIComponent(a).replace(/%40/gi,"@").replace(/%3A/gi,":").replace(/%24/g,"$").replace(/%2C/gi,",").replace(/%3B/gi,";").replace(/%20/g,
-b?"%20":"+")}function te(a,b){var d,c,e=Ja.length;for(c=0;c<e;++c)if(d=Ja[c]+b,F(d=a.getAttribute(d)))return d;return null}function ue(a,b){var d,c,e={};q(Ja,function(b){b+="app";!d&&a.hasAttribute&&a.hasAttribute(b)&&(d=a,c=a.getAttribute(b))});q(Ja,function(b){b+="app";var e;!d&&(e=a.querySelector("["+b.replace(":","\\:")+"]"))&&(d=e,c=e.getAttribute(b))});d&&(ve?(e.strictDi=null!==te(d,"strict-di"),b(d,c?[c]:[],e)):x.console.error("Angular: disabling automatic bootstrap. <script> protocol indicates an extension, document.location.href does not match."))}
-function Sc(a,b,d){C(d)||(d={});d=S({strictDi:!1},d);var c=function(){a=B(a);if(a.injector()){var c=a[0]===x.document?"document":xa(a);throw Fa("btstrpd",c.replace(/</,"&lt;").replace(/>/,"&gt;"));}b=b||[];b.unshift(["$provide",function(b){b.value("$rootElement",a)}]);d.debugInfoEnabled&&b.push(["$compileProvider",function(a){a.debugInfoEnabled(!0)}]);b.unshift("ng");c=eb(b,d.strictDi);c.invoke(["$rootScope","$rootElement","$compile","$injector",function(a,b,c,d){a.$apply(function(){b.data("$injector",
-d);c(b)(a)})}]);return c},e=/^NG_ENABLE_DEBUG_INFO!/,f=/^NG_DEFER_BOOTSTRAP!/;x&&e.test(x.name)&&(d.debugInfoEnabled=!0,x.name=x.name.replace(e,""));if(x&&!f.test(x.name))return c();x.name=x.name.replace(f,"");ea.resumeBootstrap=function(a){q(a,function(a){b.push(a)});return c()};D(ea.resumeDeferredBootstrap)&&ea.resumeDeferredBootstrap()}function we(){x.name="NG_ENABLE_DEBUG_INFO!"+x.name;x.location.reload()}function xe(a){a=ea.element(a).injector();if(!a)throw Fa("test");return a.get("$$testability")}
-function Tc(a,b){b=b||"_";return a.replace(ye,function(a,c){return(c?b:"")+a.toLowerCase()})}function ze(){var a;if(!Uc){var b=rb();(na=w(b)?x.jQuery:b?x[b]:void 0)&&na.fn.on?(B=na,S(na.fn,{scope:Na.scope,isolateScope:Na.isolateScope,controller:Na.controller,injector:Na.injector,inheritedData:Na.inheritedData}),a=na.cleanData,na.cleanData=function(b){for(var c,e=0,f;null!=(f=b[e]);e++)(c=na._data(f,"events"))&&c.$destroy&&na(f).triggerHandler("$destroy");a(b)}):B=W;ea.element=B;Uc=!0}}function fb(a,
-b,d){if(!a)throw Fa("areq",b||"?",d||"required");return a}function sb(a,b,d){d&&H(a)&&(a=a[a.length-1]);fb(D(a),b,"not a function, got "+(a&&"object"===typeof a?a.constructor.name||"Object":typeof a));return a}function Ka(a,b){if("hasOwnProperty"===a)throw Fa("badname",b);}function Vc(a,b,d){if(!b)return a;b=b.split(".");for(var c,e=a,f=b.length,g=0;g<f;g++)c=b[g],a&&(a=(e=a)[c]);return!d&&D(a)?bb(e,a):a}function tb(a){for(var b=a[0],d=a[a.length-1],c,e=1;b!==d&&(b=b.nextSibling);e++)if(c||a[e]!==
-b)c||(c=B(va.call(a,0,e))),c.push(b);return c||a}function V(){return Object.create(null)}function $b(a){if(null==a)return"";switch(typeof a){case "string":break;case "number":a=""+a;break;default:a=!Wb(a)||H(a)||ga(a)?cb(a):a.toString()}return a}function Ae(a){function b(a,b,c){return a[b]||(a[b]=c())}var d=L("$injector"),c=L("ng");a=b(a,"angular",Object);a.$$minErr=a.$$minErr||L;return b(a,"module",function(){var a={};return function(f,g,h){var k={};if("hasOwnProperty"===f)throw c("badname","module");
-g&&a.hasOwnProperty(f)&&(a[f]=null);return b(a,f,function(){function a(b,c,d,f){f||(f=e);return function(){f[d||"push"]([b,c,arguments]);return v}}function b(a,c,d){d||(d=e);return function(b,e){e&&D(e)&&(e.$$moduleName=f);d.push([a,c,arguments]);return v}}if(!g)throw d("nomod",f);var e=[],p=[],r=[],J=a("$injector","invoke","push",p),v={_invokeQueue:e,_configBlocks:p,_runBlocks:r,info:function(a){if(u(a)){if(!C(a))throw c("aobj","value");k=a;return this}return k},requires:g,name:f,provider:b("$provide",
-"provider"),factory:b("$provide","factory"),service:b("$provide","service"),value:a("$provide","value"),constant:a("$provide","constant","unshift"),decorator:b("$provide","decorator",p),animation:b("$animateProvider","register"),filter:b("$filterProvider","register"),controller:b("$controllerProvider","register"),directive:b("$compileProvider","directive"),component:b("$compileProvider","component"),config:J,run:function(a){r.push(a);return this}};h&&J(h);return v})}})}function pa(a,b){if(H(a)){b=
-b||[];for(var d=0,c=a.length;d<c;d++)b[d]=a[d]}else if(C(a))for(d in b=b||{},a)if("$"!==d.charAt(0)||"$"!==d.charAt(1))b[d]=a[d];return b||a}function Be(a,b){var d=[];Sb(b)&&(a=ra(a,null,b));return JSON.stringify(a,function(a,b){b=Nc(a,b);if(C(b)){if(0<=d.indexOf(b))return"...";d.push(b)}return b})}function Ce(a){S(a,{errorHandlingConfig:me,bootstrap:Sc,copy:ra,extend:S,merge:oe,equals:sa,element:B,forEach:q,injector:eb,noop:z,bind:bb,toJson:cb,fromJson:Oc,identity:Ya,isUndefined:w,isDefined:u,isString:F,
-isFunction:D,isObject:C,isNumber:ba,isElement:Ub,isArray:H,version:De,isDate:ga,lowercase:Q,uppercase:ub,callbacks:{$$counter:0},getTestability:xe,reloadWithDebugInfo:we,$$minErr:L,$$csp:Ga,$$encodeUriSegment:db,$$encodeUriQuery:$,$$stringify:$b});ac=Ae(x);ac("ng",["ngLocale"],["$provide",function(a){a.provider({$$sanitizeUri:Ee});a.provider("$compile",Wc).directive({a:Fe,input:Xc,textarea:Xc,form:Ge,script:He,select:Ie,option:Je,ngBind:Ke,ngBindHtml:Le,ngBindTemplate:Me,ngClass:Ne,ngClassEven:Oe,
-ngClassOdd:Pe,ngCloak:Qe,ngController:Re,ngForm:Se,ngHide:Te,ngIf:Ue,ngInclude:Ve,ngInit:We,ngNonBindable:Xe,ngPluralize:Ye,ngRepeat:Ze,ngShow:$e,ngStyle:af,ngSwitch:bf,ngSwitchWhen:cf,ngSwitchDefault:df,ngOptions:ef,ngTransclude:ff,ngModel:gf,ngList:hf,ngChange:jf,pattern:Yc,ngPattern:Yc,required:Zc,ngRequired:Zc,minlength:$c,ngMinlength:$c,maxlength:ad,ngMaxlength:ad,ngValue:kf,ngModelOptions:lf}).directive({ngInclude:mf}).directive(vb).directive(bd);a.provider({$anchorScroll:nf,$animate:of,$animateCss:pf,
-$$animateJs:qf,$$animateQueue:rf,$$AnimateRunner:sf,$$animateAsyncRun:tf,$browser:uf,$cacheFactory:vf,$controller:wf,$document:xf,$$isDocumentHidden:yf,$exceptionHandler:zf,$filter:cd,$$forceReflow:Af,$interpolate:Bf,$interval:Cf,$http:Df,$httpParamSerializer:Ef,$httpParamSerializerJQLike:Ff,$httpBackend:Gf,$xhrFactory:Hf,$jsonpCallbacks:If,$location:Jf,$log:Kf,$parse:Lf,$rootScope:Mf,$q:Nf,$$q:Of,$sce:Pf,$sceDelegate:Qf,$sniffer:Rf,$templateCache:Sf,$templateRequest:Tf,$$testability:Uf,$timeout:Vf,
-$window:Wf,$$rAF:Xf,$$jqLite:Yf,$$Map:Zf,$$cookieReader:$f})}]).info({angularVersion:"1.6.4"})}function gb(a,b){return b.toUpperCase()}function wb(a){return a.replace(ag,gb)}function bc(a){a=a.nodeType;return 1===a||!a||9===a}function dd(a,b){var d,c,e=b.createDocumentFragment(),f=[];if(cc.test(a)){d=e.appendChild(b.createElement("div"));c=(bg.exec(a)||["",""])[1].toLowerCase();c=ha[c]||ha._default;d.innerHTML=c[1]+a.replace(cg,"<$1></$2>")+c[2];for(c=c[0];c--;)d=d.lastChild;f=ab(f,d.childNodes);
-d=e.firstChild;d.textContent=""}else f.push(b.createTextNode(a));e.textContent="";e.innerHTML="";q(f,function(a){e.appendChild(a)});return e}function W(a){if(a instanceof W)return a;var b;F(a)&&(a=T(a),b=!0);if(!(this instanceof W)){if(b&&"<"!==a.charAt(0))throw dc("nosel");return new W(a)}if(b){b=x.document;var d;a=(d=dg.exec(a))?[b.createElement(d[1])]:(d=dd(a,b))?d.childNodes:[];ec(this,a)}else D(a)?ed(a):ec(this,a)}function fc(a){return a.cloneNode(!0)}function xb(a,b){!b&&bc(a)&&B.cleanData([a]);
-a.querySelectorAll&&B.cleanData(a.querySelectorAll("*"))}function fd(a,b,d,c){if(u(c))throw dc("offargs");var e=(c=yb(a))&&c.events,f=c&&c.handle;if(f)if(b){var g=function(b){var c=e[b];u(d)&&$a(c||[],d);u(d)&&c&&0<c.length||(a.removeEventListener(b,f),delete e[b])};q(b.split(" "),function(a){g(a);zb[a]&&g(zb[a])})}else for(b in e)"$destroy"!==b&&a.removeEventListener(b,f),delete e[b]}function gc(a,b){var d=a.ng339,c=d&&hb[d];c&&(b?delete c.data[b]:(c.handle&&(c.events.$destroy&&c.handle({},"$destroy"),
-fd(a)),delete hb[d],a.ng339=void 0))}function yb(a,b){var d=a.ng339,d=d&&hb[d];b&&!d&&(a.ng339=d=++eg,d=hb[d]={events:{},data:{},handle:void 0});return d}function hc(a,b,d){if(bc(a)){var c,e=u(d),f=!e&&b&&!C(b),g=!b;a=(a=yb(a,!f))&&a.data;if(e)a[wb(b)]=d;else{if(g)return a;if(f)return a&&a[wb(b)];for(c in b)a[wb(c)]=b[c]}}}function Ab(a,b){return a.getAttribute?-1<(" "+(a.getAttribute("class")||"")+" ").replace(/[\n\t]/g," ").indexOf(" "+b+" "):!1}function Bb(a,b){b&&a.setAttribute&&q(b.split(" "),
-function(b){a.setAttribute("class",T((" "+(a.getAttribute("class")||"")+" ").replace(/[\n\t]/g," ").replace(" "+T(b)+" "," ")))})}function Cb(a,b){if(b&&a.setAttribute){var d=(" "+(a.getAttribute("class")||"")+" ").replace(/[\n\t]/g," ");q(b.split(" "),function(a){a=T(a);-1===d.indexOf(" "+a+" ")&&(d+=a+" ")});a.setAttribute("class",T(d))}}function ec(a,b){if(b)if(b.nodeType)a[a.length++]=b;else{var d=b.length;if("number"===typeof d&&b.window!==b){if(d)for(var c=0;c<d;c++)a[a.length++]=b[c]}else a[a.length++]=
-b}}function gd(a,b){return Db(a,"$"+(b||"ngController")+"Controller")}function Db(a,b,d){9===a.nodeType&&(a=a.documentElement);for(b=H(b)?b:[b];a;){for(var c=0,e=b.length;c<e;c++)if(u(d=B.data(a,b[c])))return d;a=a.parentNode||11===a.nodeType&&a.host}}function hd(a){for(xb(a,!0);a.firstChild;)a.removeChild(a.firstChild)}function Eb(a,b){b||xb(a);var d=a.parentNode;d&&d.removeChild(a)}function fg(a,b){b=b||x;if("complete"===b.document.readyState)b.setTimeout(a);else B(b).on("load",a)}function ed(a){function b(){x.document.removeEventListener("DOMContentLoaded",
-b);x.removeEventListener("load",b);a()}"complete"===x.document.readyState?x.setTimeout(a):(x.document.addEventListener("DOMContentLoaded",b),x.addEventListener("load",b))}function id(a,b){var d=Fb[b.toLowerCase()];return d&&jd[wa(a)]&&d}function gg(a,b){var d=function(c,d){c.isDefaultPrevented=function(){return c.defaultPrevented};var f=b[d||c.type],g=f?f.length:0;if(g){if(w(c.immediatePropagationStopped)){var h=c.stopImmediatePropagation;c.stopImmediatePropagation=function(){c.immediatePropagationStopped=
-!0;c.stopPropagation&&c.stopPropagation();h&&h.call(c)}}c.isImmediatePropagationStopped=function(){return!0===c.immediatePropagationStopped};var k=f.specialHandlerWrapper||hg;1<g&&(f=pa(f));for(var l=0;l<g;l++)c.isImmediatePropagationStopped()||k(a,c,f[l])}};d.elem=a;return d}function hg(a,b,d){d.call(a,b)}function ig(a,b,d){var c=b.relatedTarget;c&&(c===a||jg.call(a,c))||d.call(a,b)}function Yf(){this.$get=function(){return S(W,{hasClass:function(a,b){a.attr&&(a=a[0]);return Ab(a,b)},addClass:function(a,
-b){a.attr&&(a=a[0]);return Cb(a,b)},removeClass:function(a,b){a.attr&&(a=a[0]);return Bb(a,b)}})}}function Pa(a,b){var d=a&&a.$$hashKey;if(d)return"function"===typeof d&&(d=a.$$hashKey()),d;d=typeof a;return d="function"===d||"object"===d&&null!==a?a.$$hashKey=d+":"+(b||ne)():d+":"+a}function kd(){this._keys=[];this._values=[];this._lastKey=NaN;this._lastIndex=-1}function ld(a){a=Function.prototype.toString.call(a).replace(kg,"");return a.match(lg)||a.match(mg)}function ng(a){return(a=ld(a))?"function("+
-(a[1]||"").replace(/[\s\r\n]+/," ")+")":"fn"}function eb(a,b){function d(a){return function(b,c){if(C(b))q(b,Lc(a));else return a(b,c)}}function c(a,b){Ka(a,"service");if(D(b)||H(b))b=p.instantiate(b);if(!b.$get)throw ya("pget",a);return n[a+"Provider"]=b}function e(a,b){return function(){var c=v.invoke(b,this);if(w(c))throw ya("undef",a);return c}}function f(a,b,d){return c(a,{$get:!1!==d?e(a,b):b})}function g(a){fb(w(a)||H(a),"modulesToLoad","not an array");var b=[],c;q(a,function(a){function d(a){var b,
-c;b=0;for(c=a.length;b<c;b++){var e=a[b],f=p.get(e[0]);f[e[1]].apply(f,e[2])}}if(!m.get(a)){m.set(a,!0);try{F(a)?(c=ac(a),v.modules[a]=c,b=b.concat(g(c.requires)).concat(c._runBlocks),d(c._invokeQueue),d(c._configBlocks)):D(a)?b.push(p.invoke(a)):H(a)?b.push(p.invoke(a)):sb(a,"module")}catch(e){throw H(a)&&(a=a[a.length-1]),e.message&&e.stack&&-1===e.stack.indexOf(e.message)&&(e=e.message+"\n"+e.stack),ya("modulerr",a,e.stack||e.message||e);}}});return b}function h(a,c){function d(b,e){if(a.hasOwnProperty(b)){if(a[b]===
-k)throw ya("cdep",b+" <- "+l.join(" <- "));return a[b]}try{return l.unshift(b),a[b]=k,a[b]=c(b,e),a[b]}catch(f){throw a[b]===k&&delete a[b],f;}finally{l.shift()}}function e(a,c,f){var g=[];a=eb.$$annotate(a,b,f);for(var k=0,h=a.length;k<h;k++){var l=a[k];if("string"!==typeof l)throw ya("itkn",l);g.push(c&&c.hasOwnProperty(l)?c[l]:d(l,f))}return g}return{invoke:function(a,b,c,d){"string"===typeof c&&(d=c,c=null);c=e(a,c,d);H(a)&&(a=a[a.length-1]);d=a;if(za||"function"!==typeof d)d=!1;else{var f=d.$$ngIsClass;
-Ha(f)||(f=d.$$ngIsClass=/^(?:class\b|constructor\()/.test(Function.prototype.toString.call(d)));d=f}return d?(c.unshift(null),new (Function.prototype.bind.apply(a,c))):a.apply(b,c)},instantiate:function(a,b,c){var d=H(a)?a[a.length-1]:a;a=e(a,b,c);a.unshift(null);return new (Function.prototype.bind.apply(d,a))},get:d,annotate:eb.$$annotate,has:function(b){return n.hasOwnProperty(b+"Provider")||a.hasOwnProperty(b)}}}b=!0===b;var k={},l=[],m=new Gb,n={$provide:{provider:d(c),factory:d(f),service:d(function(a,
-b){return f(a,["$injector",function(a){return a.instantiate(b)}])}),value:d(function(a,b){return f(a,la(b),!1)}),constant:d(function(a,b){Ka(a,"constant");n[a]=b;r[a]=b}),decorator:function(a,b){var c=p.get(a+"Provider"),d=c.$get;c.$get=function(){var a=v.invoke(d,c);return v.invoke(b,null,{$delegate:a})}}}},p=n.$injector=h(n,function(a,b){ea.isString(b)&&l.push(b);throw ya("unpr",l.join(" <- "));}),r={},J=h(r,function(a,b){var c=p.get(a+"Provider",b);return v.invoke(c.$get,c,void 0,a)}),v=J;n.$injectorProvider=
-{$get:la(J)};v.modules=p.modules=V();var t=g(a),v=J.get("$injector");v.strictDi=b;q(t,function(a){a&&v.invoke(a)});return v}function nf(){var a=!0;this.disableAutoScrolling=function(){a=!1};this.$get=["$window","$location","$rootScope",function(b,d,c){function e(a){var b=null;Array.prototype.some.call(a,function(a){if("a"===wa(a))return b=a,!0});return b}function f(a){if(a){a.scrollIntoView();var c;c=g.yOffset;D(c)?c=c():Ub(c)?(c=c[0],c="fixed"!==b.getComputedStyle(c).position?0:c.getBoundingClientRect().bottom):
-ba(c)||(c=0);c&&(a=a.getBoundingClientRect().top,b.scrollBy(0,a-c))}else b.scrollTo(0,0)}function g(a){a=F(a)?a:ba(a)?a.toString():d.hash();var b;a?(b=h.getElementById(a))?f(b):(b=e(h.getElementsByName(a)))?f(b):"top"===a&&f(null):f(null)}var h=b.document;a&&c.$watch(function(){return d.hash()},function(a,b){a===b&&""===a||fg(function(){c.$evalAsync(g)})});return g}]}function ib(a,b){if(!a&&!b)return"";if(!a)return b;if(!b)return a;H(a)&&(a=a.join(" "));H(b)&&(b=b.join(" "));return a+" "+b}function og(a){F(a)&&
-(a=a.split(" "));var b=V();q(a,function(a){a.length&&(b[a]=!0)});return b}function ia(a){return C(a)?a:{}}function pg(a,b,d,c){function e(a){try{a.apply(null,va.call(arguments,1))}finally{if(J--,0===J)for(;v.length;)try{v.pop()()}catch(b){d.error(b)}}}function f(){Oa=null;h()}function g(){t=I();t=w(t)?null:t;sa(t,G)&&(t=G);M=G=t}function h(){var a=M;g();if(N!==k.url()||a!==t)N=k.url(),M=t,q(K,function(a){a(k.url(),t)})}var k=this,l=a.location,m=a.history,n=a.setTimeout,p=a.clearTimeout,r={};k.isMock=
-!1;var J=0,v=[];k.$$completeOutstandingRequest=e;k.$$incOutstandingRequestCount=function(){J++};k.notifyWhenNoOutstandingRequests=function(a){0===J?a():v.push(a)};var t,M,N=l.href,A=b.find("base"),Oa=null,I=c.history?function(){try{return m.state}catch(a){}}:z;g();k.url=function(b,d,e){w(e)&&(e=null);l!==a.location&&(l=a.location);m!==a.history&&(m=a.history);if(b){var f=M===e;if(N===b&&(!c.history||f))return k;var h=N&&Aa(N)===Aa(b);N=b;M=e;!c.history||h&&f?(h||(Oa=b),d?l.replace(b):h?(d=l,e=b.indexOf("#"),
-e=-1===e?"":b.substr(e),d.hash=e):l.href=b,l.href!==b&&(Oa=b)):(m[d?"replaceState":"pushState"](e,"",b),g());Oa&&(Oa=b);return k}return Oa||l.href.replace(/%27/g,"'")};k.state=function(){return t};var K=[],E=!1,G=null;k.onUrlChange=function(b){if(!E){if(c.history)B(a).on("popstate",f);B(a).on("hashchange",f);E=!0}K.push(b);return b};k.$$applicationDestroyed=function(){B(a).off("hashchange popstate",f)};k.$$checkUrlChange=h;k.baseHref=function(){var a=A.attr("href");return a?a.replace(/^(https?:)?\/\/[^/]*/,
-""):""};k.defer=function(a,b){var c;J++;c=n(function(){delete r[c];e(a)},b||0);r[c]=!0;return c};k.defer.cancel=function(a){return r[a]?(delete r[a],p(a),e(z),!0):!1}}function uf(){this.$get=["$window","$log","$sniffer","$document",function(a,b,d,c){return new pg(a,c,b,d)}]}function vf(){this.$get=function(){function a(a,c){function e(a){a!==n&&(p?p===a&&(p=a.n):p=a,f(a.n,a.p),f(a,n),n=a,n.n=null)}function f(a,b){a!==b&&(a&&(a.p=b),b&&(b.n=a))}if(a in b)throw L("$cacheFactory")("iid",a);var g=0,h=
-S({},c,{id:a}),k=V(),l=c&&c.capacity||Number.MAX_VALUE,m=V(),n=null,p=null;return b[a]={put:function(a,b){if(!w(b)){if(l<Number.MAX_VALUE){var c=m[a]||(m[a]={key:a});e(c)}a in k||g++;k[a]=b;g>l&&this.remove(p.key);return b}},get:function(a){if(l<Number.MAX_VALUE){var b=m[a];if(!b)return;e(b)}return k[a]},remove:function(a){if(l<Number.MAX_VALUE){var b=m[a];if(!b)return;b===n&&(n=b.p);b===p&&(p=b.n);f(b.n,b.p);delete m[a]}a in k&&(delete k[a],g--)},removeAll:function(){k=V();g=0;m=V();n=p=null},destroy:function(){m=
-h=k=null;delete b[a]},info:function(){return S({},h,{size:g})}}}var b={};a.info=function(){var a={};q(b,function(b,e){a[e]=b.info()});return a};a.get=function(a){return b[a]};return a}}function Sf(){this.$get=["$cacheFactory",function(a){return a("templates")}]}function Wc(a,b){function d(a,b,c){var d=/^\s*([@&<]|=(\*?))(\??)\s*([\w$]*)\s*$/,e=V();q(a,function(a,f){if(a in n)e[f]=n[a];else{var g=a.match(d);if(!g)throw fa("iscp",b,f,a,c?"controller bindings definition":"isolate scope definition");
-e[f]={mode:g[1][0],collection:"*"===g[2],optional:"?"===g[3],attrName:g[4]||f};g[4]&&(n[a]=e[f])}});return e}function c(a){var b=a.charAt(0);if(!b||b!==Q(b))throw fa("baddir",a);if(a!==a.trim())throw fa("baddir",a);}function e(a){var b=a.require||a.controller&&a.name;!H(b)&&C(b)&&q(b,function(a,c){var d=a.match(l);a.substring(d[0].length)||(b[c]=d[0]+c)});return b}var f={},g=/^\s*directive:\s*([\w-]+)\s+(.*)$/,h=/(([\w-]+)(?::([^;]+))?;?)/,k=re("ngSrc,ngSrcset,src,srcset"),l=/^(?:(\^\^?)?(\?)?(\^\^?)?)?/,
-m=/^(on[a-z]+|formaction)$/,n=V();this.directive=function N(b,d){fb(b,"name");Ka(b,"directive");F(b)?(c(b),fb(d,"directiveFactory"),f.hasOwnProperty(b)||(f[b]=[],a.factory(b+"Directive",["$injector","$exceptionHandler",function(a,c){var d=[];q(f[b],function(f,g){try{var h=a.invoke(f);D(h)?h={compile:la(h)}:!h.compile&&h.link&&(h.compile=la(h.link));h.priority=h.priority||0;h.index=g;h.name=h.name||b;h.require=e(h);var k=h,l=h.restrict;if(l&&(!F(l)||!/[EACM]/.test(l)))throw fa("badrestrict",l,b);k.restrict=
-l||"EA";h.$$moduleName=f.$$moduleName;d.push(h)}catch(m){c(m)}});return d}])),f[b].push(d)):q(b,Lc(N));return this};this.component=function(a,b){function c(a){function e(b){return D(b)||H(b)?function(c,d){return a.invoke(b,this,{$element:c,$attrs:d})}:b}var f=b.template||b.templateUrl?b.template:"",g={controller:d,controllerAs:qg(b.controller)||b.controllerAs||"$ctrl",template:e(f),templateUrl:e(b.templateUrl),transclude:b.transclude,scope:{},bindToController:b.bindings||{},restrict:"E",require:b.require};
-q(b,function(a,b){"$"===b.charAt(0)&&(g[b]=a)});return g}var d=b.controller||function(){};q(b,function(a,b){"$"===b.charAt(0)&&(c[b]=a,D(d)&&(d[b]=a))});c.$inject=["$injector"];return this.directive(a,c)};this.aHrefSanitizationWhitelist=function(a){return u(a)?(b.aHrefSanitizationWhitelist(a),this):b.aHrefSanitizationWhitelist()};this.imgSrcSanitizationWhitelist=function(a){return u(a)?(b.imgSrcSanitizationWhitelist(a),this):b.imgSrcSanitizationWhitelist()};var p=!0;this.debugInfoEnabled=function(a){return u(a)?
-(p=a,this):p};var r=!1;this.preAssignBindingsEnabled=function(a){return u(a)?(r=a,this):r};var J=10;this.onChangesTtl=function(a){return arguments.length?(J=a,this):J};var v=!0;this.commentDirectivesEnabled=function(a){return arguments.length?(v=a,this):v};var t=!0;this.cssClassDirectivesEnabled=function(a){return arguments.length?(t=a,this):t};this.$get=["$injector","$interpolate","$exceptionHandler","$templateRequest","$parse","$controller","$rootScope","$sce","$animate","$$sanitizeUri",function(a,
-b,c,e,n,E,G,y,O,X){function P(){try{if(!--ya)throw ia=void 0,fa("infchng",J);G.$apply(function(){for(var a=[],b=0,c=ia.length;b<c;++b)try{ia[b]()}catch(d){a.push(d)}ia=void 0;if(a.length)throw a;})}finally{ya++}}function s(a,b){if(b){var c=Object.keys(b),d,e,f;d=0;for(e=c.length;d<e;d++)f=c[d],this[f]=b[f]}else this.$attr={};this.$$element=a}function R(a,b,c){ta.innerHTML="<span "+b+">";b=ta.firstChild.attributes;var d=b[0];b.removeNamedItem(d.name);d.value=c;a.attributes.setNamedItem(d)}function La(a,
-b){try{a.addClass(b)}catch(c){}}function ca(a,b,c,d,e){a instanceof B||(a=B(a));var f=Ma(a,b,a,c,d,e);ca.$$addScopeClass(a);var g=null;return function(b,c,d){if(!a)throw fa("multilink");fb(b,"scope");e&&e.needsNewScope&&(b=b.$parent.$new());d=d||{};var h=d.parentBoundTranscludeFn,k=d.transcludeControllers;d=d.futureParentElement;h&&h.$$boundTransclude&&(h=h.$$boundTransclude);g||(g=(d=d&&d[0])?"foreignobject"!==wa(d)&&ma.call(d).match(/SVG/)?"svg":"html":"html");d="html"!==g?B(ha(g,B("<div>").append(a).html())):
-c?Na.clone.call(a):a;if(k)for(var l in k)d.data("$"+l+"Controller",k[l].instance);ca.$$addScopeInfo(d,b);c&&c(d,b);f&&f(b,d,d,h);c||(a=f=null);return d}}function Ma(a,b,c,d,e,f){function g(a,c,d,e){var f,k,l,m,n,p,r;if(K)for(r=Array(c.length),m=0;m<h.length;m+=3)f=h[m],r[f]=c[f];else r=c;m=0;for(n=h.length;m<n;)k=r[h[m++]],c=h[m++],f=h[m++],c?(c.scope?(l=a.$new(),ca.$$addScopeInfo(B(k),l)):l=a,p=c.transcludeOnThisElement?ja(a,c.transclude,e):!c.templateOnThisElement&&e?e:!e&&b?ja(a,b):null,c(f,l,
-k,d,p)):f&&f(a,k.childNodes,void 0,e)}for(var h=[],k=H(a)||a instanceof B,l,m,n,p,K,r=0;r<a.length;r++){l=new s;11===za&&L(a,r,k);m=jc(a[r],[],l,0===r?d:void 0,e);(f=m.length?W(m,a[r],l,b,c,null,[],[],f):null)&&f.scope&&ca.$$addScopeClass(l.$$element);l=f&&f.terminal||!(n=a[r].childNodes)||!n.length?null:Ma(n,f?(f.transcludeOnThisElement||!f.templateOnThisElement)&&f.transclude:b);if(f||l)h.push(r,f,l),p=!0,K=K||f;f=null}return p?g:null}function L(a,b,c){var d=a[b],e=d.parentNode,f;if(d.nodeType===
-Ia)for(;;){f=e?d.nextSibling:a[b+1];if(!f||f.nodeType!==Ia)break;d.nodeValue+=f.nodeValue;f.parentNode&&f.parentNode.removeChild(f);c&&f===a[b+1]&&a.splice(b+1,1)}}function ja(a,b,c){function d(e,f,g,h,k){e||(e=a.$new(!1,k),e.$$transcluded=!0);return b(e,f,{parentBoundTranscludeFn:c,transcludeControllers:g,futureParentElement:h})}var e=d.$$slots=V(),f;for(f in b.$$slots)e[f]=b.$$slots[f]?ja(a,b.$$slots[f],c):null;return d}function jc(a,b,c,d,e){var f=c.$attr,g;switch(a.nodeType){case 1:g=wa(a);Y(b,
-Ba(g),"E",d,e);for(var k,l,m,n,p=a.attributes,K=0,r=p&&p.length;K<r;K++){var G=!1,E=!1;k=p[K];l=k.name;m=k.value;k=Ba(l);(n=Ja.test(k))&&(l=l.replace(md,"").substr(8).replace(/_(.)/g,function(a,b){return b.toUpperCase()}));(k=k.match(Ka))&&Z(k[1])&&(G=l,E=l.substr(0,l.length-5)+"end",l=l.substr(0,l.length-6));k=Ba(l.toLowerCase());f[k]=l;if(n||!c.hasOwnProperty(k))c[k]=m,id(a,k)&&(c[k]=!0);pa(a,b,m,k,n);Y(b,k,"A",d,e,G,E)}"input"===g&&"hidden"===a.getAttribute("type")&&a.setAttribute("autocomplete",
-"off");if(!Ga)break;f=a.className;C(f)&&(f=f.animVal);if(F(f)&&""!==f)for(;a=h.exec(f);)k=Ba(a[2]),Y(b,k,"C",d,e)&&(c[k]=T(a[3])),f=f.substr(a.index+a[0].length);break;case Ia:la(b,a.nodeValue);break;case 8:if(!Fa)break;jb(a,b,c,d,e)}b.sort(ea);return b}function jb(a,b,c,d,e){try{var f=g.exec(a.nodeValue);if(f){var h=Ba(f[1]);Y(b,h,"M",d,e)&&(c[h]=T(f[2]))}}catch(k){}}function nd(a,b,c){var d=[],e=0;if(b&&a.hasAttribute&&a.hasAttribute(b)){do{if(!a)throw fa("uterdir",b,c);1===a.nodeType&&(a.hasAttribute(b)&&
-e++,a.hasAttribute(c)&&e--);d.push(a);a=a.nextSibling}while(0<e)}else d.push(a);return B(d)}function od(a,b,c){return function(d,e,f,g,h){e=nd(e[0],b,c);return a(d,e,f,g,h)}}function kc(a,b,c,d,e,f){var g;return a?ca(b,c,d,e,f):function(){g||(g=ca(b,c,d,e,f),b=c=f=null);return g.apply(this,arguments)}}function W(a,b,d,e,f,g,h,k,l){function m(a,b,c,d){if(a){c&&(a=od(a,c,d));a.require=y.require;a.directiveName=P;if(E===y||y.$$isolateScope)a=qa(a,{isolateScope:!0});h.push(a)}if(b){c&&(b=od(b,c,d));b.require=
-y.require;b.directiveName=P;if(E===y||y.$$isolateScope)b=qa(b,{isolateScope:!0});k.push(b)}}function n(a,e,f,g,l){function m(a,b,c,d){var e;Za(a)||(d=c,c=b,b=a,a=void 0);X&&(e=O);c||(c=X?P.parent():P);if(d){var f=l.$$slots[d];if(f)return f(a,b,e,c,R);if(w(f))throw fa("noslot",d,xa(P));}else return l(a,b,e,c,R)}var p,y,t,v,J,O,N,P;b===f?(g=d,P=d.$$element):(P=B(f),g=new s(P,d));J=e;E?v=e.$new(!0):K&&(J=e.$parent);l&&(N=m,N.$$boundTransclude=l,N.isSlotFilled=function(a){return!!l.$$slots[a]});G&&(O=
-ba(P,g,N,G,v,e,E));E&&(ca.$$addScopeInfo(P,v,!0,!(I&&(I===E||I===E.$$originalDirective))),ca.$$addScopeClass(P,!0),v.$$isolateBindings=E.$$isolateBindings,y=na(e,g,v,v.$$isolateBindings,E),y.removeWatches&&v.$on("$destroy",y.removeWatches));for(p in O){y=G[p];t=O[p];var Hb=y.$$bindings.bindToController;if(r){t.bindingInfo=Hb?na(J,g,t.instance,Hb,y):{};var A=t();A!==t.instance&&(t.instance=A,P.data("$"+y.name+"Controller",A),t.bindingInfo.removeWatches&&t.bindingInfo.removeWatches(),t.bindingInfo=
-na(J,g,t.instance,Hb,y))}else t.instance=t(),P.data("$"+y.name+"Controller",t.instance),t.bindingInfo=na(J,g,t.instance,Hb,y)}q(G,function(a,b){var c=a.require;a.bindToController&&!H(c)&&C(c)&&S(O[b].instance,U(b,c,P,O))});q(O,function(a){var b=a.instance;if(D(b.$onChanges))try{b.$onChanges(a.bindingInfo.initialChanges)}catch(d){c(d)}if(D(b.$onInit))try{b.$onInit()}catch(e){c(e)}D(b.$doCheck)&&(J.$watch(function(){b.$doCheck()}),b.$doCheck());D(b.$onDestroy)&&J.$on("$destroy",function(){b.$onDestroy()})});
-p=0;for(y=h.length;p<y;p++)t=h[p],ra(t,t.isolateScope?v:e,P,g,t.require&&U(t.directiveName,t.require,P,O),N);var R=e;E&&(E.template||null===E.templateUrl)&&(R=v);a&&a(R,f.childNodes,void 0,l);for(p=k.length-1;0<=p;p--)t=k[p],ra(t,t.isolateScope?v:e,P,g,t.require&&U(t.directiveName,t.require,P,O),N);q(O,function(a){a=a.instance;D(a.$postLink)&&a.$postLink()})}l=l||{};for(var p=-Number.MAX_VALUE,K=l.newScopeDirective,G=l.controllerDirectives,E=l.newIsolateScopeDirective,I=l.templateDirective,t=l.nonTlbTranscludeDirective,
-J=!1,O=!1,X=l.hasElementTranscludeDirective,v=d.$$element=B(b),y,P,N,A=e,R,u=!1,La=!1,x,z=0,F=a.length;z<F;z++){y=a[z];var Ma=y.$$start,L=y.$$end;Ma&&(v=nd(b,Ma,L));N=void 0;if(p>y.priority)break;if(x=y.scope)y.templateUrl||(C(x)?($("new/isolated scope",E||K,y,v),E=y):$("new/isolated scope",E,y,v)),K=K||y;P=y.name;if(!u&&(y.replace&&(y.templateUrl||y.template)||y.transclude&&!y.$$tlb)){for(x=z+1;u=a[x++];)if(u.transclude&&!u.$$tlb||u.replace&&(u.templateUrl||u.template)){La=!0;break}u=!0}!y.templateUrl&&
-y.controller&&(G=G||V(),$("'"+P+"' controller",G[P],y,v),G[P]=y);if(x=y.transclude)if(J=!0,y.$$tlb||($("transclusion",t,y,v),t=y),"element"===x)X=!0,p=y.priority,N=v,v=d.$$element=B(ca.$$createComment(P,d[P])),b=v[0],ka(f,va.call(N,0),b),N[0].$$parentNode=N[0].parentNode,A=kc(La,N,e,p,g&&g.name,{nonTlbTranscludeDirective:t});else{var ja=V();if(C(x)){N=[];var Q=V(),jb=V();q(x,function(a,b){var c="?"===a.charAt(0);a=c?a.substring(1):a;Q[a]=b;ja[b]=null;jb[b]=c});q(v.contents(),function(a){var b=Q[Ba(wa(a))];
-b?(jb[b]=!0,ja[b]=ja[b]||[],ja[b].push(a)):N.push(a)});q(jb,function(a,b){if(!a)throw fa("reqslot",b);});for(var ic in ja)ja[ic]&&(ja[ic]=kc(La,ja[ic],e))}else N=B(fc(b)).contents();v.empty();A=kc(La,N,e,void 0,void 0,{needsNewScope:y.$$isolateScope||y.$$newScope});A.$$slots=ja}if(y.template)if(O=!0,$("template",I,y,v),I=y,x=D(y.template)?y.template(v,d):y.template,x=Ea(x),y.replace){g=y;N=cc.test(x)?pd(ha(y.templateNamespace,T(x))):[];b=N[0];if(1!==N.length||1!==b.nodeType)throw fa("tplrt",P,"");
-ka(f,v,b);F={$attr:{}};x=jc(b,[],F);var Y=a.splice(z+1,a.length-(z+1));(E||K)&&aa(x,E,K);a=a.concat(x).concat(Y);da(d,F);F=a.length}else v.html(x);if(y.templateUrl)O=!0,$("template",I,y,v),I=y,y.replace&&(g=y),n=ga(a.splice(z,a.length-z),v,d,f,J&&A,h,k,{controllerDirectives:G,newScopeDirective:K!==y&&K,newIsolateScopeDirective:E,templateDirective:I,nonTlbTranscludeDirective:t}),F=a.length;else if(y.compile)try{R=y.compile(v,d,A);var Z=y.$$originalDirective||y;D(R)?m(null,bb(Z,R),Ma,L):R&&m(bb(Z,R.pre),
-bb(Z,R.post),Ma,L)}catch(ea){c(ea,xa(v))}y.terminal&&(n.terminal=!0,p=Math.max(p,y.priority))}n.scope=K&&!0===K.scope;n.transcludeOnThisElement=J;n.templateOnThisElement=O;n.transclude=A;l.hasElementTranscludeDirective=X;return n}function U(a,b,c,d){var e;if(F(b)){var f=b.match(l);b=b.substring(f[0].length);var g=f[1]||f[3],f="?"===f[2];"^^"===g?c=c.parent():e=(e=d&&d[b])&&e.instance;if(!e){var h="$"+b+"Controller";e=g?c.inheritedData(h):c.data(h)}if(!e&&!f)throw fa("ctreq",b,a);}else if(H(b))for(e=
-[],g=0,f=b.length;g<f;g++)e[g]=U(a,b[g],c,d);else C(b)&&(e={},q(b,function(b,f){e[f]=U(a,b,c,d)}));return e||null}function ba(a,b,c,d,e,f,g){var h=V(),k;for(k in d){var l=d[k],m={$scope:l===g||l.$$isolateScope?e:f,$element:a,$attrs:b,$transclude:c},n=l.controller;"@"===n&&(n=b[l.name]);m=E(n,m,!0,l.controllerAs);h[l.name]=m;a.data("$"+l.name+"Controller",m.instance)}return h}function aa(a,b,c){for(var d=0,e=a.length;d<e;d++)a[d]=Vb(a[d],{$$isolateScope:b,$$newScope:c})}function Y(b,c,e,g,h,k,l){if(c===
-h)return null;var m=null;if(f.hasOwnProperty(c)){h=a.get(c+"Directive");for(var n=0,p=h.length;n<p;n++)if(c=h[n],(w(g)||g>c.priority)&&-1!==c.restrict.indexOf(e)){k&&(c=Vb(c,{$$start:k,$$end:l}));if(!c.$$bindings){var K=m=c,r=c.name,t={isolateScope:null,bindToController:null};C(K.scope)&&(!0===K.bindToController?(t.bindToController=d(K.scope,r,!0),t.isolateScope={}):t.isolateScope=d(K.scope,r,!1));C(K.bindToController)&&(t.bindToController=d(K.bindToController,r,!0));if(t.bindToController&&!K.controller)throw fa("noctrl",
-r);m=m.$$bindings=t;C(m.isolateScope)&&(c.$$isolateBindings=m.isolateScope)}b.push(c);m=c}}return m}function Z(b){if(f.hasOwnProperty(b))for(var c=a.get(b+"Directive"),d=0,e=c.length;d<e;d++)if(b=c[d],b.multiElement)return!0;return!1}function da(a,b){var c=b.$attr,d=a.$attr;q(a,function(d,e){"$"!==e.charAt(0)&&(b[e]&&b[e]!==d&&(d=d.length?d+(("style"===e?";":" ")+b[e]):b[e]),a.$set(e,d,!0,c[e]))});q(b,function(b,e){a.hasOwnProperty(e)||"$"===e.charAt(0)||(a[e]=b,"class"!==e&&"style"!==e&&(d[e]=c[e]))})}
-function ga(a,b,d,f,g,h,k,l){var m=[],n,p,K=b[0],r=a.shift(),t=Vb(r,{templateUrl:null,transclude:null,replace:null,$$originalDirective:r}),y=D(r.templateUrl)?r.templateUrl(b,d):r.templateUrl,E=r.templateNamespace;b.empty();e(y).then(function(c){var e,G;c=Ea(c);if(r.replace){c=cc.test(c)?pd(ha(E,T(c))):[];e=c[0];if(1!==c.length||1!==e.nodeType)throw fa("tplrt",r.name,y);c={$attr:{}};ka(f,b,e);var I=jc(e,[],c);C(r.scope)&&aa(I,!0);a=I.concat(a);da(d,c)}else e=K,b.html(c);a.unshift(t);n=W(a,e,d,g,b,
-r,h,k,l);q(f,function(a,c){a===e&&(f[c]=b[0])});for(p=Ma(b[0].childNodes,g);m.length;){c=m.shift();G=m.shift();var v=m.shift(),J=m.shift(),I=b[0];if(!c.$$destroyed){if(G!==K){var O=G.className;l.hasElementTranscludeDirective&&r.replace||(I=fc(e));ka(v,B(G),I);La(B(I),O)}G=n.transcludeOnThisElement?ja(c,n.transclude,J):J;n(p,c,I,f,G)}}m=null}).catch(function(a){a instanceof Error&&c(a)});return function(a,b,c,d,e){a=e;b.$$destroyed||(m?m.push(b,c,d,a):(n.transcludeOnThisElement&&(a=ja(b,n.transclude,
-e)),n(p,b,c,d,a)))}}function ea(a,b){var c=b.priority-a.priority;return 0!==c?c:a.name!==b.name?a.name<b.name?-1:1:a.index-b.index}function $(a,b,c,d){function e(a){return a?" (module: "+a+")":""}if(b)throw fa("multidir",b.name,e(b.$$moduleName),c.name,e(c.$$moduleName),a,xa(d));}function la(a,c){var d=b(c,!0);d&&a.push({priority:0,compile:function(a){a=a.parent();var b=!!a.length;b&&ca.$$addBindingClass(a);return function(a,c){var e=c.parent();b||ca.$$addBindingClass(e);ca.$$addBindingInfo(e,d.expressions);
-a.$watch(d,function(a){c[0].nodeValue=a})}}})}function ha(a,b){a=Q(a||"html");switch(a){case "svg":case "math":var c=x.document.createElement("div");c.innerHTML="<"+a+">"+b+"</"+a+">";return c.childNodes[0].childNodes;default:return b}}function oa(a,b){if("srcdoc"===b)return y.HTML;var c=wa(a);if("src"===b||"ngSrc"===b){if(-1===["img","video","audio","source","track"].indexOf(c))return y.RESOURCE_URL}else if("xlinkHref"===b||"form"===c&&"action"===b||"link"===c&&"href"===b)return y.RESOURCE_URL}function pa(a,
-c,d,e,f){var g=oa(a,e),h=k[e]||f,l=b(d,!f,g,h);if(l){if("multiple"===e&&"select"===wa(a))throw fa("selmulti",xa(a));if(m.test(e))throw fa("nodomevents");c.push({priority:100,compile:function(){return{pre:function(a,c,f){c=f.$$observers||(f.$$observers=V());var k=f[e];k!==d&&(l=k&&b(k,!0,g,h),d=k);l&&(f[e]=l(a),(c[e]||(c[e]=[])).$$inter=!0,(f.$$observers&&f.$$observers[e].$$scope||a).$watch(l,function(a,b){"class"===e&&a!==b?f.$updateClass(a,b):f.$set(e,a)}))}}}})}}function ka(a,b,c){var d=b[0],e=
-b.length,f=d.parentNode,g,h;if(a)for(g=0,h=a.length;g<h;g++)if(a[g]===d){a[g++]=c;h=g+e-1;for(var k=a.length;g<k;g++,h++)h<k?a[g]=a[h]:delete a[g];a.length-=e-1;a.context===d&&(a.context=c);break}f&&f.replaceChild(c,d);a=x.document.createDocumentFragment();for(g=0;g<e;g++)a.appendChild(b[g]);B.hasData(d)&&(B.data(c,B.data(d)),B(d).off("$destroy"));B.cleanData(a.querySelectorAll("*"));for(g=1;g<e;g++)delete b[g];b[0]=c;b.length=1}function qa(a,b){return S(function(){return a.apply(null,arguments)},
-a,b)}function ra(a,b,d,e,f,g){try{a(b,d,e,f,g)}catch(h){c(h,xa(d))}}function na(a,c,d,e,f){function g(b,c,e){D(d.$onChanges)&&!Xb(c,e)&&(ia||(a.$$postDigest(P),ia=[]),m||(m={},ia.push(h)),m[b]&&(e=m[b].previousValue),m[b]=new Ib(e,c))}function h(){d.$onChanges(m);m=void 0}var k=[],l={},m;q(e,function(e,h){var m=e.attrName,p=e.optional,r,t,y,G;switch(e.mode){case "@":p||ua.call(c,m)||(d[h]=c[m]=void 0);p=c.$observe(m,function(a){if(F(a)||Ha(a))g(h,a,d[h]),d[h]=a});c.$$observers[m].$$scope=a;r=c[m];
-F(r)?d[h]=b(r)(a):Ha(r)&&(d[h]=r);l[h]=new Ib(lc,d[h]);k.push(p);break;case "=":if(!ua.call(c,m)){if(p)break;c[m]=void 0}if(p&&!c[m])break;t=n(c[m]);G=t.literal?sa:Xb;y=t.assign||function(){r=d[h]=t(a);throw fa("nonassign",c[m],m,f.name);};r=d[h]=t(a);p=function(b){G(b,d[h])||(G(b,r)?y(a,b=d[h]):d[h]=b);return r=b};p.$stateful=!0;p=e.collection?a.$watchCollection(c[m],p):a.$watch(n(c[m],p),null,t.literal);k.push(p);break;case "<":if(!ua.call(c,m)){if(p)break;c[m]=void 0}if(p&&!c[m])break;t=n(c[m]);
-var E=t.literal,I=d[h]=t(a);l[h]=new Ib(lc,d[h]);p=a.$watch(t,function(a,b){if(b===a){if(b===I||E&&sa(b,I))return;b=I}g(h,a,b);d[h]=a},E);k.push(p);break;case "&":t=c.hasOwnProperty(m)?n(c[m]):z;if(t===z&&p)break;d[h]=function(b){return t(a,b)}}});return{initialChanges:l,removeWatches:k.length&&function(){for(var a=0,b=k.length;a<b;++a)k[a]()}}}var Ca=/^\w/,ta=x.document.createElement("div"),Fa=v,Ga=t,ya=J,ia;s.prototype={$normalize:Ba,$addClass:function(a){a&&0<a.length&&O.addClass(this.$$element,
-a)},$removeClass:function(a){a&&0<a.length&&O.removeClass(this.$$element,a)},$updateClass:function(a,b){var c=qd(a,b);c&&c.length&&O.addClass(this.$$element,c);(c=qd(b,a))&&c.length&&O.removeClass(this.$$element,c)},$set:function(a,b,d,e){var f=id(this.$$element[0],a),g=rd[a],h=a;f?(this.$$element.prop(a,b),e=f):g&&(this[g]=b,h=g);this[a]=b;e?this.$attr[a]=e:(e=this.$attr[a])||(this.$attr[a]=e=Tc(a,"-"));f=wa(this.$$element);if("a"===f&&("href"===a||"xlinkHref"===a)||"img"===f&&"src"===a)this[a]=
-b=X(b,"src"===a);else if("img"===f&&"srcset"===a&&u(b)){for(var f="",g=T(b),k=/(\s+\d+x\s*,|\s+\d+w\s*,|\s+,|,\s+)/,k=/\s/.test(g)?k:/(,)/,g=g.split(k),k=Math.floor(g.length/2),l=0;l<k;l++)var m=2*l,f=f+X(T(g[m]),!0),f=f+(" "+T(g[m+1]));g=T(g[2*l]).split(/\s/);f+=X(T(g[0]),!0);2===g.length&&(f+=" "+T(g[1]));this[a]=b=f}!1!==d&&(null===b||w(b)?this.$$element.removeAttr(e):Ca.test(e)?this.$$element.attr(e,b):R(this.$$element[0],e,b));(a=this.$$observers)&&q(a[h],function(a){try{a(b)}catch(d){c(d)}})},
-$observe:function(a,b){var c=this,d=c.$$observers||(c.$$observers=V()),e=d[a]||(d[a]=[]);e.push(b);G.$evalAsync(function(){e.$$inter||!c.hasOwnProperty(a)||w(c[a])||b(c[a])});return function(){$a(e,b)}}};var Aa=b.startSymbol(),Da=b.endSymbol(),Ea="{{"===Aa&&"}}"===Da?Ya:function(a){return a.replace(/\{\{/g,Aa).replace(/}}/g,Da)},Ja=/^ngAttr[A-Z]/,Ka=/^(.+)Start$/;ca.$$addBindingInfo=p?function(a,b){var c=a.data("$binding")||[];H(b)?c=c.concat(b):c.push(b);a.data("$binding",c)}:z;ca.$$addBindingClass=
-p?function(a){La(a,"ng-binding")}:z;ca.$$addScopeInfo=p?function(a,b,c,d){a.data(c?d?"$isolateScopeNoTemplate":"$isolateScope":"$scope",b)}:z;ca.$$addScopeClass=p?function(a,b){La(a,b?"ng-isolate-scope":"ng-scope")}:z;ca.$$createComment=function(a,b){var c="";p&&(c=" "+(a||"")+": ",b&&(c+=b+" "));return x.document.createComment(c)};return ca}]}function Ib(a,b){this.previousValue=a;this.currentValue=b}function Ba(a){return a.replace(md,"").replace(rg,gb)}function qd(a,b){var d="",c=a.split(/\s+/),
-e=b.split(/\s+/),f=0;a:for(;f<c.length;f++){for(var g=c[f],h=0;h<e.length;h++)if(g===e[h])continue a;d+=(0<d.length?" ":"")+g}return d}function pd(a){a=B(a);var b=a.length;if(1>=b)return a;for(;b--;){var d=a[b];(8===d.nodeType||d.nodeType===Ia&&""===d.nodeValue.trim())&&sg.call(a,b,1)}return a}function qg(a,b){if(b&&F(b))return b;if(F(a)){var d=sd.exec(a);if(d)return d[3]}}function wf(){var a={},b=!1;this.has=function(b){return a.hasOwnProperty(b)};this.register=function(b,c){Ka(b,"controller");C(b)?
-S(a,b):a[b]=c};this.allowGlobals=function(){b=!0};this.$get=["$injector","$window",function(d,c){function e(a,b,c,d){if(!a||!C(a.$scope))throw L("$controller")("noscp",d,b);a.$scope[b]=c}return function(f,g,h,k){var l,m,n;h=!0===h;k&&F(k)&&(n=k);if(F(f)){k=f.match(sd);if(!k)throw td("ctrlfmt",f);m=k[1];n=n||k[3];f=a.hasOwnProperty(m)?a[m]:Vc(g.$scope,m,!0)||(b?Vc(c,m,!0):void 0);if(!f)throw td("ctrlreg",m);sb(f,m,!0)}if(h)return h=(H(f)?f[f.length-1]:f).prototype,l=Object.create(h||null),n&&e(g,n,
-l,m||f.name),S(function(){var a=d.invoke(f,l,g,m);a!==l&&(C(a)||D(a))&&(l=a,n&&e(g,n,l,m||f.name));return l},{instance:l,identifier:n});l=d.instantiate(f,g,m);n&&e(g,n,l,m||f.name);return l}}]}function xf(){this.$get=["$window",function(a){return B(a.document)}]}function yf(){this.$get=["$document","$rootScope",function(a,b){function d(){e=c.hidden}var c=a[0],e=c&&c.hidden;a.on("visibilitychange",d);b.$on("$destroy",function(){a.off("visibilitychange",d)});return function(){return e}}]}function zf(){this.$get=
-["$log",function(a){return function(b,d){a.error.apply(a,arguments)}}]}function mc(a){return C(a)?ga(a)?a.toISOString():cb(a):a}function Ef(){this.$get=function(){return function(a){if(!a)return"";var b=[];Kc(a,function(a,c){null===a||w(a)||(H(a)?q(a,function(a){b.push($(c)+"="+$(mc(a)))}):b.push($(c)+"="+$(mc(a))))});return b.join("&")}}}function Ff(){this.$get=function(){return function(a){function b(a,e,f){null===a||w(a)||(H(a)?q(a,function(a,c){b(a,e+"["+(C(a)?c:"")+"]")}):C(a)&&!ga(a)?Kc(a,function(a,
-c){b(a,e+(f?"":"[")+c+(f?"":"]"))}):d.push($(e)+"="+$(mc(a))))}if(!a)return"";var d=[];b(a,"",!0);return d.join("&")}}}function nc(a,b){if(F(a)){var d=a.replace(tg,"").trim();if(d){var c=b("Content-Type");(c=c&&0===c.indexOf(ud))||(c=(c=d.match(ug))&&vg[c[0]].test(d));if(c)try{a=Oc(d)}catch(e){throw oc("baddata",a,e);}}}return a}function vd(a){var b=V(),d;F(a)?q(a.split("\n"),function(a){d=a.indexOf(":");var e=Q(T(a.substr(0,d)));a=T(a.substr(d+1));e&&(b[e]=b[e]?b[e]+", "+a:a)}):C(a)&&q(a,function(a,
-d){var f=Q(d),g=T(a);f&&(b[f]=b[f]?b[f]+", "+g:g)});return b}function wd(a){var b;return function(d){b||(b=vd(a));return d?(d=b[Q(d)],void 0===d&&(d=null),d):b}}function xd(a,b,d,c){if(D(c))return c(a,b,d);q(c,function(c){a=c(a,b,d)});return a}function Df(){var a=this.defaults={transformResponse:[nc],transformRequest:[function(a){return C(a)&&"[object File]"!==ma.call(a)&&"[object Blob]"!==ma.call(a)&&"[object FormData]"!==ma.call(a)?cb(a):a}],headers:{common:{Accept:"application/json, text/plain, */*"},
-post:pa(pc),put:pa(pc),patch:pa(pc)},xsrfCookieName:"XSRF-TOKEN",xsrfHeaderName:"X-XSRF-TOKEN",paramSerializer:"$httpParamSerializer",jsonpCallbackParam:"callback"},b=!1;this.useApplyAsync=function(a){return u(a)?(b=!!a,this):b};var d=this.interceptors=[];this.$get=["$browser","$httpBackend","$$cookieReader","$cacheFactory","$rootScope","$q","$injector","$sce",function(c,e,f,g,h,k,l,m){function n(b){function d(a,b){for(var c=0,e=b.length;c<e;){var f=b[c++],g=b[c++];a=a.then(f,g)}b.length=0;return a}
-function e(a,b){var c,d={};q(a,function(a,e){D(a)?(c=a(b),null!=c&&(d[e]=c)):d[e]=a});return d}function f(a){var b=S({},a);b.data=xd(a.data,a.headers,a.status,g.transformResponse);a=a.status;return 200<=a&&300>a?b:k.reject(b)}if(!C(b))throw L("$http")("badreq",b);if(!F(m.valueOf(b.url)))throw L("$http")("badreq",b.url);var g=S({method:"get",transformRequest:a.transformRequest,transformResponse:a.transformResponse,paramSerializer:a.paramSerializer,jsonpCallbackParam:a.jsonpCallbackParam},b);g.headers=
-function(b){var c=a.headers,d=S({},b.headers),f,g,h,c=S({},c.common,c[Q(b.method)]);a:for(f in c){g=Q(f);for(h in d)if(Q(h)===g)continue a;d[f]=c[f]}return e(d,pa(b))}(b);g.method=ub(g.method);g.paramSerializer=F(g.paramSerializer)?l.get(g.paramSerializer):g.paramSerializer;c.$$incOutstandingRequestCount();var h=[],n=[];b=k.resolve(g);q(t,function(a){(a.request||a.requestError)&&h.unshift(a.request,a.requestError);(a.response||a.responseError)&&n.push(a.response,a.responseError)});b=d(b,h);b=b.then(function(b){var c=
-b.headers,d=xd(b.data,wd(c),void 0,b.transformRequest);w(d)&&q(c,function(a,b){"content-type"===Q(b)&&delete c[b]});w(b.withCredentials)&&!w(a.withCredentials)&&(b.withCredentials=a.withCredentials);return p(b,d).then(f,f)});b=d(b,n);return b=b.finally(function(){c.$$completeOutstandingRequest(z)})}function p(c,d){function g(a){if(a){var c={};q(a,function(a,d){c[d]=function(c){function d(){a(c)}b?h.$applyAsync(d):h.$$phase?d():h.$apply(d)}});return c}}function l(a,c,d,e){function f(){p(c,a,d,e)}O&&
-(200<=a&&300>a?O.put(R,[a,c,vd(d),e]):O.remove(R));b?h.$applyAsync(f):(f(),h.$$phase||h.$apply())}function p(a,b,d,e){b=-1<=b?b:0;(200<=b&&300>b?G.resolve:G.reject)({data:a,status:b,headers:wd(d),config:c,statusText:e})}function K(a){p(a.data,a.status,pa(a.headers()),a.statusText)}function t(){var a=n.pendingRequests.indexOf(c);-1!==a&&n.pendingRequests.splice(a,1)}var G=k.defer(),y=G.promise,O,X,P=c.headers,s="jsonp"===Q(c.method),R=c.url;s?R=m.getTrustedResourceUrl(R):F(R)||(R=m.valueOf(R));R=r(R,
-c.paramSerializer(c.params));s&&(R=J(R,c.jsonpCallbackParam));n.pendingRequests.push(c);y.then(t,t);!c.cache&&!a.cache||!1===c.cache||"GET"!==c.method&&"JSONP"!==c.method||(O=C(c.cache)?c.cache:C(a.cache)?a.cache:v);O&&(X=O.get(R),u(X)?X&&D(X.then)?X.then(K,K):H(X)?p(X[1],X[0],pa(X[2]),X[3]):p(X,200,{},"OK"):O.put(R,y));w(X)&&((X=yd(c.url)?f()[c.xsrfCookieName||a.xsrfCookieName]:void 0)&&(P[c.xsrfHeaderName||a.xsrfHeaderName]=X),e(c.method,R,d,l,P,c.timeout,c.withCredentials,c.responseType,g(c.eventHandlers),
-g(c.uploadEventHandlers)));return y}function r(a,b){0<b.length&&(a+=(-1===a.indexOf("?")?"?":"&")+b);return a}function J(a,b){if(/[&?][^=]+=JSON_CALLBACK/.test(a))throw oc("badjsonp",a);if((new RegExp("[&?]"+b+"=")).test(a))throw oc("badjsonp",b,a);return a+=(-1===a.indexOf("?")?"?":"&")+b+"=JSON_CALLBACK"}var v=g("$http");a.paramSerializer=F(a.paramSerializer)?l.get(a.paramSerializer):a.paramSerializer;var t=[];q(d,function(a){t.unshift(F(a)?l.get(a):l.invoke(a))});n.pendingRequests=[];(function(a){q(arguments,
-function(a){n[a]=function(b,c){return n(S({},c||{},{method:a,url:b}))}})})("get","delete","head","jsonp");(function(a){q(arguments,function(a){n[a]=function(b,c,d){return n(S({},d||{},{method:a,url:b,data:c}))}})})("post","put","patch");n.defaults=a;return n}]}function Hf(){this.$get=function(){return function(){return new x.XMLHttpRequest}}}function Gf(){this.$get=["$browser","$jsonpCallbacks","$document","$xhrFactory",function(a,b,d,c){return wg(a,c,a.defer,b,d[0])}]}function wg(a,b,d,c,e){function f(a,
-b,d){a=a.replace("JSON_CALLBACK",b);var f=e.createElement("script"),m=null;f.type="text/javascript";f.src=a;f.async=!0;m=function(a){f.removeEventListener("load",m);f.removeEventListener("error",m);e.body.removeChild(f);f=null;var g=-1,r="unknown";a&&("load"!==a.type||c.wasCalled(b)||(a={type:"error"}),r=a.type,g="error"===a.type?404:200);d&&d(g,r)};f.addEventListener("load",m);f.addEventListener("error",m);e.body.appendChild(f);return m}return function(e,h,k,l,m,n,p,r,J,v){function t(){N&&N();A&&
-A.abort()}h=h||a.url();if("jsonp"===Q(e))var M=c.createCallback(h),N=f(h,M,function(a,b){var e=200===a&&c.getResponse(M);u(I)&&d.cancel(I);N=A=null;l(a,e,"",b);c.removeCallback(M)});else{var A=b(e,h);A.open(e,h,!0);q(m,function(a,b){u(a)&&A.setRequestHeader(b,a)});A.onload=function(){var a=A.statusText||"",b="response"in A?A.response:A.responseText,c=1223===A.status?204:A.status;0===c&&(c=b?200:"file"===Ca(h).protocol?404:0);var e=A.getAllResponseHeaders();u(I)&&d.cancel(I);N=A=null;l(c,b,e,a)};e=
-function(){u(I)&&d.cancel(I);N=A=null;l(-1,null,null,"")};A.onerror=e;A.onabort=e;A.ontimeout=e;q(J,function(a,b){A.addEventListener(b,a)});q(v,function(a,b){A.upload.addEventListener(b,a)});p&&(A.withCredentials=!0);if(r)try{A.responseType=r}catch(s){if("json"!==r)throw s;}A.send(w(k)?null:k)}if(0<n)var I=d(t,n);else n&&D(n.then)&&n.then(t)}}function Bf(){var a="{{",b="}}";this.startSymbol=function(b){return b?(a=b,this):a};this.endSymbol=function(a){return a?(b=a,this):b};this.$get=["$parse","$exceptionHandler",
-"$sce",function(d,c,e){function f(a){return"\\\\\\"+a}function g(c){return c.replace(n,a).replace(p,b)}function h(a,b,c,d){var e=a.$watch(function(a){e();return d(a)},b,c);return e}function k(f,k,n,p){function M(a){try{var b=a;a=n?e.getTrusted(n,b):e.valueOf(b);return p&&!u(a)?a:$b(a)}catch(d){c(Da.interr(f,d))}}if(!f.length||-1===f.indexOf(a)){var q;k||(k=g(f),q=la(k),q.exp=f,q.expressions=[],q.$$watchDelegate=h);return q}p=!!p;var A,s,I=0,K=[],E=[];q=f.length;for(var G=[],y=[];I<q;)if(-1!==(A=f.indexOf(a,
-I))&&-1!==(s=f.indexOf(b,A+l)))I!==A&&G.push(g(f.substring(I,A))),I=f.substring(A+l,s),K.push(I),E.push(d(I,M)),I=s+m,y.push(G.length),G.push("");else{I!==q&&G.push(g(f.substring(I)));break}n&&1<G.length&&Da.throwNoconcat(f);if(!k||K.length){var O=function(a){for(var b=0,c=K.length;b<c;b++){if(p&&w(a[b]))return;G[y[b]]=a[b]}return G.join("")};return S(function(a){var b=0,d=K.length,e=Array(d);try{for(;b<d;b++)e[b]=E[b](a);return O(e)}catch(g){c(Da.interr(f,g))}},{exp:f,expressions:K,$$watchDelegate:function(a,
-b){var c;return a.$watchGroup(E,function(d,e){var f=O(d);D(b)&&b.call(this,f,d!==e?c:f,a);c=f})}})}}var l=a.length,m=b.length,n=new RegExp(a.replace(/./g,f),"g"),p=new RegExp(b.replace(/./g,f),"g");k.startSymbol=function(){return a};k.endSymbol=function(){return b};return k}]}function Cf(){this.$get=["$rootScope","$window","$q","$$q","$browser",function(a,b,d,c,e){function f(f,k,l,m){function n(){p?f.apply(null,r):f(t)}var p=4<arguments.length,r=p?va.call(arguments,4):[],J=b.setInterval,v=b.clearInterval,
-t=0,M=u(m)&&!m,q=(M?c:d).defer(),A=q.promise;l=u(l)?l:0;A.$$intervalId=J(function(){M?e.defer(n):a.$evalAsync(n);q.notify(t++);0<l&&t>=l&&(q.resolve(t),v(A.$$intervalId),delete g[A.$$intervalId]);M||a.$apply()},k);g[A.$$intervalId]=q;return A}var g={};f.cancel=function(a){return a&&a.$$intervalId in g?(g[a.$$intervalId].promise.catch(z),g[a.$$intervalId].reject("canceled"),b.clearInterval(a.$$intervalId),delete g[a.$$intervalId],!0):!1};return f}]}function qc(a){a=a.split("/");for(var b=a.length;b--;)a[b]=
-db(a[b]);return a.join("/")}function zd(a,b){var d=Ca(a);b.$$protocol=d.protocol;b.$$host=d.hostname;b.$$port=Z(d.port)||xg[d.protocol]||null}function Ad(a,b){if(yg.test(a))throw kb("badpath",a);var d="/"!==a.charAt(0);d&&(a="/"+a);var c=Ca(a);b.$$path=decodeURIComponent(d&&"/"===c.pathname.charAt(0)?c.pathname.substring(1):c.pathname);b.$$search=Rc(c.search);b.$$hash=decodeURIComponent(c.hash);b.$$path&&"/"!==b.$$path.charAt(0)&&(b.$$path="/"+b.$$path)}function rc(a,b){return a.slice(0,b.length)===
-b}function ka(a,b){if(rc(b,a))return b.substr(a.length)}function Aa(a){var b=a.indexOf("#");return-1===b?a:a.substr(0,b)}function lb(a){return a.replace(/(#.+)|#$/,"$1")}function sc(a,b,d){this.$$html5=!0;d=d||"";zd(a,this);this.$$parse=function(a){var d=ka(b,a);if(!F(d))throw kb("ipthprfx",a,b);Ad(d,this);this.$$path||(this.$$path="/");this.$$compose()};this.$$compose=function(){var a=Zb(this.$$search),d=this.$$hash?"#"+db(this.$$hash):"";this.$$url=qc(this.$$path)+(a?"?"+a:"")+d;this.$$absUrl=b+
-this.$$url.substr(1);this.$$urlUpdatedByLocation=!0};this.$$parseLinkUrl=function(c,e){if(e&&"#"===e[0])return this.hash(e.slice(1)),!0;var f,g;u(f=ka(a,c))?(g=f,g=d&&u(f=ka(d,f))?b+(ka("/",f)||f):a+g):u(f=ka(b,c))?g=b+f:b===c+"/"&&(g=b);g&&this.$$parse(g);return!!g}}function tc(a,b,d){zd(a,this);this.$$parse=function(c){var e=ka(a,c)||ka(b,c),f;w(e)||"#"!==e.charAt(0)?this.$$html5?f=e:(f="",w(e)&&(a=c,this.replace())):(f=ka(d,e),w(f)&&(f=e));Ad(f,this);c=this.$$path;var e=a,g=/^\/[A-Z]:(\/.*)/;rc(f,
-e)&&(f=f.replace(e,""));g.exec(f)||(c=(f=g.exec(c))?f[1]:c);this.$$path=c;this.$$compose()};this.$$compose=function(){var b=Zb(this.$$search),e=this.$$hash?"#"+db(this.$$hash):"";this.$$url=qc(this.$$path)+(b?"?"+b:"")+e;this.$$absUrl=a+(this.$$url?d+this.$$url:"");this.$$urlUpdatedByLocation=!0};this.$$parseLinkUrl=function(b,d){return Aa(a)===Aa(b)?(this.$$parse(b),!0):!1}}function Bd(a,b,d){this.$$html5=!0;tc.apply(this,arguments);this.$$parseLinkUrl=function(c,e){if(e&&"#"===e[0])return this.hash(e.slice(1)),
-!0;var f,g;a===Aa(c)?f=c:(g=ka(b,c))?f=a+d+g:b===c+"/"&&(f=b);f&&this.$$parse(f);return!!f};this.$$compose=function(){var b=Zb(this.$$search),e=this.$$hash?"#"+db(this.$$hash):"";this.$$url=qc(this.$$path)+(b?"?"+b:"")+e;this.$$absUrl=a+d+this.$$url;this.$$urlUpdatedByLocation=!0}}function Jb(a){return function(){return this[a]}}function Cd(a,b){return function(d){if(w(d))return this[a];this[a]=b(d);this.$$compose();return this}}function Jf(){var a="!",b={enabled:!1,requireBase:!0,rewriteLinks:!0};
-this.hashPrefix=function(b){return u(b)?(a=b,this):a};this.html5Mode=function(a){if(Ha(a))return b.enabled=a,this;if(C(a)){Ha(a.enabled)&&(b.enabled=a.enabled);Ha(a.requireBase)&&(b.requireBase=a.requireBase);if(Ha(a.rewriteLinks)||F(a.rewriteLinks))b.rewriteLinks=a.rewriteLinks;return this}return b};this.$get=["$rootScope","$browser","$sniffer","$rootElement","$window",function(d,c,e,f,g){function h(a,b,d){var e=l.url(),f=l.$$state;try{c.url(a,b,d),l.$$state=c.state()}catch(g){throw l.url(e),l.$$state=
-f,g;}}function k(a,b){d.$broadcast("$locationChangeSuccess",l.absUrl(),a,l.$$state,b)}var l,m;m=c.baseHref();var n=c.url(),p;if(b.enabled){if(!m&&b.requireBase)throw kb("nobase");p=n.substring(0,n.indexOf("/",n.indexOf("//")+2))+(m||"/");m=e.history?sc:Bd}else p=Aa(n),m=tc;var r=p.substr(0,Aa(p).lastIndexOf("/")+1);l=new m(p,r,"#"+a);l.$$parseLinkUrl(n,n);l.$$state=c.state();var J=/^\s*(javascript|mailto):/i;f.on("click",function(a){var e=b.rewriteLinks;if(e&&!a.ctrlKey&&!a.metaKey&&!a.shiftKey&&
-2!==a.which&&2!==a.button){for(var h=B(a.target);"a"!==wa(h[0]);)if(h[0]===f[0]||!(h=h.parent())[0])return;if(!F(e)||!w(h.attr(e))){var e=h.prop("href"),k=h.attr("href")||h.attr("xlink:href");C(e)&&"[object SVGAnimatedString]"===e.toString()&&(e=Ca(e.animVal).href);J.test(e)||!e||h.attr("target")||a.isDefaultPrevented()||!l.$$parseLinkUrl(e,k)||(a.preventDefault(),l.absUrl()!==c.url()&&(d.$apply(),g.angular["ff-684208-preventDefault"]=!0))}}});lb(l.absUrl())!==lb(n)&&c.url(l.absUrl(),!0);var v=!0;
-c.onUrlChange(function(a,b){rc(a,r)?(d.$evalAsync(function(){var c=l.absUrl(),e=l.$$state,f;a=lb(a);l.$$parse(a);l.$$state=b;f=d.$broadcast("$locationChangeStart",a,c,b,e).defaultPrevented;l.absUrl()===a&&(f?(l.$$parse(c),l.$$state=e,h(c,!1,e)):(v=!1,k(c,e)))}),d.$$phase||d.$digest()):g.location.href=a});d.$watch(function(){if(v||l.$$urlUpdatedByLocation){l.$$urlUpdatedByLocation=!1;var a=lb(c.url()),b=lb(l.absUrl()),f=c.state(),g=l.$$replace,m=a!==b||l.$$html5&&e.history&&f!==l.$$state;if(v||m)v=
-!1,d.$evalAsync(function(){var b=l.absUrl(),c=d.$broadcast("$locationChangeStart",b,a,l.$$state,f).defaultPrevented;l.absUrl()===b&&(c?(l.$$parse(a),l.$$state=f):(m&&h(b,g,f===l.$$state?null:l.$$state),k(a,f)))})}l.$$replace=!1});return l}]}function Kf(){var a=!0,b=this;this.debugEnabled=function(b){return u(b)?(a=b,this):a};this.$get=["$window",function(d){function c(a){a instanceof Error&&(a.stack&&f?a=a.message&&-1===a.stack.indexOf(a.message)?"Error: "+a.message+"\n"+a.stack:a.stack:a.sourceURL&&
-(a=a.message+"\n"+a.sourceURL+":"+a.line));return a}function e(a){var b=d.console||{},e=b[a]||b.log||z;a=!1;try{a=!!e.apply}catch(f){}return a?function(){var a=[];q(arguments,function(b){a.push(c(b))});return e.apply(b,a)}:function(a,b){e(a,null==b?"":b)}}var f=za||/\bEdge\//.test(d.navigator&&d.navigator.userAgent);return{log:e("log"),info:e("info"),warn:e("warn"),error:e("error"),debug:function(){var c=e("debug");return function(){a&&c.apply(b,arguments)}}()}}]}function zg(a){return a+""}function Ag(a,
-b){return"undefined"!==typeof a?a:b}function Dd(a,b){return"undefined"===typeof a?b:"undefined"===typeof b?a:a+b}function U(a,b){var d,c,e;switch(a.type){case s.Program:d=!0;q(a.body,function(a){U(a.expression,b);d=d&&a.expression.constant});a.constant=d;break;case s.Literal:a.constant=!0;a.toWatch=[];break;case s.UnaryExpression:U(a.argument,b);a.constant=a.argument.constant;a.toWatch=a.argument.toWatch;break;case s.BinaryExpression:U(a.left,b);U(a.right,b);a.constant=a.left.constant&&a.right.constant;
-a.toWatch=a.left.toWatch.concat(a.right.toWatch);break;case s.LogicalExpression:U(a.left,b);U(a.right,b);a.constant=a.left.constant&&a.right.constant;a.toWatch=a.constant?[]:[a];break;case s.ConditionalExpression:U(a.test,b);U(a.alternate,b);U(a.consequent,b);a.constant=a.test.constant&&a.alternate.constant&&a.consequent.constant;a.toWatch=a.constant?[]:[a];break;case s.Identifier:a.constant=!1;a.toWatch=[a];break;case s.MemberExpression:U(a.object,b);a.computed&&U(a.property,b);a.constant=a.object.constant&&
-(!a.computed||a.property.constant);a.toWatch=[a];break;case s.CallExpression:d=e=a.filter?!b(a.callee.name).$stateful:!1;c=[];q(a.arguments,function(a){U(a,b);d=d&&a.constant;a.constant||c.push.apply(c,a.toWatch)});a.constant=d;a.toWatch=e?c:[a];break;case s.AssignmentExpression:U(a.left,b);U(a.right,b);a.constant=a.left.constant&&a.right.constant;a.toWatch=[a];break;case s.ArrayExpression:d=!0;c=[];q(a.elements,function(a){U(a,b);d=d&&a.constant;a.constant||c.push.apply(c,a.toWatch)});a.constant=
-d;a.toWatch=c;break;case s.ObjectExpression:d=!0;c=[];q(a.properties,function(a){U(a.value,b);d=d&&a.value.constant&&!a.computed;a.value.constant||c.push.apply(c,a.value.toWatch);a.computed&&(U(a.key,b),a.key.constant||c.push.apply(c,a.key.toWatch))});a.constant=d;a.toWatch=c;break;case s.ThisExpression:a.constant=!1;a.toWatch=[];break;case s.LocalsExpression:a.constant=!1,a.toWatch=[]}}function Ed(a){if(1===a.length){a=a[0].expression;var b=a.toWatch;return 1!==b.length?b:b[0]!==a?b:void 0}}function Fd(a){return a.type===
-s.Identifier||a.type===s.MemberExpression}function Gd(a){if(1===a.body.length&&Fd(a.body[0].expression))return{type:s.AssignmentExpression,left:a.body[0].expression,right:{type:s.NGValueParameter},operator:"="}}function Hd(a){this.$filter=a}function Id(a){this.$filter=a}function uc(a,b,d){this.ast=new s(a,d);this.astCompiler=d.csp?new Id(b):new Hd(b)}function vc(a){return D(a.valueOf)?a.valueOf():Bg.call(a)}function Lf(){var a=V(),b={"true":!0,"false":!1,"null":null,undefined:void 0},d,c;this.addLiteral=
-function(a,c){b[a]=c};this.setIdentifierFns=function(a,b){d=a;c=b;return this};this.$get=["$filter",function(e){function f(a,b,c){return null==a||null==b?a===b:"object"!==typeof a||(a=vc(a),"object"!==typeof a||c)?a===b||a!==a&&b!==b:!1}function g(a,b,c,d,e){var g=d.inputs,h;if(1===g.length){var k=f,g=g[0];return a.$watch(function(a){var b=g(a);f(b,k,d.literal)||(h=d(a,void 0,void 0,[b]),k=b&&vc(b));return h},b,c,e)}for(var l=[],m=[],n=0,E=g.length;n<E;n++)l[n]=f,m[n]=null;return a.$watch(function(a){for(var b=
-!1,c=0,e=g.length;c<e;c++){var k=g[c](a);if(b||(b=!f(k,l[c],d.literal)))m[c]=k,l[c]=k&&vc(k)}b&&(h=d(a,void 0,void 0,m));return h},b,c,e)}function h(a,b,c,d,e){function f(a){return d(a)}function h(a,c,d){n=a;D(b)&&b(a,c,d);l(a)&&d.$$postDigest(function(){l(n)&&m()})}var l=d.literal?k:u,m,n;return m=d.inputs?g(a,h,c,d,e):a.$watch(f,h,c)}function k(a){var b=!0;q(a,function(a){u(a)||(b=!1)});return b}function l(a,b,c,d){var e=a.$watch(function(a){e();return d(a)},b,c);return e}function m(a,b){function c(d,
-e,g,h){g=f&&h?h[0]:a(d,e,g,h);return b(g,d,e)}function d(c,e,g,k){g=f&&k?k[0]:a(c,e,g,k);c=b(g,c,e);return h(g)?c:g}if(!b)return a;var e=a.$$watchDelegate,f=!1,h=a.literal?k:u,l=a.oneTime?d:c;l.literal=a.literal;l.oneTime=a.oneTime;f=!a.inputs;e&&e!==g?(l.$$watchDelegate=e,l.inputs=a.inputs):b.$stateful||(l.$$watchDelegate=g,l.inputs=a.inputs?a.inputs:[a]);return l}var n={csp:Ga().noUnsafeEval,literals:ra(b),isIdentifierStart:D(d)&&d,isIdentifierContinue:D(c)&&c};return function(b,c){var d,f,k;switch(typeof b){case "string":return k=
-b=b.trim(),d=a[k],d||(":"===b.charAt(0)&&":"===b.charAt(1)&&(f=!0,b=b.substring(2)),d=new wc(n),d=(new uc(d,e,n)).parse(b),d.constant?d.$$watchDelegate=l:f?(d.oneTime=!0,d.$$watchDelegate=h):d.inputs&&(d.$$watchDelegate=g),a[k]=d),m(d,c);case "function":return m(b,c);default:return m(z,c)}}}]}function Nf(){var a=!0;this.$get=["$rootScope","$exceptionHandler",function(b,d){return Jd(function(a){b.$evalAsync(a)},d,a)}];this.errorOnUnhandledRejections=function(b){return u(b)?(a=b,this):a}}function Of(){var a=
-!0;this.$get=["$browser","$exceptionHandler",function(b,d){return Jd(function(a){b.defer(a)},d,a)}];this.errorOnUnhandledRejections=function(b){return u(b)?(a=b,this):a}}function Jd(a,b,d){function c(){return new e}function e(){var a=this.promise=new f;this.resolve=function(b){k(a,b)};this.reject=function(b){m(a,b)};this.notify=function(b){p(a,b)}}function f(){this.$$state={status:0}}function g(){for(;!s&&A.length;){var a=A.shift();if(!a.pur){a.pur=!0;var c=a.value,c="Possibly unhandled rejection: "+
-("function"===typeof c?c.toString().replace(/ \{[\s\S]*$/,""):w(c)?"undefined":"string"!==typeof c?Be(c,void 0):c);a.value instanceof Error?b(a.value,c):b(c)}}}function h(b){!d||b.pending||2!==b.status||b.pur||(0===s&&0===A.length&&a(g),A.push(b));!b.processScheduled&&b.pending&&(b.processScheduled=!0,++s,a(function(){var c,e,f;f=b.pending;b.processScheduled=!1;b.pending=void 0;try{for(var h=0,l=f.length;h<l;++h){b.pur=!0;e=f[h][0];c=f[h][b.status];try{D(c)?k(e,c(b.value)):1===b.status?k(e,b.value):
-m(e,b.value)}catch(n){m(e,n)}}}finally{--s,d&&0===s&&a(g)}}))}function k(a,b){a.$$state.status||(b===a?n(a,M("qcycle",b)):l(a,b))}function l(a,b){function c(b){g||(g=!0,l(a,b))}function d(b){g||(g=!0,n(a,b))}function e(b){p(a,b)}var f,g=!1;try{if(C(b)||D(b))f=b.then;D(f)?(a.$$state.status=-1,f.call(b,c,d,e)):(a.$$state.value=b,a.$$state.status=1,h(a.$$state))}catch(k){d(k)}}function m(a,b){a.$$state.status||n(a,b)}function n(a,b){a.$$state.value=b;a.$$state.status=2;h(a.$$state)}function p(c,d){var e=
-c.$$state.pending;0>=c.$$state.status&&e&&e.length&&a(function(){for(var a,c,f=0,g=e.length;f<g;f++){c=e[f][0];a=e[f][3];try{p(c,D(a)?a(d):d)}catch(h){b(h)}}})}function r(a){var b=new f;m(b,a);return b}function J(a,b,c){var d=null;try{D(c)&&(d=c())}catch(e){return r(e)}return d&&D(d.then)?d.then(function(){return b(a)},r):b(a)}function v(a,b,c,d){var e=new f;k(e,a);return e.then(b,c,d)}function t(a){if(!D(a))throw M("norslvr",a);var b=new f;a(function(a){k(b,a)},function(a){m(b,a)});return b}var M=
-L("$q",TypeError),s=0,A=[];S(f.prototype,{then:function(a,b,c){if(w(a)&&w(b)&&w(c))return this;var d=new f;this.$$state.pending=this.$$state.pending||[];this.$$state.pending.push([d,a,b,c]);0<this.$$state.status&&h(this.$$state);return d},"catch":function(a){return this.then(null,a)},"finally":function(a,b){return this.then(function(b){return J(b,u,a)},function(b){return J(b,r,a)},b)}});var u=v;t.prototype=f.prototype;t.defer=c;t.reject=r;t.when=v;t.resolve=u;t.all=function(a){var b=new f,c=0,d=H(a)?
-[]:{};q(a,function(a,e){c++;v(a).then(function(a){d[e]=a;--c||k(b,d)},function(a){m(b,a)})});0===c&&k(b,d);return b};t.race=function(a){var b=c();q(a,function(a){v(a).then(b.resolve,b.reject)});return b.promise};return t}function Xf(){this.$get=["$window","$timeout",function(a,b){var d=a.requestAnimationFrame||a.webkitRequestAnimationFrame,c=a.cancelAnimationFrame||a.webkitCancelAnimationFrame||a.webkitCancelRequestAnimationFrame,e=!!d,f=e?function(a){var b=d(a);return function(){c(b)}}:function(a){var c=
-b(a,16.66,!1);return function(){b.cancel(c)}};f.supported=e;return f}]}function Mf(){function a(a){function b(){this.$$watchers=this.$$nextSibling=this.$$childHead=this.$$childTail=null;this.$$listeners={};this.$$listenerCount={};this.$$watchersCount=0;this.$id=++qb;this.$$ChildScope=null}b.prototype=a;return b}var b=10,d=L("$rootScope"),c=null,e=null;this.digestTtl=function(a){arguments.length&&(b=a);return b};this.$get=["$exceptionHandler","$parse","$browser",function(f,g,h){function k(a){a.currentScope.$$destroyed=
-!0}function l(a){9===za&&(a.$$childHead&&l(a.$$childHead),a.$$nextSibling&&l(a.$$nextSibling));a.$parent=a.$$nextSibling=a.$$prevSibling=a.$$childHead=a.$$childTail=a.$root=a.$$watchers=null}function m(){this.$id=++qb;this.$$phase=this.$parent=this.$$watchers=this.$$nextSibling=this.$$prevSibling=this.$$childHead=this.$$childTail=null;this.$root=this;this.$$destroyed=!1;this.$$listeners={};this.$$listenerCount={};this.$$watchersCount=0;this.$$isolateBindings=null}function n(a){if(M.$$phase)throw d("inprog",
-M.$$phase);M.$$phase=a}function p(a,b){do a.$$watchersCount+=b;while(a=a.$parent)}function r(a,b,c){do a.$$listenerCount[c]-=b,0===a.$$listenerCount[c]&&delete a.$$listenerCount[c];while(a=a.$parent)}function J(){}function v(){for(;u.length;)try{u.shift()()}catch(a){f(a)}e=null}function t(){null===e&&(e=h.defer(function(){M.$apply(v)}))}m.prototype={constructor:m,$new:function(b,c){var d;c=c||this;b?(d=new m,d.$root=this.$root):(this.$$ChildScope||(this.$$ChildScope=a(this)),d=new this.$$ChildScope);
-d.$parent=c;d.$$prevSibling=c.$$childTail;c.$$childHead?(c.$$childTail.$$nextSibling=d,c.$$childTail=d):c.$$childHead=c.$$childTail=d;(b||c!==this)&&d.$on("$destroy",k);return d},$watch:function(a,b,d,e){var f=g(a);if(f.$$watchDelegate)return f.$$watchDelegate(this,b,d,f,a);var h=this,k=h.$$watchers,l={fn:b,last:J,get:f,exp:e||a,eq:!!d};c=null;D(b)||(l.fn=z);k||(k=h.$$watchers=[],k.$$digestWatchIndex=-1);k.unshift(l);k.$$digestWatchIndex++;p(this,1);return function(){var a=$a(k,l);0<=a&&(p(h,-1),
-a<k.$$digestWatchIndex&&k.$$digestWatchIndex--);c=null}},$watchGroup:function(a,b){function c(){h=!1;k?(k=!1,b(e,e,g)):b(e,d,g)}var d=Array(a.length),e=Array(a.length),f=[],g=this,h=!1,k=!0;if(!a.length){var l=!0;g.$evalAsync(function(){l&&b(e,e,g)});return function(){l=!1}}if(1===a.length)return this.$watch(a[0],function(a,c,f){e[0]=a;d[0]=c;b(e,a===c?e:d,f)});q(a,function(a,b){var k=g.$watch(a,function(a,f){e[b]=a;d[b]=f;h||(h=!0,g.$evalAsync(c))});f.push(k)});return function(){for(;f.length;)f.shift()()}},
-$watchCollection:function(a,b){function c(a){e=a;var b,d,g,h;if(!w(e)){if(C(e))if(qa(e))for(f!==n&&(f=n,t=f.length=0,l++),a=e.length,t!==a&&(l++,f.length=t=a),b=0;b<a;b++)h=f[b],g=e[b],d=h!==h&&g!==g,d||h===g||(l++,f[b]=g);else{f!==p&&(f=p={},t=0,l++);a=0;for(b in e)ua.call(e,b)&&(a++,g=e[b],h=f[b],b in f?(d=h!==h&&g!==g,d||h===g||(l++,f[b]=g)):(t++,f[b]=g,l++));if(t>a)for(b in l++,f)ua.call(e,b)||(t--,delete f[b])}else f!==e&&(f=e,l++);return l}}c.$stateful=!0;var d=this,e,f,h,k=1<b.length,l=0,m=
-g(a,c),n=[],p={},r=!0,t=0;return this.$watch(m,function(){r?(r=!1,b(e,e,d)):b(e,h,d);if(k)if(C(e))if(qa(e)){h=Array(e.length);for(var a=0;a<e.length;a++)h[a]=e[a]}else for(a in h={},e)ua.call(e,a)&&(h[a]=e[a]);else h=e})},$digest:function(){var a,g,k,l,m,p,r,t=b,q,u=[],w,x;n("$digest");h.$$checkUrlChange();this===M&&null!==e&&(h.defer.cancel(e),v());c=null;do{r=!1;q=this;for(p=0;p<s.length;p++){try{x=s[p],l=x.fn,l(x.scope,x.locals)}catch(z){f(z)}c=null}s.length=0;a:do{if(p=q.$$watchers)for(p.$$digestWatchIndex=
-p.length;p.$$digestWatchIndex--;)try{if(a=p[p.$$digestWatchIndex])if(m=a.get,(g=m(q))!==(k=a.last)&&!(a.eq?sa(g,k):da(g)&&da(k)))r=!0,c=a,a.last=a.eq?ra(g,null):g,l=a.fn,l(g,k===J?g:k,q),5>t&&(w=4-t,u[w]||(u[w]=[]),u[w].push({msg:D(a.exp)?"fn: "+(a.exp.name||a.exp.toString()):a.exp,newVal:g,oldVal:k}));else if(a===c){r=!1;break a}}catch(B){f(B)}if(!(p=q.$$watchersCount&&q.$$childHead||q!==this&&q.$$nextSibling))for(;q!==this&&!(p=q.$$nextSibling);)q=q.$parent}while(q=p);if((r||s.length)&&!t--)throw M.$$phase=
-null,d("infdig",b,u);}while(r||s.length);for(M.$$phase=null;I<A.length;)try{A[I++]()}catch(F){f(F)}A.length=I=0;h.$$checkUrlChange()},$destroy:function(){if(!this.$$destroyed){var a=this.$parent;this.$broadcast("$destroy");this.$$destroyed=!0;this===M&&h.$$applicationDestroyed();p(this,-this.$$watchersCount);for(var b in this.$$listenerCount)r(this,this.$$listenerCount[b],b);a&&a.$$childHead===this&&(a.$$childHead=this.$$nextSibling);a&&a.$$childTail===this&&(a.$$childTail=this.$$prevSibling);this.$$prevSibling&&
-(this.$$prevSibling.$$nextSibling=this.$$nextSibling);this.$$nextSibling&&(this.$$nextSibling.$$prevSibling=this.$$prevSibling);this.$destroy=this.$digest=this.$apply=this.$evalAsync=this.$applyAsync=z;this.$on=this.$watch=this.$watchGroup=function(){return z};this.$$listeners={};this.$$nextSibling=null;l(this)}},$eval:function(a,b){return g(a)(this,b)},$evalAsync:function(a,b){M.$$phase||s.length||h.defer(function(){s.length&&M.$digest()});s.push({scope:this,fn:g(a),locals:b})},$$postDigest:function(a){A.push(a)},
-$apply:function(a){try{n("$apply");try{return this.$eval(a)}finally{M.$$phase=null}}catch(b){f(b)}finally{try{M.$digest()}catch(c){throw f(c),c;}}},$applyAsync:function(a){function b(){c.$eval(a)}var c=this;a&&u.push(b);a=g(a);t()},$on:function(a,b){var c=this.$$listeners[a];c||(this.$$listeners[a]=c=[]);c.push(b);var d=this;do d.$$listenerCount[a]||(d.$$listenerCount[a]=0),d.$$listenerCount[a]++;while(d=d.$parent);var e=this;return function(){var d=c.indexOf(b);-1!==d&&(c[d]=null,r(e,1,a))}},$emit:function(a,
-b){var c=[],d,e=this,g=!1,h={name:a,targetScope:e,stopPropagation:function(){g=!0},preventDefault:function(){h.defaultPrevented=!0},defaultPrevented:!1},k=ab([h],arguments,1),l,m;do{d=e.$$listeners[a]||c;h.currentScope=e;l=0;for(m=d.length;l<m;l++)if(d[l])try{d[l].apply(null,k)}catch(n){f(n)}else d.splice(l,1),l--,m--;if(g)return h.currentScope=null,h;e=e.$parent}while(e);h.currentScope=null;return h},$broadcast:function(a,b){var c=this,d=this,e={name:a,targetScope:this,preventDefault:function(){e.defaultPrevented=
-!0},defaultPrevented:!1};if(!this.$$listenerCount[a])return e;for(var g=ab([e],arguments,1),h,k;c=d;){e.currentScope=c;d=c.$$listeners[a]||[];h=0;for(k=d.length;h<k;h++)if(d[h])try{d[h].apply(null,g)}catch(l){f(l)}else d.splice(h,1),h--,k--;if(!(d=c.$$listenerCount[a]&&c.$$childHead||c!==this&&c.$$nextSibling))for(;c!==this&&!(d=c.$$nextSibling);)c=c.$parent}e.currentScope=null;return e}};var M=new m,s=M.$$asyncQueue=[],A=M.$$postDigestQueue=[],u=M.$$applyAsyncQueue=[],I=0;return M}]}function Ee(){var a=
-/^\s*(https?|ftp|mailto|tel|file):/,b=/^\s*((https?|ftp|file|blob):|data:image\/)/;this.aHrefSanitizationWhitelist=function(b){return u(b)?(a=b,this):a};this.imgSrcSanitizationWhitelist=function(a){return u(a)?(b=a,this):b};this.$get=function(){return function(d,c){var e=c?b:a,f;f=Ca(d).href;return""===f||f.match(e)?d:"unsafe:"+f}}}function Cg(a){if("self"===a)return a;if(F(a)){if(-1<a.indexOf("***"))throw ta("iwcard",a);a=Kd(a).replace(/\\\*\\\*/g,".*").replace(/\\\*/g,"[^:/.?&;]*");return new RegExp("^"+
-a+"$")}if(Xa(a))return new RegExp("^"+a.source+"$");throw ta("imatcher");}function Ld(a){var b=[];u(a)&&q(a,function(a){b.push(Cg(a))});return b}function Qf(){this.SCE_CONTEXTS=oa;var a=["self"],b=[];this.resourceUrlWhitelist=function(b){arguments.length&&(a=Ld(b));return a};this.resourceUrlBlacklist=function(a){arguments.length&&(b=Ld(a));return b};this.$get=["$injector",function(d){function c(a,b){return"self"===a?yd(b):!!a.exec(b.href)}function e(a){var b=function(a){this.$$unwrapTrustedValue=
-function(){return a}};a&&(b.prototype=new a);b.prototype.valueOf=function(){return this.$$unwrapTrustedValue()};b.prototype.toString=function(){return this.$$unwrapTrustedValue().toString()};return b}var f=function(a){throw ta("unsafe");};d.has("$sanitize")&&(f=d.get("$sanitize"));var g=e(),h={};h[oa.HTML]=e(g);h[oa.CSS]=e(g);h[oa.URL]=e(g);h[oa.JS]=e(g);h[oa.RESOURCE_URL]=e(h[oa.URL]);return{trustAs:function(a,b){var c=h.hasOwnProperty(a)?h[a]:null;if(!c)throw ta("icontext",a,b);if(null===b||w(b)||
-""===b)return b;if("string"!==typeof b)throw ta("itype",a);return new c(b)},getTrusted:function(d,e){if(null===e||w(e)||""===e)return e;var g=h.hasOwnProperty(d)?h[d]:null;if(g&&e instanceof g)return e.$$unwrapTrustedValue();if(d===oa.RESOURCE_URL){var g=Ca(e.toString()),n,p,r=!1;n=0;for(p=a.length;n<p;n++)if(c(a[n],g)){r=!0;break}if(r)for(n=0,p=b.length;n<p;n++)if(c(b[n],g)){r=!1;break}if(r)return e;throw ta("insecurl",e.toString());}if(d===oa.HTML)return f(e);throw ta("unsafe");},valueOf:function(a){return a instanceof
-g?a.$$unwrapTrustedValue():a}}}]}function Pf(){var a=!0;this.enabled=function(b){arguments.length&&(a=!!b);return a};this.$get=["$parse","$sceDelegate",function(b,d){if(a&&8>za)throw ta("iequirks");var c=pa(oa);c.isEnabled=function(){return a};c.trustAs=d.trustAs;c.getTrusted=d.getTrusted;c.valueOf=d.valueOf;a||(c.trustAs=c.getTrusted=function(a,b){return b},c.valueOf=Ya);c.parseAs=function(a,d){var e=b(d);return e.literal&&e.constant?e:b(d,function(b){return c.getTrusted(a,b)})};var e=c.parseAs,
-f=c.getTrusted,g=c.trustAs;q(oa,function(a,b){var d=Q(b);c[("parse_as_"+d).replace(xc,gb)]=function(b){return e(a,b)};c[("get_trusted_"+d).replace(xc,gb)]=function(b){return f(a,b)};c[("trust_as_"+d).replace(xc,gb)]=function(b){return g(a,b)}});return c}]}function Rf(){this.$get=["$window","$document",function(a,b){var d={},c=!((!a.nw||!a.nw.process)&&a.chrome&&(a.chrome.app&&a.chrome.app.runtime||!a.chrome.app&&a.chrome.runtime&&a.chrome.runtime.id))&&a.history&&a.history.pushState,e=Z((/android (\d+)/.exec(Q((a.navigator||
-{}).userAgent))||[])[1]),f=/Boxee/i.test((a.navigator||{}).userAgent),g=b[0]||{},h=g.body&&g.body.style,k=!1,l=!1;h&&(k=!!("transition"in h||"webkitTransition"in h),l=!!("animation"in h||"webkitAnimation"in h));return{history:!(!c||4>e||f),hasEvent:function(a){if("input"===a&&za)return!1;if(w(d[a])){var b=g.createElement("div");d[a]="on"+a in b}return d[a]},csp:Ga(),transitions:k,animations:l,android:e}}]}function Tf(){var a;this.httpOptions=function(b){return b?(a=b,this):a};this.$get=["$exceptionHandler",
-"$templateCache","$http","$q","$sce",function(b,d,c,e,f){function g(h,k){g.totalPendingRequests++;if(!F(h)||w(d.get(h)))h=f.getTrustedResourceUrl(h);var l=c.defaults&&c.defaults.transformResponse;H(l)?l=l.filter(function(a){return a!==nc}):l===nc&&(l=null);return c.get(h,S({cache:d,transformResponse:l},a)).finally(function(){g.totalPendingRequests--}).then(function(a){d.put(h,a.data);return a.data},function(a){k||(a=Dg("tpload",h,a.status,a.statusText),b(a));return e.reject(a)})}g.totalPendingRequests=
-0;return g}]}function Uf(){this.$get=["$rootScope","$browser","$location",function(a,b,d){return{findBindings:function(a,b,d){a=a.getElementsByClassName("ng-binding");var g=[];q(a,function(a){var c=ea.element(a).data("$binding");c&&q(c,function(c){d?(new RegExp("(^|\\s)"+Kd(b)+"(\\s|\\||$)")).test(c)&&g.push(a):-1!==c.indexOf(b)&&g.push(a)})});return g},findModels:function(a,b,d){for(var g=["ng-","data-ng-","ng\\:"],h=0;h<g.length;++h){var k=a.querySelectorAll("["+g[h]+"model"+(d?"=":"*=")+'"'+b+
-'"]');if(k.length)return k}},getLocation:function(){return d.url()},setLocation:function(b){b!==d.url()&&(d.url(b),a.$digest())},whenStable:function(a){b.notifyWhenNoOutstandingRequests(a)}}}]}function Vf(){this.$get=["$rootScope","$browser","$q","$$q","$exceptionHandler",function(a,b,d,c,e){function f(f,k,l){D(f)||(l=k,k=f,f=z);var m=va.call(arguments,3),n=u(l)&&!l,p=(n?c:d).defer(),r=p.promise,q;q=b.defer(function(){try{p.resolve(f.apply(null,m))}catch(b){p.reject(b),e(b)}finally{delete g[r.$$timeoutId]}n||
-a.$apply()},k);r.$$timeoutId=q;g[q]=p;return r}var g={};f.cancel=function(a){return a&&a.$$timeoutId in g?(g[a.$$timeoutId].promise.catch(z),g[a.$$timeoutId].reject("canceled"),delete g[a.$$timeoutId],b.defer.cancel(a.$$timeoutId)):!1};return f}]}function Ca(a){za&&(aa.setAttribute("href",a),a=aa.href);aa.setAttribute("href",a);return{href:aa.href,protocol:aa.protocol?aa.protocol.replace(/:$/,""):"",host:aa.host,search:aa.search?aa.search.replace(/^\?/,""):"",hash:aa.hash?aa.hash.replace(/^#/,""):
-"",hostname:aa.hostname,port:aa.port,pathname:"/"===aa.pathname.charAt(0)?aa.pathname:"/"+aa.pathname}}function yd(a){a=F(a)?Ca(a):a;return a.protocol===Md.protocol&&a.host===Md.host}function Wf(){this.$get=la(x)}function Nd(a){function b(a){try{return decodeURIComponent(a)}catch(b){return a}}var d=a[0]||{},c={},e="";return function(){var a,g,h,k,l;try{a=d.cookie||""}catch(m){a=""}if(a!==e)for(e=a,a=e.split("; "),c={},h=0;h<a.length;h++)g=a[h],k=g.indexOf("="),0<k&&(l=b(g.substring(0,k)),w(c[l])&&
-(c[l]=b(g.substring(k+1))));return c}}function $f(){this.$get=Nd}function cd(a){function b(d,c){if(C(d)){var e={};q(d,function(a,c){e[c]=b(c,a)});return e}return a.factory(d+"Filter",c)}this.register=b;this.$get=["$injector",function(a){return function(b){return a.get(b+"Filter")}}];b("currency",Od);b("date",Pd);b("filter",Eg);b("json",Fg);b("limitTo",Gg);b("lowercase",Hg);b("number",Qd);b("orderBy",Rd);b("uppercase",Ig)}function Eg(){return function(a,b,d,c){if(!qa(a)){if(null==a)return a;throw L("filter")("notarray",
-a);}c=c||"$";var e;switch(yc(b)){case "function":break;case "boolean":case "null":case "number":case "string":e=!0;case "object":b=Jg(b,d,c,e);break;default:return a}return Array.prototype.filter.call(a,b)}}function Jg(a,b,d,c){var e=C(a)&&d in a;!0===b?b=sa:D(b)||(b=function(a,b){if(w(a))return!1;if(null===a||null===b)return a===b;if(C(b)||C(a)&&!Wb(a))return!1;a=Q(""+a);b=Q(""+b);return-1!==a.indexOf(b)});return function(f){return e&&!C(f)?Ea(f,a[d],b,d,!1):Ea(f,a,b,d,c)}}function Ea(a,b,d,c,e,
-f){var g=yc(a),h=yc(b);if("string"===h&&"!"===b.charAt(0))return!Ea(a,b.substring(1),d,c,e);if(H(a))return a.some(function(a){return Ea(a,b,d,c,e)});switch(g){case "object":var k;if(e){for(k in a)if(k.charAt&&"$"!==k.charAt(0)&&Ea(a[k],b,d,c,!0))return!0;return f?!1:Ea(a,b,d,c,!1)}if("object"===h){for(k in b)if(f=b[k],!D(f)&&!w(f)&&(g=k===c,!Ea(g?a:a[k],f,d,c,g,g)))return!1;return!0}return d(a,b);case "function":return!1;default:return d(a,b)}}function yc(a){return null===a?"null":typeof a}function Od(a){var b=
-a.NUMBER_FORMATS;return function(a,c,e){w(c)&&(c=b.CURRENCY_SYM);w(e)&&(e=b.PATTERNS[1].maxFrac);return null==a?a:Sd(a,b.PATTERNS[1],b.GROUP_SEP,b.DECIMAL_SEP,e).replace(/\u00A4/g,c)}}function Qd(a){var b=a.NUMBER_FORMATS;return function(a,c){return null==a?a:Sd(a,b.PATTERNS[0],b.GROUP_SEP,b.DECIMAL_SEP,c)}}function Kg(a){var b=0,d,c,e,f,g;-1<(c=a.indexOf(Td))&&(a=a.replace(Td,""));0<(e=a.search(/e/i))?(0>c&&(c=e),c+=+a.slice(e+1),a=a.substring(0,e)):0>c&&(c=a.length);for(e=0;a.charAt(e)===zc;e++);
-if(e===(g=a.length))d=[0],c=1;else{for(g--;a.charAt(g)===zc;)g--;c-=e;d=[];for(f=0;e<=g;e++,f++)d[f]=+a.charAt(e)}c>Ud&&(d=d.splice(0,Ud-1),b=c-1,c=1);return{d:d,e:b,i:c}}function Lg(a,b,d,c){var e=a.d,f=e.length-a.i;b=w(b)?Math.min(Math.max(d,f),c):+b;d=b+a.i;c=e[d];if(0<d){e.splice(Math.max(a.i,d));for(var g=d;g<e.length;g++)e[g]=0}else for(f=Math.max(0,f),a.i=1,e.length=Math.max(1,d=b+1),e[0]=0,g=1;g<d;g++)e[g]=0;if(5<=c)if(0>d-1){for(c=0;c>d;c--)e.unshift(0),a.i++;e.unshift(1);a.i++}else e[d-
-1]++;for(;f<Math.max(0,b);f++)e.push(0);if(b=e.reduceRight(function(a,b,c,d){b+=a;d[c]=b%10;return Math.floor(b/10)},0))e.unshift(b),a.i++}function Sd(a,b,d,c,e){if(!F(a)&&!ba(a)||isNaN(a))return"";var f=!isFinite(a),g=!1,h=Math.abs(a)+"",k="";if(f)k="\u221e";else{g=Kg(h);Lg(g,e,b.minFrac,b.maxFrac);k=g.d;h=g.i;e=g.e;f=[];for(g=k.reduce(function(a,b){return a&&!b},!0);0>h;)k.unshift(0),h++;0<h?f=k.splice(h,k.length):(f=k,k=[0]);h=[];for(k.length>=b.lgSize&&h.unshift(k.splice(-b.lgSize,k.length).join(""));k.length>
-b.gSize;)h.unshift(k.splice(-b.gSize,k.length).join(""));k.length&&h.unshift(k.join(""));k=h.join(d);f.length&&(k+=c+f.join(""));e&&(k+="e+"+e)}return 0>a&&!g?b.negPre+k+b.negSuf:b.posPre+k+b.posSuf}function Kb(a,b,d,c){var e="";if(0>a||c&&0>=a)c?a=-a+1:(a=-a,e="-");for(a=""+a;a.length<b;)a=zc+a;d&&(a=a.substr(a.length-b));return e+a}function Y(a,b,d,c,e){d=d||0;return function(f){f=f["get"+a]();if(0<d||f>-d)f+=d;0===f&&-12===d&&(f=12);return Kb(f,b,c,e)}}function mb(a,b,d){return function(c,e){var f=
-c["get"+a](),g=ub((d?"STANDALONE":"")+(b?"SHORT":"")+a);return e[g][f]}}function Vd(a){var b=(new Date(a,0,1)).getDay();return new Date(a,0,(4>=b?5:12)-b)}function Wd(a){return function(b){var d=Vd(b.getFullYear());b=+new Date(b.getFullYear(),b.getMonth(),b.getDate()+(4-b.getDay()))-+d;b=1+Math.round(b/6048E5);return Kb(b,a)}}function Ac(a,b){return 0>=a.getFullYear()?b.ERAS[0]:b.ERAS[1]}function Pd(a){function b(a){var b;if(b=a.match(d)){a=new Date(0);var f=0,g=0,h=b[8]?a.setUTCFullYear:a.setFullYear,
-k=b[8]?a.setUTCHours:a.setHours;b[9]&&(f=Z(b[9]+b[10]),g=Z(b[9]+b[11]));h.call(a,Z(b[1]),Z(b[2])-1,Z(b[3]));f=Z(b[4]||0)-f;g=Z(b[5]||0)-g;h=Z(b[6]||0);b=Math.round(1E3*parseFloat("0."+(b[7]||0)));k.call(a,f,g,h,b)}return a}var d=/^(\d{4})-?(\d\d)-?(\d\d)(?:T(\d\d)(?::?(\d\d)(?::?(\d\d)(?:\.(\d+))?)?)?(Z|([+-])(\d\d):?(\d\d))?)?$/;return function(c,d,f){var g="",h=[],k,l;d=d||"mediumDate";d=a.DATETIME_FORMATS[d]||d;F(c)&&(c=Mg.test(c)?Z(c):b(c));ba(c)&&(c=new Date(c));if(!ga(c)||!isFinite(c.getTime()))return c;
-for(;d;)(l=Ng.exec(d))?(h=ab(h,l,1),d=h.pop()):(h.push(d),d=null);var m=c.getTimezoneOffset();f&&(m=Pc(f,m),c=Yb(c,f,!0));q(h,function(b){k=Og[b];g+=k?k(c,a.DATETIME_FORMATS,m):"''"===b?"'":b.replace(/(^'|'$)/g,"").replace(/''/g,"'")});return g}}function Fg(){return function(a,b){w(b)&&(b=2);return cb(a,b)}}function Gg(){return function(a,b,d){b=Infinity===Math.abs(Number(b))?Number(b):Z(b);if(da(b))return a;ba(a)&&(a=a.toString());if(!qa(a))return a;d=!d||isNaN(d)?0:Z(d);d=0>d?Math.max(0,a.length+
-d):d;return 0<=b?Bc(a,d,d+b):0===d?Bc(a,b,a.length):Bc(a,Math.max(0,d+b),d)}}function Bc(a,b,d){return F(a)?a.slice(b,d):va.call(a,b,d)}function Rd(a){function b(b){return b.map(function(b){var c=1,d=Ya;if(D(b))d=b;else if(F(b)){if("+"===b.charAt(0)||"-"===b.charAt(0))c="-"===b.charAt(0)?-1:1,b=b.substring(1);if(""!==b&&(d=a(b),d.constant))var e=d(),d=function(a){return a[e]}}return{get:d,descending:c}})}function d(a){switch(typeof a){case "number":case "boolean":case "string":return!0;default:return!1}}
-function c(a,b){var c=0,d=a.type,k=b.type;if(d===k){var k=a.value,l=b.value;"string"===d?(k=k.toLowerCase(),l=l.toLowerCase()):"object"===d&&(C(k)&&(k=a.index),C(l)&&(l=b.index));k!==l&&(c=k<l?-1:1)}else c=d<k?-1:1;return c}return function(a,f,g,h){if(null==a)return a;if(!qa(a))throw L("orderBy")("notarray",a);H(f)||(f=[f]);0===f.length&&(f=["+"]);var k=b(f),l=g?-1:1,m=D(h)?h:c;a=Array.prototype.map.call(a,function(a,b){return{value:a,tieBreaker:{value:b,type:"number",index:b},predicateValues:k.map(function(c){var e=
-c.get(a);c=typeof e;if(null===e)c="string",e="null";else if("object"===c)a:{if(D(e.valueOf)&&(e=e.valueOf(),d(e)))break a;Wb(e)&&(e=e.toString(),d(e))}return{value:e,type:c,index:b}})}});a.sort(function(a,b){for(var c=0,d=k.length;c<d;c++){var e=m(a.predicateValues[c],b.predicateValues[c]);if(e)return e*k[c].descending*l}return m(a.tieBreaker,b.tieBreaker)*l});return a=a.map(function(a){return a.value})}}function Qa(a){D(a)&&(a={link:a});a.restrict=a.restrict||"AC";return la(a)}function Lb(a,b,d,
-c,e){this.$$controls=[];this.$error={};this.$$success={};this.$pending=void 0;this.$name=e(b.name||b.ngForm||"")(d);this.$dirty=!1;this.$valid=this.$pristine=!0;this.$submitted=this.$invalid=!1;this.$$parentForm=Mb;this.$$element=a;this.$$animate=c;Xd(this)}function Xd(a){a.$$classCache={};a.$$classCache[Yd]=!(a.$$classCache[nb]=a.$$element.hasClass(nb))}function Zd(a){function b(a,b,c){c&&!a.$$classCache[b]?(a.$$animate.addClass(a.$$element,b),a.$$classCache[b]=!0):!c&&a.$$classCache[b]&&(a.$$animate.removeClass(a.$$element,
-b),a.$$classCache[b]=!1)}function d(a,c,d){c=c?"-"+Tc(c,"-"):"";b(a,nb+c,!0===d);b(a,Yd+c,!1===d)}var c=a.set,e=a.unset;a.clazz.prototype.$setValidity=function(a,g,h){w(g)?(this.$pending||(this.$pending={}),c(this.$pending,a,h)):(this.$pending&&e(this.$pending,a,h),$d(this.$pending)&&(this.$pending=void 0));Ha(g)?g?(e(this.$error,a,h),c(this.$$success,a,h)):(c(this.$error,a,h),e(this.$$success,a,h)):(e(this.$error,a,h),e(this.$$success,a,h));this.$pending?(b(this,"ng-pending",!0),this.$valid=this.$invalid=
-void 0,d(this,"",null)):(b(this,"ng-pending",!1),this.$valid=$d(this.$error),this.$invalid=!this.$valid,d(this,"",this.$valid));g=this.$pending&&this.$pending[a]?void 0:this.$error[a]?!1:this.$$success[a]?!0:null;d(this,a,g);this.$$parentForm.$setValidity(a,g,this)}}function $d(a){if(a)for(var b in a)if(

<TRUNCATED>

---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[51/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects.
Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ce23d9ad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ce23d9ad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ce23d9ad

Branch: refs/heads/HDFS-7240
Commit: ce23d9adf004358013825f2a1ec684f35a953b4a
Parents: a5dfae6
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Mar 15 09:23:33 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Thu Mar 15 09:23:33 2018 -0700

----------------------------------------------------------------------
 dev-support/bin/dist-layout-stitching           |   19 +-
 hadoop-cblock/pom.xml                           |   56 +
 .../server/dev-support/findbugsExcludeFile.xml  |   21 +
 hadoop-cblock/server/pom.xml                    |  169 +++
 .../apache/hadoop/cblock/CBlockConfigKeys.java  |  222 +++
 .../org/apache/hadoop/cblock/CBlockManager.java |  426 ++++++
 .../org/apache/hadoop/cblock/CblockUtils.java   |  129 ++
 ...ckServiceProtocolClientSideTranslatorPB.java |  135 ++
 .../cblock/client/CBlockVolumeClient.java       |   83 ++
 .../hadoop/cblock/client/package-info.java      |   18 +
 .../cblock/exception/CBlockException.java       |   29 +
 .../hadoop/cblock/exception/package-info.java   |   18 +
 .../cblock/jscsiHelper/BlockWriterTask.java     |  175 +++
 ...ockClientProtocolClientSideTranslatorPB.java |  147 ++
 .../cblock/jscsiHelper/CBlockIStorageImpl.java  |  440 ++++++
 .../jscsiHelper/CBlockManagerHandler.java       |   50 +
 .../cblock/jscsiHelper/CBlockTargetMetrics.java |  334 +++++
 .../cblock/jscsiHelper/CBlockTargetServer.java  |  128 ++
 .../jscsiHelper/ContainerCacheFlusher.java      |  599 ++++++++
 .../cblock/jscsiHelper/SCSITargetDaemon.java    |  132 ++
 .../cblock/jscsiHelper/cache/CacheModule.java   |   52 +
 .../cblock/jscsiHelper/cache/LogicalBlock.java  |   50 +
 .../cache/impl/AsyncBlockWriter.java            |  221 +++
 .../cache/impl/BlockBufferFlushTask.java        |  118 ++
 .../cache/impl/BlockBufferManager.java          |  184 +++
 .../cache/impl/CBlockLocalCache.java            |  577 ++++++++
 .../jscsiHelper/cache/impl/DiskBlock.java       |   77 ++
 .../jscsiHelper/cache/impl/SyncBlockReader.java |  263 ++++
 .../jscsiHelper/cache/impl/package-info.java    |   18 +
 .../cblock/jscsiHelper/cache/package-info.java  |   18 +
 .../hadoop/cblock/jscsiHelper/package-info.java |   18 +
 .../cblock/kubernetes/DynamicProvisioner.java   |  331 +++++
 .../hadoop/cblock/kubernetes/package-info.java  |   23 +
 .../hadoop/cblock/meta/ContainerDescriptor.java |  107 ++
 .../hadoop/cblock/meta/VolumeDescriptor.java    |  269 ++++
 .../apache/hadoop/cblock/meta/VolumeInfo.java   |   79 ++
 .../apache/hadoop/cblock/meta/package-info.java |   18 +
 .../org/apache/hadoop/cblock/package-info.java  |   18 +
 .../cblock/proto/CBlockClientProtocol.java      |   38 +
 .../cblock/proto/CBlockServiceProtocol.java     |   45 +
 .../cblock/proto/MountVolumeResponse.java       |   79 ++
 .../hadoop/cblock/proto/package-info.java       |   18 +
 .../CBlockClientServerProtocolPB.java           |   37 +
 ...entServerProtocolServerSideTranslatorPB.java |  116 ++
 .../protocolPB/CBlockServiceProtocolPB.java     |   35 +
 ...ckServiceProtocolServerSideTranslatorPB.java |  159 +++
 .../hadoop/cblock/protocolPB/package-info.java  |   18 +
 .../hadoop/cblock/storage/StorageManager.java   |  427 ++++++
 .../hadoop/cblock/storage/package-info.java     |   18 +
 .../org/apache/hadoop/cblock/util/KeyUtil.java  |   49 +
 .../apache/hadoop/cblock/util/package-info.java |   18 +
 .../main/proto/CBlockClientServerProtocol.proto |   93 ++
 .../src/main/proto/CBlockServiceProtocol.proto  |  133 ++
 .../src/main/resources/cblock-default.xml       |  347 +++++
 .../apache/hadoop/cblock/TestBufferManager.java |  456 ++++++
 .../cblock/TestCBlockConfigurationFields.java   |   35 +
 .../hadoop/cblock/TestCBlockReadWrite.java      |  377 +++++
 .../apache/hadoop/cblock/TestCBlockServer.java  |  212 +++
 .../cblock/TestCBlockServerPersistence.java     |  132 ++
 .../hadoop/cblock/TestLocalBlockCache.java      |  444 ++++++
 .../kubernetes/TestDynamicProvisioner.java      |   74 +
 .../cblock/util/ContainerLookUpService.java     |   73 +
 .../hadoop/cblock/util/MockStorageClient.java   |  176 +++
 .../dynamicprovisioner/expected1-pv.json        |   37 +
 .../dynamicprovisioner/input1-pvc.json          |   38 +
 hadoop-cblock/tools/pom.xml                     |   42 +
 .../org/apache/hadoop/cblock/cli/CBlockCli.java |  265 ++++
 .../apache/hadoop/cblock/cli/package-info.java  |   18 +
 .../org/apache/hadoop/cblock/TestCBlockCLI.java |  242 ++++
 .../src/main/bin/hadoop-functions.sh            |    7 +
 .../org/apache/hadoop/conf/ConfServlet.java     |  103 +-
 .../apache/hadoop/conf/OzoneConfiguration.java  |  155 ---
 .../java/org/apache/hadoop/fs/FileUtil.java     |   49 +-
 .../main/java/org/apache/hadoop/ipc/RPC.java    |    2 +-
 hadoop-dist/pom.xml                             |   38 +
 .../src/main/compose/cblock/docker-compose.yaml |   20 +-
 .../src/main/compose/cblock/docker-config       |    2 +-
 .../src/main/compose/ozone/docker-compose.yaml  |   16 +-
 .../src/main/compose/ozone/docker-config        |    1 +
 .../dev-support/findbugsExcludeFile.xml         |    8 +-
 hadoop-hdfs-project/hadoop-hdfs-client/pom.xml  |   41 +-
 .../apache/hadoop/cblock/CBlockConfigKeys.java  |  221 ---
 .../org/apache/hadoop/cblock/package-info.java  |   22 -
 .../apache/hadoop/hdfs/protocol/DatanodeID.java |   26 +-
 .../hadoop/hdfs/protocolPB/PBHelperClient.java  |   14 +-
 .../java/org/apache/hadoop/ozone/OzoneAcl.java  |  231 ----
 .../apache/hadoop/ozone/OzoneConfigKeys.java    |  245 ----
 .../org/apache/hadoop/ozone/OzoneConsts.java    |  162 ---
 .../apache/hadoop/ozone/client/BucketArgs.java  |  123 --
 .../apache/hadoop/ozone/client/ObjectStore.java |  210 ---
 .../apache/hadoop/ozone/client/OzoneBucket.java |  360 -----
 .../apache/hadoop/ozone/client/OzoneClient.java |  103 --
 .../hadoop/ozone/client/OzoneClientFactory.java |  308 -----
 .../client/OzoneClientInvocationHandler.java    |   62 -
 .../hadoop/ozone/client/OzoneClientUtils.java   |  874 ------------
 .../apache/hadoop/ozone/client/OzoneKey.java    |  119 --
 .../apache/hadoop/ozone/client/OzoneQuota.java  |  203 ---
 .../apache/hadoop/ozone/client/OzoneVolume.java |  293 ----
 .../hadoop/ozone/client/ReplicationFactor.java  |   63 -
 .../hadoop/ozone/client/ReplicationType.java    |   28 -
 .../apache/hadoop/ozone/client/VolumeArgs.java  |  128 --
 .../ozone/client/io/ChunkGroupInputStream.java  |  319 -----
 .../ozone/client/io/ChunkGroupOutputStream.java |  492 -------
 .../ozone/client/io/LengthInputStream.java      |   49 -
 .../client/io/OzoneContainerTranslation.java    |   51 -
 .../ozone/client/io/OzoneInputStream.java       |   60 -
 .../ozone/client/io/OzoneOutputStream.java      |   64 -
 .../hadoop/ozone/client/io/package-info.java    |   23 -
 .../hadoop/ozone/client/package-info.java       |   23 -
 .../ozone/client/protocol/ClientProtocol.java   |  323 -----
 .../ozone/client/protocol/package-info.java     |   23 -
 .../client/rest/DefaultRestServerSelector.java  |   36 -
 .../ozone/client/rest/OzoneException.java       |  267 ----
 .../ozone/client/rest/OzoneExceptionMapper.java |   45 -
 .../hadoop/ozone/client/rest/RestClient.java    |  799 -----------
 .../ozone/client/rest/RestServerSelector.java   |   40 -
 .../client/rest/exceptions/package-info.java    |   22 -
 .../ozone/client/rest/headers/Header.java       |   71 -
 .../ozone/client/rest/headers/package-info.java |   22 -
 .../hadoop/ozone/client/rest/package-info.java  |   23 -
 .../ozone/client/rest/response/BucketInfo.java  |  230 ---
 .../ozone/client/rest/response/KeyInfo.java     |  218 ---
 .../ozone/client/rest/response/VolumeInfo.java  |  215 ---
 .../ozone/client/rest/response/VolumeOwner.java |   60 -
 .../client/rest/response/package-info.java      |   24 -
 .../hadoop/ozone/client/rpc/RpcClient.java      |  567 --------
 .../hadoop/ozone/client/rpc/package-info.java   |   23 -
 .../apache/hadoop/ozone/common/BlockGroup.java  |   87 --
 .../ozone/common/DeleteBlockGroupResult.java    |   94 --
 .../hadoop/ozone/common/package-info.java       |   21 -
 .../InvalidStateTransitionException.java        |   42 -
 .../ozone/common/statemachine/StateMachine.java |   68 -
 .../ozone/common/statemachine/package-info.java |   21 -
 .../apache/hadoop/ozone/ksm/KSMConfigKeys.java  |   81 --
 .../hadoop/ozone/ksm/helpers/KsmBucketArgs.java |  233 ----
 .../hadoop/ozone/ksm/helpers/KsmBucketInfo.java |  235 ----
 .../hadoop/ozone/ksm/helpers/KsmKeyArgs.java    |  119 --
 .../hadoop/ozone/ksm/helpers/KsmKeyInfo.java    |  243 ----
 .../ozone/ksm/helpers/KsmKeyLocationInfo.java   |  136 --
 .../ksm/helpers/KsmKeyLocationInfoGroup.java    |  118 --
 .../ozone/ksm/helpers/KsmOzoneAclMap.java       |  110 --
 .../hadoop/ozone/ksm/helpers/KsmVolumeArgs.java |  223 ---
 .../ozone/ksm/helpers/OpenKeySession.java       |   50 -
 .../hadoop/ozone/ksm/helpers/ServiceInfo.java   |  237 ----
 .../hadoop/ozone/ksm/helpers/VolumeArgs.java    |  140 --
 .../hadoop/ozone/ksm/helpers/package-info.java  |   18 -
 .../apache/hadoop/ozone/ksm/package-info.java   |   21 -
 .../ksm/protocol/KeySpaceManagerProtocol.java   |  245 ----
 .../hadoop/ozone/ksm/protocol/package-info.java |   19 -
 ...ceManagerProtocolClientSideTranslatorPB.java |  744 ----------
 .../protocolPB/KeySpaceManagerProtocolPB.java   |   34 -
 .../ozone/ksm/protocolPB/package-info.java      |   19 -
 .../org/apache/hadoop/ozone/package-info.java   |   22 -
 .../hadoop/ozone/protocolPB/KSMPBHelper.java    |  113 --
 .../hadoop/ozone/protocolPB/package-info.java   |   24 -
 .../container/ContainerStates/ContainerID.java  |   97 --
 .../container/ContainerStates/package-info.java |   22 -
 .../org/apache/hadoop/scm/ScmConfigKeys.java    |  260 ----
 .../java/org/apache/hadoop/scm/ScmInfo.java     |   81 --
 .../org/apache/hadoop/scm/XceiverClient.java    |  192 ---
 .../apache/hadoop/scm/XceiverClientHandler.java |  204 ---
 .../hadoop/scm/XceiverClientInitializer.java    |   72 -
 .../apache/hadoop/scm/XceiverClientManager.java |  219 ---
 .../apache/hadoop/scm/XceiverClientMetrics.java |   92 --
 .../apache/hadoop/scm/XceiverClientRatis.java   |  264 ----
 .../org/apache/hadoop/scm/XceiverClientSpi.java |  129 --
 .../scm/client/ContainerOperationClient.java    |  403 ------
 .../org/apache/hadoop/scm/client/ScmClient.java |  139 --
 .../apache/hadoop/scm/client/package-info.java  |   24 -
 .../common/helpers/AllocatedBlock.java          |   77 --
 .../container/common/helpers/ContainerInfo.java |  333 -----
 .../common/helpers/DeleteBlockResult.java       |   52 -
 .../scm/container/common/helpers/Pipeline.java  |  253 ----
 .../common/helpers/PipelineChannel.java         |  121 --
 .../helpers/StorageContainerException.java      |  104 --
 .../container/common/helpers/package-info.java  |   22 -
 .../org/apache/hadoop/scm/package-info.java     |   24 -
 .../hadoop/scm/protocol/LocatedContainer.java   |  127 --
 .../scm/protocol/ScmBlockLocationProtocol.java  |   72 -
 .../hadoop/scm/protocol/ScmLocatedBlock.java    |  100 --
 .../StorageContainerLocationProtocol.java       |  123 --
 .../hadoop/scm/protocol/package-info.java       |   19 -
 ...kLocationProtocolClientSideTranslatorPB.java |  207 ---
 .../protocolPB/ScmBlockLocationProtocolPB.java  |   35 -
 ...rLocationProtocolClientSideTranslatorPB.java |  304 ----
 .../StorageContainerLocationProtocolPB.java     |   34 -
 .../hadoop/scm/protocolPB/package-info.java     |   24 -
 .../hadoop/scm/storage/ChunkInputStream.java    |  261 ----
 .../hadoop/scm/storage/ChunkOutputStream.java   |  227 ---
 .../scm/storage/ContainerProtocolCalls.java     |  397 ------
 .../apache/hadoop/scm/storage/package-info.java |   23 -
 .../main/java/org/apache/ratis/RatisHelper.java |  128 --
 .../java/org/apache/ratis/package-info.java     |   22 -
 .../com/google/protobuf/ShadedProtoUtil.java    |   38 -
 .../com/google/protobuf/package-info.java       |   22 -
 .../main/proto/DatanodeContainerProtocol.proto  |  415 ------
 .../main/proto/KeySpaceManagerProtocol.proto    |  457 ------
 .../src/main/proto/Ozone.proto                  |  161 ---
 .../main/proto/ScmBlockLocationProtocol.proto   |  166 ---
 .../StorageContainerLocationProtocol.proto      |  214 ---
 .../src/main/proto/hdfs.proto                   |    1 +
 .../org/apache/hadoop/ozone/TestOzoneAcls.java  |  141 --
 .../hadoop/ozone/common/TestStateMachine.java   |  104 --
 .../java/org/apache/hadoop/scm/TestArchive.java |  114 --
 .../org/apache/hadoop/scm/package-info.java     |   21 -
 .../dev-support/findbugsExcludeFile.xml         |    9 -
 hadoop-hdfs-project/hadoop-hdfs/pom.xml         |  124 +-
 .../hadoop-hdfs/src/main/bin/hdfs               |   47 -
 .../hadoop-hdfs/src/main/bin/start-balancer.sh  |    4 -
 .../hadoop-hdfs/src/main/bin/start-ozone.sh     |  105 --
 .../hadoop-hdfs/src/main/bin/stop-balancer.sh   |    4 -
 .../hadoop-hdfs/src/main/bin/stop-ozone.sh      |   99 --
 .../hadoop-hdfs/src/main/conf/ozone-site.xml    |   24 -
 .../org/apache/hadoop/cblock/CBlockManager.java |  421 ------
 .../org/apache/hadoop/cblock/cli/CBlockCli.java |  289 ----
 .../apache/hadoop/cblock/cli/package-info.java  |   18 -
 ...ckServiceProtocolClientSideTranslatorPB.java |  135 --
 .../cblock/client/CBlockVolumeClient.java       |   82 --
 .../hadoop/cblock/client/package-info.java      |   18 -
 .../cblock/exception/CBlockException.java       |   29 -
 .../hadoop/cblock/exception/package-info.java   |   18 -
 .../cblock/jscsiHelper/BlockWriterTask.java     |  175 ---
 ...ockClientProtocolClientSideTranslatorPB.java |  147 --
 .../cblock/jscsiHelper/CBlockIStorageImpl.java  |  440 ------
 .../jscsiHelper/CBlockManagerHandler.java       |   50 -
 .../cblock/jscsiHelper/CBlockTargetMetrics.java |  334 -----
 .../cblock/jscsiHelper/CBlockTargetServer.java  |  128 --
 .../jscsiHelper/ContainerCacheFlusher.java      |  599 --------
 .../cblock/jscsiHelper/SCSITargetDaemon.java    |  129 --
 .../cblock/jscsiHelper/cache/CacheModule.java   |   52 -
 .../cblock/jscsiHelper/cache/LogicalBlock.java  |   50 -
 .../cache/impl/AsyncBlockWriter.java            |  221 ---
 .../cache/impl/BlockBufferFlushTask.java        |  118 --
 .../cache/impl/BlockBufferManager.java          |  184 ---
 .../cache/impl/CBlockLocalCache.java            |  577 --------
 .../jscsiHelper/cache/impl/DiskBlock.java       |   77 --
 .../jscsiHelper/cache/impl/SyncBlockReader.java |  263 ----
 .../jscsiHelper/cache/impl/package-info.java    |   18 -
 .../cblock/jscsiHelper/cache/package-info.java  |   18 -
 .../hadoop/cblock/jscsiHelper/package-info.java |   18 -
 .../cblock/kubernetes/DynamicProvisioner.java   |  330 -----
 .../hadoop/cblock/kubernetes/package-info.java  |   23 -
 .../hadoop/cblock/meta/ContainerDescriptor.java |  107 --
 .../hadoop/cblock/meta/VolumeDescriptor.java    |  269 ----
 .../apache/hadoop/cblock/meta/VolumeInfo.java   |   79 --
 .../apache/hadoop/cblock/meta/package-info.java |   18 -
 .../org/apache/hadoop/cblock/package-info.java  |   18 -
 .../cblock/proto/CBlockClientProtocol.java      |   38 -
 .../cblock/proto/CBlockServiceProtocol.java     |   45 -
 .../cblock/proto/MountVolumeResponse.java       |   79 --
 .../hadoop/cblock/proto/package-info.java       |   18 -
 .../CBlockClientServerProtocolPB.java           |   37 -
 ...entServerProtocolServerSideTranslatorPB.java |  116 --
 .../protocolPB/CBlockServiceProtocolPB.java     |   35 -
 ...ckServiceProtocolServerSideTranslatorPB.java |  159 ---
 .../hadoop/cblock/protocolPB/package-info.java  |   18 -
 .../hadoop/cblock/storage/StorageManager.java   |  427 ------
 .../hadoop/cblock/storage/package-info.java     |   18 -
 .../org/apache/hadoop/cblock/util/KeyUtil.java  |   49 -
 .../apache/hadoop/cblock/util/package-info.java |   18 -
 .../java/org/apache/hadoop/hdfs/DFSUtil.java    |   20 -
 .../hdfs/server/common/HdfsServerConstants.java |    7 +-
 .../hadoop/hdfs/server/datanode/DataNode.java   |  132 +-
 .../server/datanode/DataNodeServicePlugin.java  |   48 +
 .../server/datanode/ObjectStoreHandler.java     |  189 ---
 .../datanode/fsdataset/impl/FsDatasetImpl.java  |    2 +-
 .../server/datanode/web/DatanodeHttpServer.java |   19 +-
 .../web/RestCsrfPreventionFilterHandler.java    |    2 +-
 .../hdfs/server/datanode/web/URLDispatcher.java |   76 +-
 .../hdfs/server/protocol/NamespaceInfo.java     |   12 +-
 .../org/apache/hadoop/hdfs/tools/GetConf.java   |   49 +-
 .../InconsistentStorageStateException.java      |   51 -
 .../org/apache/hadoop/ozone/common/Storage.java |  249 ----
 .../apache/hadoop/ozone/common/StorageInfo.java |  184 ---
 .../hadoop/ozone/common/package-info.java       |   18 -
 .../container/common/helpers/ChunkInfo.java     |  185 ---
 .../container/common/helpers/ChunkUtils.java    |  345 -----
 .../container/common/helpers/ContainerData.java |  326 -----
 .../common/helpers/ContainerMetrics.java        |  121 --
 .../common/helpers/ContainerReport.java         |  218 ---
 .../common/helpers/ContainerUtils.java          |  477 -------
 .../helpers/DeletedContainerBlocksSummary.java  |  103 --
 .../container/common/helpers/FileUtils.java     |   81 --
 .../ozone/container/common/helpers/KeyData.java |  170 ---
 .../container/common/helpers/KeyUtils.java      |  147 --
 .../container/common/helpers/package-info.java  |   22 -
 .../container/common/impl/ChunkManagerImpl.java |  232 ----
 .../impl/ContainerLocationManagerImpl.java      |  149 --
 .../common/impl/ContainerManagerImpl.java       | 1101 ---------------
 .../common/impl/ContainerReportManagerImpl.java |   87 --
 .../container/common/impl/ContainerStatus.java  |  217 ---
 .../common/impl/ContainerStorageLocation.java   |  203 ---
 .../ozone/container/common/impl/Dispatcher.java |  708 ----------
 .../container/common/impl/KeyManagerImpl.java   |  201 ---
 .../RandomContainerDeletionChoosingPolicy.java  |   69 -
 .../common/impl/StorageLocationReport.java      |   63 -
 ...NOrderedContainerDeletionChoosingPolicy.java |   90 --
 .../container/common/impl/package-info.java     |   22 -
 .../common/interfaces/ChunkManager.java         |   75 -
 .../ContainerDeletionChoosingPolicy.java        |   45 -
 .../common/interfaces/ContainerDispatcher.java  |   49 -
 .../interfaces/ContainerLocationManager.java    |   58 -
 .../ContainerLocationManagerMXBean.java         |   36 -
 .../common/interfaces/ContainerManager.java     |  276 ----
 .../interfaces/ContainerReportManager.java      |   32 -
 .../container/common/interfaces/KeyManager.java |   75 -
 .../common/interfaces/package-info.java         |   20 -
 .../ozone/container/common/package-info.java    |   28 -
 .../statemachine/DatanodeStateMachine.java      |  385 ------
 .../statemachine/EndpointStateMachine.java      |  293 ----
 .../statemachine/EndpointStateMachineMBean.java |   34 -
 .../statemachine/SCMConnectionManager.java      |  201 ---
 .../SCMConnectionManagerMXBean.java             |   27 -
 .../common/statemachine/StateContext.java       |  281 ----
 .../background/BlockDeletingService.java        |  237 ----
 .../statemachine/background/package-info.java   |   18 -
 .../commandhandler/CloseContainerHandler.java   |  112 --
 .../commandhandler/CommandDispatcher.java       |  177 ---
 .../commandhandler/CommandHandler.java          |   59 -
 .../commandhandler/ContainerReportHandler.java  |  110 --
 .../DeleteBlocksCommandHandler.java             |  204 ---
 .../commandhandler/package-info.java            |   18 -
 .../common/statemachine/package-info.java       |   28 -
 .../container/common/states/DatanodeState.java  |   55 -
 .../states/datanode/InitDatanodeState.java      |  165 ---
 .../states/datanode/RunningDatanodeState.java   |  253 ----
 .../common/states/datanode/package-info.java    |   21 -
 .../states/endpoint/HeartbeatEndpointTask.java  |  270 ----
 .../states/endpoint/RegisterEndpointTask.java   |  198 ---
 .../states/endpoint/VersionEndpointTask.java    |   66 -
 .../common/states/endpoint/package-info.java    |   20 -
 .../container/common/states/package-info.java   |   18 -
 .../common/transport/server/XceiverServer.java  |  128 --
 .../transport/server/XceiverServerHandler.java  |   80 --
 .../server/XceiverServerInitializer.java        |   61 -
 .../transport/server/XceiverServerSpi.java      |   43 -
 .../common/transport/server/package-info.java   |   24 -
 .../server/ratis/ContainerStateMachine.java     |  290 ----
 .../server/ratis/XceiverServerRatis.java        |  214 ---
 .../transport/server/ratis/package-info.java    |   23 -
 .../container/common/utils/ContainerCache.java  |  168 ---
 .../container/common/utils/package-info.java    |   18 -
 .../container/ozoneimpl/OzoneContainer.java     |  269 ----
 .../ozone/container/ozoneimpl/package-info.java |   21 -
 .../apache/hadoop/ozone/ksm/BucketManager.java  |   79 --
 .../hadoop/ozone/ksm/BucketManagerImpl.java     |  315 -----
 .../org/apache/hadoop/ozone/ksm/KSMMXBean.java  |   31 -
 .../hadoop/ozone/ksm/KSMMetadataManager.java    |  253 ----
 .../ozone/ksm/KSMMetadataManagerImpl.java       |  524 -------
 .../org/apache/hadoop/ozone/ksm/KSMMetrics.java |  437 ------
 .../org/apache/hadoop/ozone/ksm/KSMStorage.java |   89 --
 .../hadoop/ozone/ksm/KeyDeletingService.java    |  141 --
 .../org/apache/hadoop/ozone/ksm/KeyManager.java |  165 ---
 .../apache/hadoop/ozone/ksm/KeyManagerImpl.java |  512 -------
 .../hadoop/ozone/ksm/KeySpaceManager.java       |  902 ------------
 .../ozone/ksm/KeySpaceManagerHttpServer.java    |   78 --
 .../hadoop/ozone/ksm/OpenKeyCleanupService.java |  116 --
 .../ozone/ksm/ServiceListJSONServlet.java       |  103 --
 .../apache/hadoop/ozone/ksm/VolumeManager.java  |  100 --
 .../hadoop/ozone/ksm/VolumeManagerImpl.java     |  391 ------
 .../ozone/ksm/exceptions/KSMException.java      |  116 --
 .../ozone/ksm/exceptions/package-info.java      |   19 -
 .../apache/hadoop/ozone/ksm/package-info.java   |   21 -
 .../org/apache/hadoop/ozone/lease/Lease.java    |  189 ---
 .../ozone/lease/LeaseAlreadyExistException.java |   46 -
 .../ozone/lease/LeaseCallbackExecutor.java      |   65 -
 .../hadoop/ozone/lease/LeaseException.java      |   45 -
 .../ozone/lease/LeaseExpiredException.java      |   45 -
 .../apache/hadoop/ozone/lease/LeaseManager.java |  247 ----
 .../lease/LeaseManagerNotRunningException.java  |   45 -
 .../ozone/lease/LeaseNotFoundException.java     |   46 -
 .../apache/hadoop/ozone/lease/package-info.java |   26 -
 .../org/apache/hadoop/ozone/package-info.java   |   35 -
 .../StorageContainerDatanodeProtocol.java       |   84 --
 .../protocol/StorageContainerNodeProtocol.java  |   69 -
 .../hadoop/ozone/protocol/VersionResponse.java  |  150 --
 .../commands/CloseContainerCommand.java         |   75 -
 .../protocol/commands/DeleteBlocksCommand.java  |   63 -
 .../protocol/commands/RegisteredCommand.java    |  177 ---
 .../protocol/commands/ReregisterCommand.java    |   59 -
 .../ozone/protocol/commands/SCMCommand.java     |   41 -
 .../protocol/commands/SendContainerCommand.java |   80 --
 .../ozone/protocol/commands/package-info.java   |   21 -
 .../hadoop/ozone/protocol/package-info.java     |   23 -
 ...ceManagerProtocolServerSideTranslatorPB.java |  539 --------
 ...kLocationProtocolServerSideTranslatorPB.java |  169 ---
 ...rDatanodeProtocolClientSideTranslatorPB.java |  195 ---
 .../StorageContainerDatanodeProtocolPB.java     |   32 -
 ...rDatanodeProtocolServerSideTranslatorPB.java |  113 --
 ...rLocationProtocolServerSideTranslatorPB.java |  202 ---
 .../hadoop/ozone/protocolPB/package-info.java   |   24 -
 .../org/apache/hadoop/ozone/scm/SCMMXBean.java  |   50 -
 .../org/apache/hadoop/ozone/scm/SCMStorage.java |   74 -
 .../ozone/scm/StorageContainerManager.java      | 1260 -----------------
 .../scm/StorageContainerManagerHttpServer.java  |   77 --
 .../apache/hadoop/ozone/scm/VersionInfo.java    |   81 --
 .../hadoop/ozone/scm/block/BlockManager.java    |   86 --
 .../ozone/scm/block/BlockManagerImpl.java       |  530 -------
 .../ozone/scm/block/BlockmanagerMXBean.java     |   30 -
 .../block/DatanodeDeletedBlockTransactions.java |  130 --
 .../hadoop/ozone/scm/block/DeletedBlockLog.java |  124 --
 .../ozone/scm/block/DeletedBlockLogImpl.java    |  351 -----
 .../scm/block/SCMBlockDeletingService.java      |  172 ---
 .../hadoop/ozone/scm/block/package-info.java    |   22 -
 .../hadoop/ozone/scm/cli/OzoneBaseCLI.java      |   44 -
 .../ozone/scm/cli/OzoneCommandHandler.java      |   79 --
 .../apache/hadoop/ozone/scm/cli/ResultCode.java |   31 -
 .../org/apache/hadoop/ozone/scm/cli/SCMCLI.java |  231 ----
 .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java |  720 ----------
 .../cli/container/CloseContainerHandler.java    |   84 --
 .../cli/container/ContainerCommandHandler.java  |  128 --
 .../cli/container/CreateContainerHandler.java   |   81 --
 .../cli/container/DeleteContainerHandler.java   |   92 --
 .../scm/cli/container/InfoContainerHandler.java |  115 --
 .../scm/cli/container/ListContainerHandler.java |  121 --
 .../ozone/scm/cli/container/package-info.java   |   19 -
 .../hadoop/ozone/scm/cli/package-info.java      |   19 -
 .../ozone/scm/container/ContainerMapping.java   |  606 --------
 .../scm/container/ContainerStateManager.java    |  451 ------
 .../ContainerStates/ContainerAttribute.java     |  244 ----
 .../ContainerStates/ContainerState.java         |   96 --
 .../ContainerStates/ContainerStateMap.java      |  402 ------
 .../container/ContainerStates/package-info.java |   22 -
 .../hadoop/ozone/scm/container/Mapping.java     |  107 --
 .../scm/container/closer/ContainerCloser.java   |  192 ---
 .../scm/container/closer/package-info.java      |   23 -
 .../ozone/scm/container/package-info.java       |   22 -
 .../algorithms/ContainerPlacementPolicy.java    |   41 -
 .../placement/algorithms/SCMCommonPolicy.java   |  194 ---
 .../SCMContainerPlacementCapacity.java          |  133 --
 .../algorithms/SCMContainerPlacementRandom.java |   89 --
 .../placement/algorithms/package-info.java      |   18 -
 .../placement/metrics/ContainerStat.java        |  166 ---
 .../placement/metrics/DatanodeMetric.java       |   91 --
 .../container/placement/metrics/LongMetric.java |  162 ---
 .../container/placement/metrics/NodeStat.java   |   67 -
 .../container/placement/metrics/SCMMetrics.java |  154 ---
 .../placement/metrics/SCMNodeMetric.java        |  223 ---
 .../placement/metrics/SCMNodeStat.java          |  139 --
 .../placement/metrics/package-info.java         |   20 -
 .../scm/container/placement/package-info.java   |   19 -
 .../replication/ContainerSupervisor.java        |  333 -----
 .../container/replication/InProgressPool.java   |  313 -----
 .../scm/container/replication/PeriodicPool.java |  119 --
 .../scm/container/replication/package-info.java |   23 -
 .../ozone/scm/exceptions/SCMException.java      |  119 --
 .../ozone/scm/exceptions/package-info.java      |   18 -
 .../hadoop/ozone/scm/node/CommandQueue.java     |  190 ---
 .../ozone/scm/node/HeartbeatQueueItem.java      |  116 --
 .../hadoop/ozone/scm/node/NodeManager.java      |  152 --
 .../ozone/scm/node/NodeManagerMXBean.java       |   57 -
 .../hadoop/ozone/scm/node/NodePoolManager.java  |   72 -
 .../hadoop/ozone/scm/node/SCMNodeManager.java   |  883 ------------
 .../ozone/scm/node/SCMNodePoolManager.java      |  266 ----
 .../hadoop/ozone/scm/node/package-info.java     |   31 -
 .../apache/hadoop/ozone/scm/package-info.java   |   22 -
 .../ozone/scm/pipelines/PipelineManager.java    |  176 ---
 .../ozone/scm/pipelines/PipelineSelector.java   |  227 ---
 .../ozone/scm/pipelines/package-info.java       |   38 -
 .../scm/pipelines/ratis/RatisManagerImpl.java   |  155 ---
 .../ozone/scm/pipelines/ratis/package-info.java |   18 -
 .../standalone/StandaloneManagerImpl.java       |  142 --
 .../scm/pipelines/standalone/package-info.java  |   18 -
 .../hadoop/ozone/scm/ratis/package-info.java    |   22 -
 .../org/apache/hadoop/ozone/tools/Freon.java    | 1146 ---------------
 .../apache/hadoop/ozone/tools/package-info.java |   22 -
 .../ozone/web/ObjectStoreApplication.java       |   59 -
 .../hadoop/ozone/web/OzoneHttpServer.java       |  216 ---
 .../hadoop/ozone/web/client/OzoneBucket.java    |  642 ---------
 .../hadoop/ozone/web/client/OzoneKey.java       |   44 -
 .../ozone/web/client/OzoneRestClient.java       |  801 -----------
 .../web/client/OzoneRestClientException.java    |   54 -
 .../hadoop/ozone/web/client/OzoneVolume.java    |  581 --------
 .../hadoop/ozone/web/client/package-info.java   |   34 -
 .../hadoop/ozone/web/exceptions/ErrorTable.java |  225 ---
 .../ozone/web/exceptions/package-info.java      |   22 -
 .../hadoop/ozone/web/handlers/BucketArgs.java   |  214 ---
 .../ozone/web/handlers/BucketHandler.java       |  197 ---
 .../web/handlers/BucketProcessTemplate.java     |  323 -----
 .../hadoop/ozone/web/handlers/KeyArgs.java      |  117 --
 .../hadoop/ozone/web/handlers/KeyHandler.java   |  245 ----
 .../ozone/web/handlers/KeyProcessTemplate.java  |  233 ----
 .../hadoop/ozone/web/handlers/ListArgs.java     |  142 --
 .../ozone/web/handlers/ServiceFilter.java       |   61 -
 .../web/handlers/StorageHandlerBuilder.java     |   76 -
 .../hadoop/ozone/web/handlers/UserArgs.java     |  168 ---
 .../ozone/web/handlers/UserHandlerBuilder.java  |   75 -
 .../hadoop/ozone/web/handlers/VolumeArgs.java   |  142 --
 .../ozone/web/handlers/VolumeHandler.java       |  272 ----
 .../web/handlers/VolumeProcessTemplate.java     |  274 ----
 .../hadoop/ozone/web/handlers/package-info.java |   22 -
 .../hadoop/ozone/web/interfaces/Accounting.java |   57 -
 .../hadoop/ozone/web/interfaces/Bucket.java     |  183 ---
 .../hadoop/ozone/web/interfaces/Keys.java       |  146 --
 .../ozone/web/interfaces/StorageHandler.java    |  295 ----
 .../hadoop/ozone/web/interfaces/UserAuth.java   |  101 --
 .../hadoop/ozone/web/interfaces/Volume.java     |  182 ---
 .../ozone/web/interfaces/package-info.java      |   22 -
 .../web/localstorage/LocalStorageHandler.java   |  379 -----
 .../web/localstorage/OzoneMetadataManager.java  | 1138 ---------------
 .../ozone/web/localstorage/package-info.java    |   18 -
 .../LengthInputStreamMessageBodyWriter.java     |   59 -
 .../web/messages/StringMessageBodyWriter.java   |   62 -
 .../hadoop/ozone/web/messages/package-info.java |   18 -
 .../web/netty/CloseableCleanupListener.java     |   46 -
 .../web/netty/ObjectStoreChannelHandler.java    |   78 --
 .../web/netty/ObjectStoreJerseyContainer.java   |  348 -----
 .../ObjectStoreJerseyContainerProvider.java     |   40 -
 ...RequestContentObjectStoreChannelHandler.java |  117 --
 ...equestDispatchObjectStoreChannelHandler.java |  103 --
 .../hadoop/ozone/web/netty/package-info.java    |   26 -
 .../hadoop/ozone/web/ozShell/Handler.java       |   76 -
 .../apache/hadoop/ozone/web/ozShell/Shell.java  |  415 ------
 .../web/ozShell/bucket/CreateBucketHandler.java |   96 --
 .../web/ozShell/bucket/DeleteBucketHandler.java |   87 --
 .../web/ozShell/bucket/InfoBucketHandler.java   |   94 --
 .../web/ozShell/bucket/ListBucketHandler.java   |  113 --
 .../web/ozShell/bucket/UpdateBucketHandler.java |   94 --
 .../ozone/web/ozShell/bucket/package-info.java  |   23 -
 .../web/ozShell/keys/DeleteKeyHandler.java      |   96 --
 .../ozone/web/ozShell/keys/GetKeyHandler.java   |  116 --
 .../ozone/web/ozShell/keys/InfoKeyHandler.java  |   98 --
 .../ozone/web/ozShell/keys/ListKeyHandler.java  |  116 --
 .../ozone/web/ozShell/keys/PutKeyHandler.java   |  106 --
 .../ozone/web/ozShell/keys/package-info.java    |   23 -
 .../hadoop/ozone/web/ozShell/package-info.java  |   27 -
 .../web/ozShell/volume/CreateVolumeHandler.java |   99 --
 .../web/ozShell/volume/DeleteVolumeHandler.java |   81 --
 .../web/ozShell/volume/InfoVolumeHandler.java   |   95 --
 .../web/ozShell/volume/ListVolumeHandler.java   |  114 --
 .../web/ozShell/volume/UpdateVolumeHandler.java |   92 --
 .../ozone/web/ozShell/volume/package-info.java  |   23 -
 .../apache/hadoop/ozone/web/package-info.java   |   24 -
 .../hadoop/ozone/web/request/OzoneQuota.java    |  214 ---
 .../hadoop/ozone/web/request/package-info.java  |   23 -
 .../hadoop/ozone/web/response/BucketInfo.java   |  325 -----
 .../hadoop/ozone/web/response/KeyInfo.java      |  290 ----
 .../hadoop/ozone/web/response/ListBuckets.java  |  154 ---
 .../hadoop/ozone/web/response/ListKeys.java     |  209 ---
 .../hadoop/ozone/web/response/ListVolumes.java  |  152 --
 .../hadoop/ozone/web/response/VolumeInfo.java   |  308 -----
 .../hadoop/ozone/web/response/VolumeOwner.java  |   60 -
 .../hadoop/ozone/web/response/package-info.java |   23 -
 .../web/storage/DistributedStorageHandler.java  |  539 --------
 .../hadoop/ozone/web/storage/package-info.java  |   27 -
 .../hadoop/ozone/web/userauth/Simple.java       |  169 ---
 .../hadoop/ozone/web/userauth/package-info.java |   23 -
 .../hadoop/ozone/web/utils/JsonUtils.java       |   71 -
 .../hadoop/ozone/web/utils/OzoneUtils.java      |  303 ----
 .../hadoop/ozone/web/utils/package-info.java    |   18 -
 .../apache/hadoop/utils/BackgroundService.java  |  161 ---
 .../org/apache/hadoop/utils/BackgroundTask.java |   28 -
 .../hadoop/utils/BackgroundTaskQueue.java       |   64 -
 .../hadoop/utils/BackgroundTaskResult.java      |   44 -
 .../org/apache/hadoop/utils/BatchOperation.java |   90 --
 .../org/apache/hadoop/utils/EntryConsumer.java  |   38 -
 .../org/apache/hadoop/utils/LevelDBStore.java   |  379 -----
 .../apache/hadoop/utils/MetadataKeyFilters.java |  118 --
 .../org/apache/hadoop/utils/MetadataStore.java  |  172 ---
 .../hadoop/utils/MetadataStoreBuilder.java      |  125 --
 .../org/apache/hadoop/utils/RocksDBStore.java   |  382 -----
 .../apache/hadoop/utils/RocksDBStoreMBean.java  |  148 --
 .../org/apache/hadoop/utils/package-info.java   |   18 -
 .../main/proto/CBlockClientServerProtocol.proto |   93 --
 .../src/main/proto/CBlockServiceProtocol.proto  |  133 --
 .../StorageContainerDatanodeProtocol.proto      |  367 -----
 ...m.sun.jersey.spi.container.ContainerProvider |   16 -
 .../src/main/resources/ozone-default.xml        | 1308 ------------------
 .../hadoop-hdfs/src/main/webapps/datanode/dn.js |   28 +-
 .../hadoop-hdfs/src/main/webapps/ksm/index.html |   70 -
 .../src/main/webapps/ksm/ksm-metrics.html       |   44 -
 .../hadoop-hdfs/src/main/webapps/ksm/ksm.js     |  108 --
 .../hadoop-hdfs/src/main/webapps/ksm/main.css   |   23 -
 .../hadoop-hdfs/src/main/webapps/ksm/main.html  |   18 -
 .../hadoop-hdfs/src/main/webapps/scm/index.html |   76 -
 .../hadoop-hdfs/src/main/webapps/scm/main.html  |   20 -
 .../src/main/webapps/scm/scm-overview.html      |   60 -
 .../hadoop-hdfs/src/main/webapps/scm/scm.js     |   54 -
 .../main/webapps/static/angular-1.6.4.min.js    |  332 -----
 .../webapps/static/angular-nvd3-1.0.9.min.js    |    1 -
 .../webapps/static/angular-route-1.6.4.min.js   |   17 -
 .../src/main/webapps/static/d3-3.5.17.min.js    |    5 -
 .../src/main/webapps/static/dfs-dust.js         |    5 +-
 .../src/main/webapps/static/nvd3-1.8.5.min.css  |    2 -
 .../main/webapps/static/nvd3-1.8.5.min.css.map  |    1 -
 .../src/main/webapps/static/nvd3-1.8.5.min.js   |   11 -
 .../main/webapps/static/nvd3-1.8.5.min.js.map   |    1 -
 .../src/main/webapps/static/ozone.css           |   60 -
 .../src/main/webapps/static/ozone.js            |  355 -----
 .../main/webapps/static/templates/config.html   |   92 --
 .../src/main/webapps/static/templates/jvm.html  |   26 -
 .../src/main/webapps/static/templates/menu.html |   60 -
 .../main/webapps/static/templates/overview.html |   39 -
 .../webapps/static/templates/rpc-metrics.html   |   87 --
 .../src/site/markdown/OzoneCommandShell.md      |  150 --
 .../src/site/markdown/OzoneGettingStarted.md.vm |  320 -----
 .../src/site/markdown/OzoneMetrics.md           |  166 ---
 .../src/site/markdown/OzoneOverview.md          |   88 --
 .../hadoop-hdfs/src/site/markdown/OzoneRest.md  |  549 --------
 .../apache/hadoop/cblock/TestBufferManager.java |  456 ------
 .../org/apache/hadoop/cblock/TestCBlockCLI.java |  242 ----
 .../hadoop/cblock/TestCBlockReadWrite.java      |  377 -----
 .../apache/hadoop/cblock/TestCBlockServer.java  |  212 ---
 .../cblock/TestCBlockServerPersistence.java     |  132 --
 .../hadoop/cblock/TestLocalBlockCache.java      |  444 ------
 .../kubernetes/TestDynamicProvisioner.java      |   73 -
 .../cblock/util/ContainerLookUpService.java     |   73 -
 .../hadoop/cblock/util/MockStorageClient.java   |  176 ---
 .../server/namenode/TestNameNodeHttpServer.java |    9 +-
 .../namenode/ha/TestPipelinesFailover.java      |    6 +-
 .../hadoop/ozone/MiniOzoneClassicCluster.java   |  606 --------
 .../apache/hadoop/ozone/MiniOzoneCluster.java   |   62 -
 .../apache/hadoop/ozone/RatisTestHelper.java    |  111 --
 .../hadoop/ozone/TestContainerOperations.java   |   90 --
 .../apache/hadoop/ozone/TestMetadataStore.java  |  414 ------
 .../hadoop/ozone/TestMiniOzoneCluster.java      |  254 ----
 .../ozone/TestOzoneConfigurationFields.java     |   39 -
 .../ozone/TestStorageContainerManager.java      |  447 ------
 .../TestStorageContainerManagerHelper.java      |  180 ---
 .../ozone/client/TestOzoneClientUtils.java      |  362 -----
 .../hadoop/ozone/client/package-info.java       |   23 -
 .../ozone/client/rest/TestOzoneRestClient.java  |  405 ------
 .../hadoop/ozone/client/rest/package-info.java  |   23 -
 .../ozone/client/rpc/TestOzoneRpcClient.java    |  815 -----------
 .../hadoop/ozone/client/rpc/package-info.java   |   23 -
 .../ozone/container/ContainerTestHelper.java    |  595 --------
 .../TestUtils/BlockDeletingServiceTestImpl.java |  103 --
 .../ReplicationDatanodeStateManager.java        |   97 --
 .../TestUtils/ReplicationNodeManagerMock.java   |  323 -----
 .../ReplicationNodePoolManagerMock.java         |  132 --
 .../ozone/container/TestUtils/package-info.java |   18 -
 .../ozone/container/common/SCMTestUtils.java    |  221 ---
 .../ozone/container/common/ScmTestMock.java     |  261 ----
 .../common/TestBlockDeletingService.java        |  396 ------
 .../common/TestDatanodeStateMachine.java        |  361 -----
 .../ozone/container/common/TestEndPoint.java    |  461 ------
 .../TestContainerDeletionChoosingPolicy.java    |  197 ---
 .../common/impl/TestContainerPersistence.java   |  905 ------------
 .../TestCloseContainerHandler.java              |  112 --
 .../container/metrics/TestContainerMetrics.java |  124 --
 .../container/ozoneimpl/TestOzoneContainer.java |  584 --------
 .../ozoneimpl/TestOzoneContainerRatis.java      |  143 --
 .../container/ozoneimpl/TestRatisManager.java   |  129 --
 .../placement/TestContainerPlacement.java       |  131 --
 .../placement/TestDatanodeMetrics.java          |   59 -
 .../replication/TestContainerSupervisor.java    |  269 ----
 .../container/replication/package-info.java     |   18 -
 .../transport/server/TestContainerServer.java   |  258 ----
 .../hadoop/ozone/ksm/TestBucketManagerImpl.java |  395 ------
 .../hadoop/ozone/ksm/TestChunkStreams.java      |  234 ----
 .../ozone/ksm/TestContainerReportWithKeys.java  |  138 --
 .../apache/hadoop/ozone/ksm/TestKSMMetrcis.java |  307 ----
 .../apache/hadoop/ozone/ksm/TestKSMSQLCli.java  |  273 ----
 .../hadoop/ozone/ksm/TestKeySpaceManager.java   | 1238 -----------------
 .../ksm/TestKeySpaceManagerHttpServer.java      |  141 --
 .../ksm/TestKeySpaceManagerRestInterface.java   |  141 --
 .../ozone/ksm/TestKsmBlockVersioning.java       |  254 ----
 .../ksm/TestMultipleContainerReadWrite.java     |  216 ---
 .../apache/hadoop/ozone/ksm/package-info.java   |   21 -
 .../hadoop/ozone/lease/TestLeaseManager.java    |  374 -----
 .../apache/hadoop/ozone/lease/package-info.java |   21 -
 .../hadoop/ozone/ozShell/TestOzoneShell.java    |  804 -----------
 .../org/apache/hadoop/ozone/package-info.java   |   21 -
 .../hadoop/ozone/scm/TestAllocateContainer.java |  104 --
 .../hadoop/ozone/scm/TestContainerSQLCli.java   |  300 ----
 .../ozone/scm/TestContainerSmallFile.java       |  151 --
 .../org/apache/hadoop/ozone/scm/TestSCMCli.java |  535 -------
 .../apache/hadoop/ozone/scm/TestSCMMXBean.java  |  162 ---
 .../apache/hadoop/ozone/scm/TestSCMMetrics.java |  251 ----
 .../TestStorageContainerManagerHttpServer.java  |  141 --
 .../ozone/scm/TestXceiverClientManager.java     |  196 ---
 .../ozone/scm/TestXceiverClientMetrics.java     |  172 ---
 .../ozone/scm/block/TestBlockManager.java       |  160 ---
 .../ozone/scm/block/TestDeletedBlockLog.java    |  348 -----
 .../BenchmarkContainerStateMap.java             |  180 ---
 .../ContainerStates/TestContainerAttribute.java |  139 --
 .../ContainerStates/TestContainerStateMap.java  |  226 ---
 .../ozone/scm/container/MockNodeManager.java    |  519 -------
 .../scm/container/TestContainerMapping.java     |  333 -----
 .../container/TestContainerStateManager.java    |  303 ----
 .../container/closer/TestContainerCloser.java   |  221 ---
 .../ozone/scm/node/TestContainerPlacement.java  |  172 ---
 .../hadoop/ozone/scm/node/TestNodeManager.java  | 1145 ---------------
 .../hadoop/ozone/scm/node/TestQueryNode.java    |  122 --
 .../ozone/scm/node/TestSCMNodePoolManager.java  |  156 ---
 .../apache/hadoop/ozone/scm/package-info.java   |   21 -
 .../hadoop/ozone/tools/TestDataValidate.java    |  146 --
 .../apache/hadoop/ozone/tools/TestFreon.java    |  136 --
 .../apache/hadoop/ozone/tools/package-info.java |   21 -
 .../apache/hadoop/ozone/web/TestBucketInfo.java |   81 --
 .../ozone/web/TestDistributedOzoneVolumes.java  |  186 ---
 .../apache/hadoop/ozone/web/TestErrorCode.java  |   53 -
 .../hadoop/ozone/web/TestLocalOzoneVolumes.java |  185 ---
 .../hadoop/ozone/web/TestOzoneHelper.java       |  414 ------
 .../hadoop/ozone/web/TestOzoneHttpServer.java   |   98 --
 .../ozone/web/TestOzoneRestWithMiniCluster.java |  258 ----
 .../hadoop/ozone/web/TestOzoneWebAccess.java    |  126 --
 .../org/apache/hadoop/ozone/web/TestQuota.java  |  116 --
 .../org/apache/hadoop/ozone/web/TestUtils.java  |  101 --
 .../hadoop/ozone/web/TestVolumeStructs.java     |   73 -
 .../hadoop/ozone/web/client/TestBuckets.java    |  251 ----
 .../ozone/web/client/TestBucketsRatis.java      |   76 -
 .../hadoop/ozone/web/client/TestKeys.java       |  688 ---------
 .../hadoop/ozone/web/client/TestKeysRatis.java  |  126 --
 .../ozone/web/client/TestOzoneClient.java       |  305 ----
 .../hadoop/ozone/web/client/TestVolume.java     |  434 ------
 .../ozone/web/client/TestVolumeRatis.java       |  131 --
 .../apache/hadoop/ozone/web/package-info.java   |   21 -
 .../org/apache/hadoop/test/OzoneTestDriver.java |   59 -
 .../hadoop/utils/TestRocksDBStoreMBean.java     |   87 --
 .../dynamicprovisioner/expected1-pv.json        |   37 -
 .../dynamicprovisioner/input1-pvc.json          |   38 -
 .../src/test/resources/ozone-site.xml           |   24 -
 hadoop-hdsl/client/pom.xml                      |   48 +
 .../hadoop/ozone/client/OzoneClientUtils.java   |  231 ++++
 .../hadoop/ozone/client/package-info.java       |   23 +
 .../org/apache/hadoop/scm/XceiverClient.java    |  192 +++
 .../apache/hadoop/scm/XceiverClientHandler.java |  204 +++
 .../hadoop/scm/XceiverClientInitializer.java    |   72 +
 .../apache/hadoop/scm/XceiverClientManager.java |  219 +++
 .../apache/hadoop/scm/XceiverClientMetrics.java |   92 ++
 .../apache/hadoop/scm/XceiverClientRatis.java   |  264 ++++
 .../scm/client/ContainerOperationClient.java    |  403 ++++++
 .../apache/hadoop/scm/client/package-info.java  |   23 +
 .../org/apache/hadoop/scm/package-info.java     |   23 +
 .../hadoop/scm/storage/ChunkInputStream.java    |  261 ++++
 .../hadoop/scm/storage/ChunkOutputStream.java   |  227 +++
 .../apache/hadoop/scm/storage/package-info.java |   23 +
 .../common/dev-support/findbugsExcludeFile.xml  |   21 +
 hadoop-hdsl/common/pom.xml                      |  146 ++
 .../java/org/apache/hadoop/hdsl/HdslUtils.java  |  273 ++++
 .../hadoop/hdsl/conf/OzoneConfiguration.java    |  161 +++
 .../apache/hadoop/hdsl/conf/package-info.java   |   18 +
 .../org/apache/hadoop/hdsl/package-info.java    |   23 +
 .../java/org/apache/hadoop/ozone/OzoneAcl.java  |  231 ++++
 .../apache/hadoop/ozone/OzoneConfigKeys.java    |  239 ++++
 .../org/apache/hadoop/ozone/OzoneConsts.java    |  167 +++
 .../apache/hadoop/ozone/client/OzoneQuota.java  |  203 +++
 .../hadoop/ozone/client/ReplicationFactor.java  |   63 +
 .../hadoop/ozone/client/ReplicationType.java    |   28 +
 .../hadoop/ozone/client/package-info.java       |   23 +
 .../apache/hadoop/ozone/common/BlockGroup.java  |   87 ++
 .../ozone/common/DeleteBlockGroupResult.java    |   94 ++
 .../InconsistentStorageStateException.java      |   51 +
 .../org/apache/hadoop/ozone/common/Storage.java |  249 ++++
 .../apache/hadoop/ozone/common/StorageInfo.java |  184 +++
 .../hadoop/ozone/common/package-info.java       |   18 +
 .../InvalidStateTransitionException.java        |   42 +
 .../ozone/common/statemachine/StateMachine.java |   68 +
 .../ozone/common/statemachine/package-info.java |   21 +
 .../container/common/helpers/ChunkInfo.java     |  185 +++
 .../ozone/container/common/helpers/KeyData.java |  170 +++
 .../container/common/helpers/package-info.java  |   23 +
 .../org/apache/hadoop/ozone/lease/Lease.java    |  189 +++
 .../ozone/lease/LeaseAlreadyExistException.java |   46 +
 .../ozone/lease/LeaseCallbackExecutor.java      |   65 +
 .../hadoop/ozone/lease/LeaseException.java      |   45 +
 .../ozone/lease/LeaseExpiredException.java      |   45 +
 .../apache/hadoop/ozone/lease/LeaseManager.java |  247 ++++
 .../lease/LeaseManagerNotRunningException.java  |   45 +
 .../ozone/lease/LeaseNotFoundException.java     |   46 +
 .../apache/hadoop/ozone/lease/package-info.java |   26 +
 .../org/apache/hadoop/ozone/package-info.java   |   35 +
 ...kLocationProtocolServerSideTranslatorPB.java |  169 +++
 ...rLocationProtocolServerSideTranslatorPB.java |  202 +++
 .../hadoop/ozone/protocolPB/package-info.java   |   24 +
 .../hadoop/ozone/web/utils/JsonUtils.java       |   71 +
 .../org/apache/hadoop/scm/ScmConfigKeys.java    |  271 ++++
 .../java/org/apache/hadoop/scm/ScmInfo.java     |   81 ++
 .../org/apache/hadoop/scm/XceiverClientSpi.java |  129 ++
 .../org/apache/hadoop/scm/client/ScmClient.java |  139 ++
 .../apache/hadoop/scm/client/package-info.java  |   24 +
 .../container/ContainerStates/ContainerID.java  |   97 ++
 .../container/ContainerStates/package-info.java |   22 +
 .../common/helpers/AllocatedBlock.java          |   77 ++
 .../container/common/helpers/ContainerInfo.java |  333 +++++
 .../common/helpers/DeleteBlockResult.java       |   52 +
 .../scm/container/common/helpers/Pipeline.java  |  253 ++++
 .../common/helpers/PipelineChannel.java         |  121 ++
 .../helpers/StorageContainerException.java      |  104 ++
 .../container/common/helpers/package-info.java  |   22 +
 .../org/apache/hadoop/scm/package-info.java     |   24 +
 .../hadoop/scm/protocol/LocatedContainer.java   |  127 ++
 .../scm/protocol/ScmBlockLocationProtocol.java  |   72 +
 .../hadoop/scm/protocol/ScmLocatedBlock.java    |  100 ++
 .../StorageContainerLocationProtocol.java       |  123 ++
 .../hadoop/scm/protocol/package-info.java       |   19 +
 ...kLocationProtocolClientSideTranslatorPB.java |  207 +++
 .../protocolPB/ScmBlockLocationProtocolPB.java  |   35 +
 ...rLocationProtocolClientSideTranslatorPB.java |  304 ++++
 .../StorageContainerLocationProtocolPB.java     |   34 +
 .../hadoop/scm/protocolPB/package-info.java     |   24 +
 .../scm/storage/ContainerProtocolCalls.java     |  397 ++++++
 .../apache/hadoop/scm/storage/package-info.java |   23 +
 .../apache/hadoop/utils/BackgroundService.java  |  161 +++
 .../org/apache/hadoop/utils/BackgroundTask.java |   28 +
 .../hadoop/utils/BackgroundTaskQueue.java       |   64 +
 .../hadoop/utils/BackgroundTaskResult.java      |   44 +
 .../org/apache/hadoop/utils/BatchOperation.java |   90 ++
 .../org/apache/hadoop/utils/EntryConsumer.java  |   38 +
 .../org/apache/hadoop/utils/LevelDBStore.java   |  380 +++++
 .../apache/hadoop/utils/MetadataKeyFilters.java |  118 ++
 .../org/apache/hadoop/utils/MetadataStore.java  |  172 +++
 .../hadoop/utils/MetadataStoreBuilder.java      |  125 ++
 .../org/apache/hadoop/utils/RocksDBStore.java   |  382 +++++
 .../apache/hadoop/utils/RocksDBStoreMBean.java  |  136 ++
 .../org/apache/hadoop/utils/package-info.java   |   18 +
 .../main/java/org/apache/ratis/RatisHelper.java |  128 ++
 .../java/org/apache/ratis/package-info.java     |   22 +
 .../com/google/protobuf/ShadedProtoUtil.java    |   38 +
 .../com/google/protobuf/package-info.java       |   22 +
 .../main/proto/DatanodeContainerProtocol.proto  |  415 ++++++
 .../main/proto/ScmBlockLocationProtocol.proto   |  166 +++
 .../StorageContainerLocationProtocol.proto      |  214 +++
 hadoop-hdsl/common/src/main/proto/hdsl.proto    |  162 +++
 .../common/src/main/resources/ozone-default.xml | 1031 ++++++++++++++
 .../apache/hadoop/ozone/TestMetadataStore.java  |  414 ++++++
 .../org/apache/hadoop/ozone/TestOzoneAcls.java  |  141 ++
 .../hadoop/ozone/common/TestStateMachine.java   |  104 ++
 .../hadoop/ozone/lease/TestLeaseManager.java    |  374 +++++
 .../apache/hadoop/ozone/lease/package-info.java |   21 +
 .../org/apache/hadoop/ozone/package-info.java   |   21 +
 .../java/org/apache/hadoop/scm/TestArchive.java |  114 ++
 .../org/apache/hadoop/scm/package-info.java     |   21 +
 .../hadoop/utils/TestRocksDBStoreMBean.java     |   87 ++
 hadoop-hdsl/container-service/pom.xml           |  114 ++
 .../apache/hadoop/ozone/HdslServerPlugin.java   |   82 ++
 .../container/common/helpers/ChunkUtils.java    |  345 +++++
 .../container/common/helpers/ContainerData.java |  326 +++++
 .../common/helpers/ContainerMetrics.java        |  121 ++
 .../common/helpers/ContainerReport.java         |  218 +++
 .../common/helpers/ContainerUtils.java          |  477 +++++++
 .../helpers/DeletedContainerBlocksSummary.java  |  103 ++
 .../container/common/helpers/FileUtils.java     |   81 ++
 .../container/common/helpers/KeyUtils.java      |  147 ++
 .../container/common/helpers/package-info.java  |   22 +
 .../container/common/impl/ChunkManagerImpl.java |  232 ++++
 .../impl/ContainerLocationManagerImpl.java      |  149 ++
 .../common/impl/ContainerManagerImpl.java       | 1101 +++++++++++++++
 .../common/impl/ContainerReportManagerImpl.java |   89 ++
 .../container/common/impl/ContainerStatus.java  |  217 +++
 .../common/impl/ContainerStorageLocation.java   |  203 +++
 .../ozone/container/common/impl/Dispatcher.java |  708 ++++++++++
 .../container/common/impl/KeyManagerImpl.java   |  201 +++
 .../RandomContainerDeletionChoosingPolicy.java  |   69 +
 .../common/impl/StorageLocationReport.java      |   63 +
 ...NOrderedContainerDeletionChoosingPolicy.java |   90 ++
 .../container/common/impl/package-info.java     |   22 +
 .../common/interfaces/ChunkManager.java         |   75 +
 .../ContainerDeletionChoosingPolicy.java        |   45 +
 .../common/interfaces/ContainerDispatcher.java  |   49 +
 .../interfaces/ContainerLocationManager.java    |   58 +
 .../ContainerLocationManagerMXBean.java         |   36 +
 .../common/interfaces/ContainerManager.java     |  277 ++++
 .../interfaces/ContainerReportManager.java      |   32 +
 .../container/common/interfaces/KeyManager.java |   75 +
 .../common/interfaces/package-info.java         |   20 +
 .../ozone/container/common/package-info.java    |   28 +
 .../statemachine/DatanodeStateMachine.java      |  386 ++++++
 .../statemachine/EndpointStateMachine.java      |  295 ++++
 .../statemachine/EndpointStateMachineMBean.java |   34 +
 .../statemachine/SCMConnectionManager.java      |  203 +++
 .../SCMConnectionManagerMXBean.java             |   27 +
 .../common/statemachine/StateContext.java       |  281 ++++
 .../background/BlockDeletingService.java        |  238 ++++
 .../statemachine/background/package-info.java   |   18 +
 .../commandhandler/CloseContainerHandler.java   |  112 ++
 .../commandhandler/CommandDispatcher.java       |  177 +++
 .../commandhandler/CommandHandler.java          |   59 +
 .../commandhandler/ContainerReportHandler.java  |  110 ++
 .../DeleteBlocksCommandHandler.java             |  204 +++
 .../commandhandler/package-info.java            |   18 +
 .../common/statemachine/package-info.java       |   28 +
 .../container/common/states/DatanodeState.java  |   55 +
 .../states/datanode/InitDatanodeState.java      |  167 +++
 .../states/datanode/RunningDatanodeState.java   |  253 ++++
 .../common/states/datanode/package-info.java    |   21 +
 .../states/endpoint/HeartbeatEndpointTask.java  |  270 ++++
 .../states/endpoint/RegisterEndpointTask.java   |  198 +++
 .../states/endpoint/VersionEndpointTask.java    |   66 +
 .../common/states/endpoint/package-info.java    |   20 +
 .../container/common/states/package-info.java   |   18 +
 .../common/transport/server/XceiverServer.java  |  128 ++
 .../transport/server/XceiverServerHandler.java  |   80 ++
 .../server/XceiverServerInitializer.java        |   61 +
 .../transport/server/XceiverServerSpi.java      |   43 +
 .../common/transport/server/package-info.java   |   24 +
 .../server/ratis/ContainerStateMachine.java     |  290 ++++
 .../server/ratis/XceiverServerRatis.java        |  214 +++
 .../transport/server/ratis/package-info.java    |   23 +
 .../container/common/utils/ContainerCache.java  |  168 +++
 .../container/common/utils/package-info.java    |   18 +
 .../container/ozoneimpl/OzoneContainer.java     |  269 ++++
 .../ozone/container/ozoneimpl/package-info.java |   21 +
 .../org/apache/hadoop/ozone/package-info.java   |   23 +
 .../StorageContainerDatanodeProtocol.java       |   84 ++
 .../protocol/StorageContainerNodeProtocol.java  |   69 +
 .../hadoop/ozone/protocol/VersionResponse.java  |  150 ++
 .../commands/CloseContainerCommand.java         |   75 +
 .../protocol/commands/DeleteBlocksCommand.java  |   63 +
 .../protocol/commands/RegisteredCommand.java    |  177 +++
 .../protocol/commands/ReregisterCommand.java    |   58 +
 .../ozone/protocol/commands/SCMCommand.java     |   41 +
 .../protocol/commands/SendContainerCommand.java |   80 ++
 .../ozone/protocol/commands/package-info.java   |   21 +
 .../hadoop/ozone/protocol/package-info.java     |   23 +
 ...rDatanodeProtocolClientSideTranslatorPB.java |  195 +++
 .../StorageContainerDatanodeProtocolPB.java     |   32 +
 ...rDatanodeProtocolServerSideTranslatorPB.java |  113 ++
 .../apache/hadoop/ozone/scm/HdslServerUtil.java |  304 ++++
 .../apache/hadoop/ozone/scm/VersionInfo.java    |   81 ++
 .../StorageContainerDatanodeProtocol.proto      |  367 +++++
 ...m.sun.jersey.spi.container.ContainerProvider |   16 +
 .../container/common/ContainerTestUtils.java    |   68 +
 .../ozone/container/common/SCMTestUtils.java    |  121 ++
 .../ozone/container/common/ScmTestMock.java     |  261 ++++
 .../common/TestDatanodeStateMachine.java        |  361 +++++
 .../testutils/BlockDeletingServiceTestImpl.java |  103 ++
 .../ozone/container/testutils/package-info.java |   18 +
 hadoop-hdsl/framework/pom.xml                   |  102 ++
 .../hadoop/ozone/web/OzoneHttpServer.java       |  218 +++
 .../apache/hadoop/ozone/web/package-info.java   |   23 +
 .../hadoop/ozone/web/util/ServerUtils.java      |  140 ++
 .../hadoop/ozone/web/util/package-info.java     |   23 +
 .../framework/src/main/webapps/datanode/dn.js   |   92 ++
 .../main/webapps/static/angular-1.6.4.min.js    |  332 +++++
 .../webapps/static/angular-nvd3-1.0.9.min.js    |    1 +
 .../webapps/static/angular-route-1.6.4.min.js   |   17 +
 .../src/main/webapps/static/d3-3.5.17.min.js    |    5 +
 .../src/main/webapps/static/dfs-dust.js         |  133 ++
 .../src/main/webapps/static/nvd3-1.8.5.min.css  |    2 +
 .../main/webapps/static/nvd3-1.8.5.min.css.map  |    1 +
 .../src/main/webapps/static/nvd3-1.8.5.min.js   |   11 +
 .../main/webapps/static/nvd3-1.8.5.min.js.map   |    1 +
 .../framework/src/main/webapps/static/ozone.css |   60 +
 .../framework/src/main/webapps/static/ozone.js  |  355 +++++
 .../main/webapps/static/templates/config.html   |   91 ++
 .../src/main/webapps/static/templates/jvm.html  |   26 +
 .../src/main/webapps/static/templates/menu.html |   60 +
 .../main/webapps/static/templates/overview.html |   39 +
 .../webapps/static/templates/rpc-metrics.html   |   87 ++
 .../framework/src/test/resources/ozone-site.xml |   24 +
 hadoop-hdsl/pom.xml                             |   60 +
 hadoop-hdsl/server-scm/pom.xml                  |  141 ++
 .../org/apache/hadoop/ozone/scm/SCMMXBean.java  |   50 +
 .../org/apache/hadoop/ozone/scm/SCMStorage.java |   73 +
 .../ozone/scm/StorageContainerManager.java      | 1269 +++++++++++++++++
 .../scm/StorageContainerManagerHttpServer.java  |   77 ++
 .../hadoop/ozone/scm/block/BlockManager.java    |   86 ++
 .../ozone/scm/block/BlockManagerImpl.java       |  531 +++++++
 .../ozone/scm/block/BlockmanagerMXBean.java     |   30 +
 .../block/DatanodeDeletedBlockTransactions.java |  130 ++
 .../hadoop/ozone/scm/block/DeletedBlockLog.java |  124 ++
 .../ozone/scm/block/DeletedBlockLogImpl.java    |  352 +++++
 .../scm/block/SCMBlockDeletingService.java      |  172 +++
 .../hadoop/ozone/scm/block/package-info.java    |   22 +
 .../ozone/scm/container/ContainerMapping.java   |  607 ++++++++
 .../scm/container/ContainerStateManager.java    |  451 ++++++
 .../ContainerStates/ContainerAttribute.java     |  244 ++++
 .../ContainerStates/ContainerState.java         |   96 ++
 .../ContainerStates/ContainerStateMap.java      |  402 ++++++
 .../container/ContainerStates/package-info.java |   22 +
 .../hadoop/ozone/scm/container/Mapping.java     |  107 ++
 .../scm/container/closer/ContainerCloser.java   |  192 +++
 .../scm/container/closer/package-info.java      |   23 +
 .../ozone/scm/container/package-info.java       |   22 +
 .../algorithms/ContainerPlacementPolicy.java    |   41 +
 .../placement/algorithms/SCMCommonPolicy.java   |  194 +++
 .../SCMContainerPlacementCapacity.java          |  133 ++
 .../algorithms/SCMContainerPlacementRandom.java |   89 ++
 .../placement/algorithms/package-info.java      |   18 +
 .../placement/metrics/ContainerStat.java        |  166 +++
 .../placement/metrics/DatanodeMetric.java       |   91 ++
 .../container/placement/metrics/LongMetric.java |  162 +++
 .../container/placement/metrics/NodeStat.java   |   67 +
 .../container/placement/metrics/SCMMetrics.java |  154 +++
 .../placement/metrics/SCMNodeMetric.java        |  223 +++
 .../placement/metrics/SCMNodeStat.java          |  139 ++
 .../placement/metrics/package-info.java         |   20 +
 .../scm/container/placement/package-info.java   |   19 +
 .../replication/ContainerSupervisor.java        |  333 +++++
 .../container/replication/InProgressPool.java   |  313 +++++
 .../scm/container/replication/PeriodicPool.java |  119 ++
 .../scm/container/replication/package-info.java |   23 +
 .../ozone/scm/exceptions/SCMException.java      |  119 ++
 .../ozone/scm/exceptions/package-info.java      |   18 +
 .../hadoop/ozone/scm/node/CommandQueue.java     |  190 +++
 .../ozone/scm/node/HeartbeatQueueItem.java      |  116 ++
 .../hadoop/ozone/scm/node/NodeManager.java      |  152 ++
 .../ozone/scm/node/NodeManagerMXBean.java       |   57 +
 .../hadoop/ozone/scm/node/NodePoolManager.java  |   72 +
 .../hadoop/ozone/scm/node/SCMNodeManager.java   |  891 ++++++++++++
 .../ozone/scm/node/SCMNodePoolManager.java      |  267 ++++
 .../hadoop/ozone/scm/node/package-info.java     |   31 +
 .../apache/hadoop/ozone/scm/package-info.java   |   22 +
 .../ozone/scm/pipelines/PipelineManager.java    |  176 +++
 .../ozone/scm/pipelines/PipelineSelector.java   |  227 +++
 .../ozone/scm/pipelines/package-info.java       |   38 +
 .../scm/pipelines/ratis/RatisManagerImpl.java   |  155 +++
 .../ozone/scm/pipelines/ratis/package-info.java |   18 +
 .../standalone/StandaloneManagerImpl.java       |  142 ++
 .../scm/pipelines/standalone/package-info.java  |   18 +
 .../hadoop/ozone/scm/ratis/package-info.java    |   22 +
 .../server-scm/src/main/webapps/scm/index.html  |   76 +
 .../server-scm/src/main/webapps/scm/main.html   |   20 +
 .../src/main/webapps/scm/scm-overview.html      |   60 +
 .../server-scm/src/main/webapps/scm/scm.js      |   54 +
 .../ozone/container/common/TestEndPoint.java    |  465 +++++++
 .../placement/TestContainerPlacement.java       |  131 ++
 .../placement/TestDatanodeMetrics.java          |   59 +
 .../replication/TestContainerSupervisor.java    |  268 ++++
 .../container/replication/package-info.java     |   18 +
 .../ReplicationDatanodeStateManager.java        |   97 ++
 .../testutils/ReplicationNodeManagerMock.java   |  323 +++++
 .../ReplicationNodePoolManagerMock.java         |  132 ++
 .../ozone/container/testutils/package-info.java |   18 +
 .../hadoop/ozone/scm/HdslServerUtilTest.java    |  315 +++++
 .../TestStorageContainerManagerHttpServer.java  |  141 ++
 .../org/apache/hadoop/ozone/scm/TestUtils.java  |  103 ++
 .../ozone/scm/block/TestBlockManager.java       |  160 +++
 .../ozone/scm/block/TestDeletedBlockLog.java    |  348 +++++
 .../ContainerStates/TestContainerAttribute.java |  142 ++
 .../ozone/scm/container/MockNodeManager.java    |  520 +++++++
 .../scm/container/TestContainerMapping.java     |  336 +++++
 .../container/closer/TestContainerCloser.java   |  221 +++
 .../ozone/scm/node/TestContainerPlacement.java  |  173 +++
 .../hadoop/ozone/scm/node/TestNodeManager.java  | 1148 +++++++++++++++
 .../ozone/scm/node/TestSCMNodePoolManager.java  |  158 +++
 .../apache/hadoop/ozone/scm/package-info.java   |   21 +
 hadoop-hdsl/tools/pom.xml                       |   61 +
 .../hadoop/ozone/scm/cli/OzoneBaseCLI.java      |   43 +
 .../ozone/scm/cli/OzoneCommandHandler.java      |   87 ++
 .../apache/hadoop/ozone/scm/cli/ResultCode.java |   31 +
 .../org/apache/hadoop/ozone/scm/cli/SCMCLI.java |  230 +++
 .../cli/container/CloseContainerHandler.java    |   84 ++
 .../cli/container/ContainerCommandHandler.java  |  128 ++
 .../cli/container/CreateContainerHandler.java   |   81 ++
 .../cli/container/DeleteContainerHandler.java   |   92 ++
 .../scm/cli/container/InfoContainerHandler.java |  115 ++
 .../scm/cli/container/ListContainerHandler.java |  121 ++
 .../ozone/scm/cli/container/package-info.java   |   19 +
 .../hadoop/ozone/scm/cli/package-info.java      |   19 +
 hadoop-ozone/client/pom.xml                     |   62 +
 .../apache/hadoop/ozone/client/BucketArgs.java  |  123 ++
 .../apache/hadoop/ozone/client/ObjectStore.java |  210 +++
 .../apache/hadoop/ozone/client/OzoneBucket.java |  360 +++++
 .../apache/hadoop/ozone/client/OzoneClient.java |  103 ++
 .../hadoop/ozone/client/OzoneClientFactory.java |  307 ++++
 .../client/OzoneClientInvocationHandler.java    |   62 +
 .../apache/hadoop/ozone/client/OzoneKey.java    |  119 ++
 .../apache/hadoop/ozone/client/OzoneVolume.java |  293 ++++
 .../apache/hadoop/ozone/client/VolumeArgs.java  |  128 ++
 .../ozone/client/io/ChunkGroupInputStream.java  |  319 +++++
 .../ozone/client/io/ChunkGroupOutputStream.java |  492 +++++++
 .../client/io/OzoneContainerTranslation.java    |   51 +
 .../ozone/client/io/OzoneInputStream.java       |   60 +
 .../ozone/client/io/OzoneOutputStream.java      |   64 +
 .../hadoop/ozone/client/io/package-info.java    |   23 +
 .../hadoop/ozone/client/package-info.java       |   23 +
 .../ozone/client/protocol/ClientProtocol.java   |  323 +++++
 .../ozone/client/protocol/package-info.java     |   23 +
 .../client/rest/DefaultRestServerSelector.java  |   36 +
 .../ozone/client/rest/OzoneExceptionMapper.java |   45 +
 .../hadoop/ozone/client/rest/RestClient.java    |  799 +++++++++++
 .../ozone/client/rest/RestServerSelector.java   |   40 +
 .../client/rest/exceptions/package-info.java    |   22 +
 .../ozone/client/rest/headers/package-info.java |   22 +
 .../hadoop/ozone/client/rest/package-info.java  |   23 +
 .../hadoop/ozone/client/rpc/RpcClient.java      |  567 ++++++++
 .../hadoop/ozone/client/rpc/package-info.java   |   23 +
 .../hadoop/ozone/web/client/OzoneBucket.java    |  644 +++++++++
 .../hadoop/ozone/web/client/OzoneKey.java       |   44 +
 .../ozone/web/client/OzoneRestClient.java       |  803 +++++++++++
 .../web/client/OzoneRestClientException.java    |   54 +
 .../hadoop/ozone/web/client/OzoneVolume.java    |  583 ++++++++
 .../hadoop/ozone/web/client/package-info.java   |   34 +
 .../ozone/client/TestOzoneClientUtils.java      |  105 ++
 .../hadoop/ozone/client/package-info.java       |   23 +
 .../common/dev-support/findbugsExcludeFile.xml  |    5 +
 hadoop-ozone/common/pom.xml                     |  108 ++
 hadoop-ozone/common/src/main/bin/oz             |  332 +++++
 hadoop-ozone/common/src/main/bin/start-ozone.sh |  105 ++
 hadoop-ozone/common/src/main/bin/stop-ozone.sh  |   99 ++
 .../common/src/main/conf/ozone-site.xml         |   24 +
 .../java/org/apache/hadoop/ozone/KsmUtils.java  |   87 ++
 .../ozone/client/io/LengthInputStream.java      |   49 +
 .../hadoop/ozone/client/io/package-info.java    |   22 +
 .../ozone/client/rest/OzoneException.java       |  267 ++++
 .../ozone/client/rest/headers/Header.java       |   71 +
 .../ozone/client/rest/headers/package-info.java |   22 +
 .../hadoop/ozone/client/rest/package-info.java  |   22 +
 .../ozone/client/rest/response/BucketInfo.java  |  231 ++++
 .../ozone/client/rest/response/KeyInfo.java     |  216 +++
 .../ozone/client/rest/response/VolumeInfo.java  |  215 +++
 .../ozone/client/rest/response/VolumeOwner.java |   61 +
 .../client/rest/response/package-info.java      |   24 +
 .../apache/hadoop/ozone/ksm/KSMConfigKeys.java  |   81 ++
 .../hadoop/ozone/ksm/helpers/KsmBucketArgs.java |  233 ++++
 .../hadoop/ozone/ksm/helpers/KsmBucketInfo.java |  235 ++++
 .../hadoop/ozone/ksm/helpers/KsmKeyArgs.java    |  119 ++
 .../hadoop/ozone/ksm/helpers/KsmKeyInfo.java    |  243 ++++
 .../ozone/ksm/helpers/KsmKeyLocationInfo.java   |  136 ++
 .../ksm/helpers/KsmKeyLocationInfoGroup.java    |  118 ++
 .../ozone/ksm/helpers/KsmOzoneAclMap.java       |  110 ++
 .../hadoop/ozone/ksm/helpers/KsmVolumeArgs.java |  223 +++
 .../ozone/ksm/helpers/OpenKeySession.java       |   50 +
 .../hadoop/ozone/ksm/helpers/ServiceInfo.java   |  237 ++++
 .../hadoop/ozone/ksm/helpers/VolumeArgs.java    |  140 ++
 .../hadoop/ozone/ksm/helpers/package-info.java  |   18 +
 .../apache/hadoop/ozone/ksm/package-info.java   |   21 +
 .../ksm/protocol/KeySpaceManagerProtocol.java   |  245 ++++
 .../hadoop/ozone/ksm/protocol/package-info.java |   19 +
 ...ceManagerProtocolClientSideTranslatorPB.java |  744 ++++++++++
 .../protocolPB/KeySpaceManagerProtocolPB.java   |   34 +
 .../ozone/ksm/protocolPB/package-info.java      |   19 +
 .../org/apache/hadoop/ozone/package-info.java   |   22 +
 .../hadoop/ozone/protocolPB/KSMPBHelper.java    |  113 ++
 .../hadoop/ozone/protocolPB/package-info.java   |   24 +
 .../apache/hadoop/ozone/tools/OzoneGetConf.java |  269 ++++
 .../apache/hadoop/ozone/tools/package-info.java |   21 +
 .../hadoop/ozone/web/handlers/BucketArgs.java   |  214 +++
 .../hadoop/ozone/web/handlers/KeyArgs.java      |  117 ++
 .../hadoop/ozone/web/handlers/ListArgs.java     |  142 ++
 .../hadoop/ozone/web/handlers/UserArgs.java     |  168 +++
 .../hadoop/ozone/web/handlers/VolumeArgs.java   |  142 ++
 .../hadoop/ozone/web/handlers/package-info.java |   22 +
 .../hadoop/ozone/web/request/OzoneQuota.java    |  214 +++
 .../hadoop/ozone/web/request/package-info.java  |   23 +
 .../hadoop/ozone/web/response/BucketInfo.java   |  325 +++++
 .../hadoop/ozone/web/response/KeyInfo.java      |  290 ++++
 .../hadoop/ozone/web/response/ListBuckets.java  |  154 +++
 .../hadoop/ozone/web/response/ListKeys.java     |  209 +++
 .../hadoop/ozone/web/response/ListVolumes.java  |  152 ++
 .../hadoop/ozone/web/response/VolumeInfo.java   |  308 +++++
 .../hadoop/ozone/web/response/VolumeOwner.java  |   60 +
 .../hadoop/ozone/web/response/package-info.java |   23 +
 .../hadoop/ozone/web/utils/OzoneUtils.java      |  227 +++
 .../hadoop/ozone/web/utils/package-info.java    |   18 +
 .../main/proto/KeySpaceManagerProtocol.proto    |  457 ++++++
 .../src/main/shellprofile.d/hadoop-ozone.sh     |   46 +
 .../apache/hadoop/ozone/web/TestBucketInfo.java |   81 ++
 .../hadoop/ozone/web/TestOzoneHttpServer.java   |   98 ++
 .../org/apache/hadoop/ozone/web/TestQuota.java  |  116 ++
 .../org/apache/hadoop/ozone/web/TestUtils.java  |  101 ++
 .../hadoop/ozone/web/TestVolumeStructs.java     |   73 +
 .../apache/hadoop/ozone/web/package-info.java   |   21 +
 hadoop-ozone/integration-test/pom.xml           |  117 ++
 .../hadoop/ozone/MiniOzoneClassicCluster.java   |  615 ++++++++
 .../apache/hadoop/ozone/MiniOzoneCluster.java   |   62 +
 .../hadoop/ozone/MiniOzoneTestHelper.java       |   60 +
 .../apache/hadoop/ozone/RatisTestHelper.java    |  115 ++
 .../hadoop/ozone/TestContainerOperations.java   |   90 ++
 .../hadoop/ozone/TestMiniOzoneCluster.java      |  254 ++++
 .../ozone/TestOzoneConfigurationFields.java     |   38 +
 .../apache/hadoop/ozone/TestOzoneHelper.java    |  414 ++++++
 .../ozone/TestStorageContainerManager.java      |  447 ++++++
 .../TestStorageContainerManagerHelper.java      |  180 +++
 .../ozone/client/rest/TestOzoneRestClient.java  |  405 ++++++
 .../hadoop/ozone/client/rest/package-info.java  |   23 +
 .../ozone/client/rpc/TestOzoneRpcClient.java    |  815 +++++++++++
 .../hadoop/ozone/client/rpc/package-info.java   |   23 +
 .../ozone/container/ContainerTestHelper.java    |  595 ++++++++
 .../common/TestBlockDeletingService.java        |  398 ++++++
 .../TestContainerDeletionChoosingPolicy.java    |  197 +++
 .../common/impl/TestContainerPersistence.java   |  905 ++++++++++++
 .../TestCloseContainerHandler.java              |  113 ++
 .../container/metrics/TestContainerMetrics.java |  124 ++
 .../container/ozoneimpl/TestOzoneContainer.java |  584 ++++++++
 .../ozoneimpl/TestOzoneContainerRatis.java      |  143 ++
 .../container/ozoneimpl/TestRatisManager.java   |  129 ++
 .../container/server/TestContainerServer.java   |  258 ++++
 .../ozone/ksm/TestContainerReportWithKeys.java  |  142 ++
 .../apache/hadoop/ozone/ksm/TestKSMMetrcis.java |  307 ++++
 .../apache/hadoop/ozone/ksm/TestKSMSQLCli.java  |  273 ++++
 .../hadoop/ozone/ksm/TestKeySpaceManager.java   | 1238 +++++++++++++++++
 .../ksm/TestKeySpaceManagerRestInterface.java   |  140 ++
 .../ozone/ksm/TestKsmBlockVersioning.java       |  254 ++++
 .../ksm/TestMultipleContainerReadWrite.java     |  216 +++
 .../hadoop/ozone/ozShell/TestOzoneShell.java    |  804 +++++++++++
 .../hadoop/ozone/scm/TestAllocateContainer.java |  104 ++
 .../hadoop/ozone/scm/TestContainerSQLCli.java   |  300 ++++
 .../ozone/scm/TestContainerSmallFile.java       |  151 ++
 .../org/apache/hadoop/ozone/scm/TestSCMCli.java |  535 +++++++
 .../apache/hadoop/ozone/scm/TestSCMMXBean.java  |  162 +++
 .../apache/hadoop/ozone/scm/TestSCMMetrics.java |  251 ++++
 .../ozone/scm/TestXceiverClientManager.java     |  196 +++
 .../ozone/scm/TestXceiverClientMetrics.java     |  172 +++
 .../BenchmarkContainerStateMap.java             |  180 +++
 .../ContainerStates/TestContainerStateMap.java  |  226 +++
 .../container/TestContainerStateManager.java    |  303 ++++
 .../hadoop/ozone/scm/node/TestQueryNode.java    |  122 ++
 .../hadoop/ozone/tools/TestDataValidate.java    |  146 ++
 .../apache/hadoop/ozone/tools/TestFreon.java    |  136 ++
 .../apache/hadoop/ozone/tools/package-info.java |   21 +
 .../ozone/web/TestDistributedOzoneVolumes.java  |  186 +++
 .../hadoop/ozone/web/TestLocalOzoneVolumes.java |  185 +++
 .../ozone/web/TestOzoneRestWithMiniCluster.java |  258 ++++
 .../hadoop/ozone/web/TestOzoneWebAccess.java    |  126 ++
 .../hadoop/ozone/web/client/TestBuckets.java    |  251 ++++
 .../ozone/web/client/TestBucketsRatis.java      |   76 +
 .../hadoop/ozone/web/client/TestKeys.java       |  692 +++++++++
 .../hadoop/ozone/web/client/TestKeysRatis.java  |  126 ++
 .../ozone/web/client/TestOzoneClient.java       |  306 ++++
 .../hadoop/ozone/web/client/TestVolume.java     |  434 ++++++
 .../ozone/web/client/TestVolumeRatis.java       |  131 ++
 .../src/test/resources/webapps/ksm/.gitkeep     |   15 +
 .../src/test/resources/webapps/scm/.gitkeep     |   15 +
 hadoop-ozone/objectstore-service/pom.xml        |   67 +
 .../server/datanode/ObjectStoreHandler.java     |  191 +++
 .../hdfs/server/datanode/package-info.java      |   22 +
 .../org/apache/hadoop/ozone/OzoneRestUtils.java |  222 +++
 .../org/apache/hadoop/ozone/package-info.java   |   22 +
 .../ozone/web/ObjectStoreApplication.java       |   59 +
 .../hadoop/ozone/web/ObjectStoreRestPlugin.java |  113 ++
 .../hadoop/ozone/web/exceptions/ErrorTable.java |  225 +++
 .../ozone/web/exceptions/package-info.java      |   22 +
 .../ozone/web/handlers/BucketHandler.java       |  197 +++
 .../web/handlers/BucketProcessTemplate.java     |  323 +++++
 .../hadoop/ozone/web/handlers/KeyHandler.java   |  245 ++++
 .../ozone/web/handlers/KeyProcessTemplate.java  |  235 ++++
 .../ozone/web/handlers/ServiceFilter.java       |   61 +
 .../web/handlers/StorageHandlerBuilder.java     |   76 +
 .../ozone/web/handlers/UserHandlerBuilder.java  |   75 +
 .../ozone/web/handlers/VolumeHandler.java       |  272 ++++
 .../web/handlers/VolumeProcessTemplate.java     |  276 ++++
 .../hadoop/ozone/web/handlers/package-info.java |   22 +
 .../hadoop/ozone/web/interfaces/Accounting.java |   57 +
 .../hadoop/ozone/web/interfaces/Bucket.java     |  183 +++
 .../hadoop/ozone/web/interfaces/Keys.java       |  146 ++
 .../ozone/web/interfaces/StorageHandler.java    |  295 ++++
 .../hadoop/ozone/web/interfaces/UserAuth.java   |  101 ++
 .../hadoop/ozone/web/interfaces/Volume.java     |  182 +++
 .../ozone/web/interfaces/package-info.java      |   22 +
 .../web/localstorage/LocalStorageHandler.java   |  379 +++++
 .../web/localstorage/OzoneMetadataManager.java  | 1138 +++++++++++++++
 .../ozone/web/localstorage/package-info.java    |   18 +
 .../LengthInputStreamMessageBodyWriter.java     |   59 +
 .../web/messages/StringMessageBodyWriter.java   |   62 +
 .../hadoop/ozone/web/messages/package-info.java |   18 +
 .../web/netty/CloseableCleanupListener.java     |   46 +
 .../web/netty/ObjectStoreChannelHandler.java    |   78 ++
 .../web/netty/ObjectStoreJerseyContainer.java   |  348 +++++
 .../ObjectStoreJerseyContainerProvider.java     |   40 +
 .../web/netty/ObjectStoreRestHttpServer.java    |  256 ++++
 .../web/netty/ObjectStoreURLDispatcher.java     |   61 +
 ...RequestContentObjectStoreChannelHandler.java |  117 ++
 ...equestDispatchObjectStoreChannelHandler.java |  103 ++
 .../hadoop/ozone/web/netty/package-info.java    |   26 +
 .../apache/hadoop/ozone/web/package-info.java   |   22 +
 .../web/storage/DistributedStorageHandler.java  |  539 ++++++++
 .../hadoop/ozone/web/storage/package-info.java  |   27 +
 .../hadoop/ozone/web/userauth/Simple.java       |  169 +++
 .../hadoop/ozone/web/userauth/package-info.java |   23 +
 ...m.sun.jersey.spi.container.ContainerProvider |   16 +
 .../org/apache/hadoop/ozone/TestErrorCode.java  |   53 +
 hadoop-ozone/ozone-manager/pom.xml              |  108 ++
 .../apache/hadoop/ozone/ksm/BucketManager.java  |   79 ++
 .../hadoop/ozone/ksm/BucketManagerImpl.java     |  315 +++++
 .../org/apache/hadoop/ozone/ksm/KSMMXBean.java  |   31 +
 .../hadoop/ozone/ksm/KSMMetadataManager.java    |  253 ++++
 .../ozone/ksm/KSMMetadataManagerImpl.java       |  525 +++++++
 .../org/apache/hadoop/ozone/ksm/KSMMetrics.java |  437 ++++++
 .../org/apache/hadoop/ozone/ksm/KSMStorage.java |   90 ++
 .../hadoop/ozone/ksm/KeyDeletingService.java    |  141 ++
 .../org/apache/hadoop/ozone/ksm/KeyManager.java |  165 +++
 .../apache/hadoop/ozone/ksm/KeyManagerImpl.java |  512 +++++++
 .../hadoop/ozone/ksm/KeySpaceManager.java       |  908 ++++++++++++
 .../ozone/ksm/KeySpaceManagerHttpServer.java    |   78 ++
 .../hadoop/ozone/ksm/OpenKeyCleanupService.java |  116 ++
 .../ozone/ksm/ServiceListJSONServlet.java       |  103 ++
 .../apache/hadoop/ozone/ksm/VolumeManager.java  |  100 ++
 .../hadoop/ozone/ksm/VolumeManagerImpl.java     |  391 ++++++
 .../ozone/ksm/exceptions/KSMException.java      |  116 ++
 .../ozone/ksm/exceptions/package-info.java      |   19 +
 .../apache/hadoop/ozone/ksm/package-info.java   |   21 +
 ...ceManagerProtocolServerSideTranslatorPB.java |  539 ++++++++
 .../hadoop/ozone/protocolPB/package-info.java   |   22 +
 .../hadoop/ozone/web/ozShell/Handler.java       |   79 ++
 .../apache/hadoop/ozone/web/ozShell/Shell.java  |  415 ++++++
 .../web/ozShell/bucket/CreateBucketHandler.java |   96 ++
 .../web/ozShell/bucket/DeleteBucketHandler.java |   87 ++
 .../web/ozShell/bucket/InfoBucketHandler.java   |   94 ++
 .../web/ozShell/bucket/ListBucketHandler.java   |  113 ++
 .../web/ozShell/bucket/UpdateBucketHandler.java |   94 ++
 .../ozone/web/ozShell/bucket/package-info.java  |   23 +
 .../web/ozShell/keys/DeleteKeyHandler.java      |   96 ++
 .../ozone/web/ozShell/keys/GetKeyHandler.java   |  116 ++
 .../ozone/web/ozShell/keys/InfoKeyHandler.java  |   98 ++
 .../ozone/web/ozShell/keys/ListKeyHandler.java  |  116 ++
 .../ozone/web/ozShell/keys/PutKeyHandler.java   |  106 ++
 .../ozone/web/ozShell/keys/package-info.java    |   23 +
 .../hadoop/ozone/web/ozShell/package-info.java  |   27 +
 .../web/ozShell/volume/CreateVolumeHandler.java |   99 ++
 .../web/ozShell/volume/DeleteVolumeHandler.java |   81 ++
 .../web/ozShell/volume/InfoVolumeHandler.java   |   95 ++
 .../web/ozShell/volume/ListVolumeHandler.java   |  114 ++
 .../web/ozShell/volume/UpdateVolumeHandler.java |   92 ++
 .../ozone/web/ozShell/volume/package-info.java  |   23 +
 .../apache/hadoop/ozone/web/package-info.java   |   24 +
 .../src/main/site/markdown/OzoneCommandShell.md |  150 ++
 .../site/markdown/OzoneGettingStarted.md.vm     |  320 +++++
 .../src/main/site/markdown/OzoneMetrics.md      |  166 +++
 .../src/main/site/markdown/OzoneOverview.md     |   88 ++
 .../src/main/site/markdown/OzoneRest.md         |  549 ++++++++
 .../src/main/webapps/ksm/index.html             |   70 +
 .../src/main/webapps/ksm/ksm-metrics.html       |   44 +
 .../ozone-manager/src/main/webapps/ksm/ksm.js   |  108 ++
 .../ozone-manager/src/main/webapps/ksm/main.css |   23 +
 .../src/main/webapps/ksm/main.html              |   18 +
 .../hadoop/ozone/ksm/TestBucketManagerImpl.java |  395 ++++++
 .../hadoop/ozone/ksm/TestChunkStreams.java      |  234 ++++
 .../ksm/TestKeySpaceManagerHttpServer.java      |  141 ++
 .../apache/hadoop/ozone/ksm/package-info.java   |   21 +
 hadoop-ozone/pom.xml                            |  100 ++
 hadoop-ozone/tools/pom.xml                      |   47 +
 .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java |  720 ++++++++++
 .../hadoop/ozone/scm/cli/package-info.java      |   22 +
 .../org/apache/hadoop/ozone/tools/Freon.java    | 1146 +++++++++++++++
 .../apache/hadoop/ozone/tools/package-info.java |   22 +
 .../org/apache/hadoop/test/OzoneTestDriver.java |   59 +
 hadoop-project/pom.xml                          |  116 ++
 hadoop-tools/hadoop-ozone/pom.xml               |    9 +-
 .../hadoop/fs/ozone/TestOzoneFSInputStream.java |    2 +-
 .../fs/ozone/TestOzoneFileInterfaces.java       |  235 ----
 .../contract/ITestOzoneContractCreate.java      |   48 -
 .../contract/ITestOzoneContractDelete.java      |   48 -
 .../contract/ITestOzoneContractDistCp.java      |   50 -
 .../ITestOzoneContractGetFileStatus.java        |   61 -
 .../ozone/contract/ITestOzoneContractMkdir.java |   48 -
 .../ozone/contract/ITestOzoneContractOpen.java  |   47 -
 .../contract/ITestOzoneContractRename.java      |   49 -
 .../contract/ITestOzoneContractRootDir.java     |   51 -
 .../ozone/contract/ITestOzoneContractSeek.java  |   47 -
 .../hadoop/fs/ozone/contract/OzoneContract.java |  125 --
 .../src/test/resources/contract/ozone.xml       |  113 --
 .../src/test/resources/log4j.properties         |   23 -
 .../fs/ozone/TestOzoneFileInterfaces.java       |  235 ++++
 .../contract/ITestOzoneContractCreate.java      |   48 +
 .../contract/ITestOzoneContractDelete.java      |   48 +
 .../contract/ITestOzoneContractDistCp.java      |   50 +
 .../ITestOzoneContractGetFileStatus.java        |   61 +
 .../ozone/contract/ITestOzoneContractMkdir.java |   48 +
 .../ozone/contract/ITestOzoneContractOpen.java  |   47 +
 .../contract/ITestOzoneContractRename.java      |   49 +
 .../contract/ITestOzoneContractRootDir.java     |   51 +
 .../ozone/contract/ITestOzoneContractSeek.java  |   47 +
 .../hadoop/fs/ozone/contract/OzoneContract.java |  125 ++
 .../src/todo/resources/contract/ozone.xml       |  113 ++
 .../src/todo/resources/log4j.properties         |   23 +
 hadoop-tools/hadoop-tools-dist/pom.xml          |    2 +-
 .../yarn/api/records/ContainerRetryContext.java |   21 +-
 .../src/main/proto/yarn_protos.proto            |    1 +
 .../distributedshell/ApplicationMaster.java     |   10 +-
 .../applications/distributedshell/Client.java   |    7 +
 .../impl/pb/ContainerRetryContextPBImpl.java    |   15 +
 .../container/ContainerImpl.java                |   86 +-
 .../container/SlidingWindowRetryPolicy.java     |  165 +++
 .../recovery/NMLeveldbStateStoreService.java    |   24 +
 .../recovery/NMNullStateStoreService.java       |    5 +
 .../recovery/NMStateStoreService.java           |   21 +
 .../container/TestContainer.java                |   37 +
 .../container/TestSlidingWindowRetryPolicy.java |   77 ++
 .../recovery/NMMemoryStateStoreService.java     |    9 +
 .../TestNMLeveldbStateStoreService.java         |   16 +
 .../UserGroupMappingPlacementRule.java          |    7 +
 .../server/resourcemanager/TestAppManager.java  |    4 +-
 .../TestUserGroupMappingPlacementRule.java      |   21 +
 .../capacity/TestCapacityScheduler.java         |   34 +
 ...stCapacitySchedulerAutoCreatedQueueBase.java |   47 +-
 .../TestCapacitySchedulerAutoQueueCreation.java |   43 +-
 .../TestQueueManagementDynamicEditPolicy.java   |    4 +-
 pom.xml                                         |    3 +
 1372 files changed, 110389 insertions(+), 105816 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/dev-support/bin/dist-layout-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/dist-layout-stitching b/dev-support/bin/dist-layout-stitching
index fdbc83c..5579435 100755
--- a/dev-support/bin/dist-layout-stitching
+++ b/dev-support/bin/dist-layout-stitching
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 # project.version
 VERSION=$1
 
@@ -129,6 +128,24 @@ run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}"
 run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${VERSION}" .
 run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-${VERSION}" .
 run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-native-client/target/hadoop-hdfs-native-client-${VERSION}" .
+
+run copy "${ROOT}/hadoop-hdsl/common/target/hadoop-hdsl-common-${VERSION}" .
+run copy "${ROOT}/hadoop-hdsl/framework/target/hadoop-hdsl-server-framework-${VERSION}" .
+run copy "${ROOT}/hadoop-hdsl/server-scm/target/hadoop-hdsl-server-scm-${VERSION}" .
+run copy "${ROOT}/hadoop-hdsl/tools/target/hadoop-hdsl-tools-${VERSION}" .
+run copy "${ROOT}/hadoop-hdsl/container-service/target/hadoop-hdsl-container-service-${VERSION}" .
+run copy "${ROOT}/hadoop-ozone/common/target/hadoop-ozone-common-${VERSION}" .
+run copy "${ROOT}/hadoop-ozone/ozone-manager/target/hadoop-ozone-ozone-manager-${VERSION}" .
+run copy "${ROOT}/hadoop-ozone/ozone-client/target/hadoop-ozone-client-${VERSION}" .
+run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${VERSION}" .
+run copy "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${VERSION}" .
+run copy "${ROOT}/hadoop-cblock/server/target/hadoop-cblock-server-${VERSION}" .
+run copy "${ROOT}/hadoop-cblock/tools/target/hadoop-cblock-tools-${VERSION}" .
+run cp -r "${ROOT}/hadoop-hdsl/framework/target/hadoop-hdsl-server-framework-${VERSION}/share/hadoop/hdsl/webapps/static" share/hadoop/ozone/webapps/
+run cp -r "${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}/share/hadoop/hdfs/webapps/static" share/hadoop/ozone/webapps/
+run cp -r "${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}/share/hadoop/hdfs/webapps/static" share/hadoop/hdsl/webapps/
+
+
 run copy "${ROOT}/hadoop-yarn-project/target/hadoop-yarn-project-${VERSION}" .
 run copy "${ROOT}/hadoop-mapreduce-project/target/hadoop-mapreduce-${VERSION}" .
 run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERSION}" .

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-cblock/pom.xml b/hadoop-cblock/pom.xml
new file mode 100644
index 0000000..c6eb33e
--- /dev/null
+++ b/hadoop-cblock/pom.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-project-dist</artifactId>
+    <version>3.2.0-SNAPSHOT</version>
+    <relativePath>../hadoop-project-dist</relativePath>
+  </parent>
+  <artifactId>hadoop-cblock</artifactId>
+  <version>3.2.0-SNAPSHOT</version>
+  <description>Apache Hadoop Cblock parent project</description>
+  <name>Apache Hadoop Cblock</name>
+  <packaging>pom</packaging>
+
+  <modules>
+    <module>server</module>
+    <module>tools</module>
+  </modules>
+
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>findbugs-maven-plugin</artifactId>
+        <configuration>
+          <excludeFilterFile combine.self="override"></excludeFilterFile>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/dev-support/findbugsExcludeFile.xml b/hadoop-cblock/server/dev-support/findbugsExcludeFile.xml
new file mode 100644
index 0000000..54d602c
--- /dev/null
+++ b/hadoop-cblock/server/dev-support/findbugsExcludeFile.xml
@@ -0,0 +1,21 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<FindBugsFilter>
+  <Match>
+    <Package name="org.apache.hadoop.cblock.protocol.proto"/>
+  </Match>
+</FindBugsFilter>


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[37/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto
deleted file mode 100644
index 0cc1482..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto
+++ /dev/null
@@ -1,161 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are private and unstable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
- * for what changes are allowed for a *unstable* .proto interface.
- */
-
-option java_package = "org.apache.hadoop.ozone.protocol.proto";
-option java_outer_classname = "OzoneProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.hdfs.ozone;
-import "hdfs.proto";
-
-message PipelineChannel {
-    required string leaderID = 1;
-    repeated DatanodeIDProto members = 2;
-    optional LifeCycleState state = 3 [default = OPEN];
-    optional ReplicationType type = 4 [default = STAND_ALONE];
-    optional ReplicationFactor factor = 5 [default = ONE];
-    optional string name = 6;
-}
-
-// A pipeline is composed of PipelineChannel (Ratis/StandAlone) that back a
-// container.
-message Pipeline {
-    required string containerName = 1;
-    required PipelineChannel pipelineChannel = 2;
-}
-
-message KeyValue {
-    required string key = 1;
-    optional string value = 2;
-}
-
-/**
- * Type of the node.
- */
-enum NodeType {
-    KSM = 1;
-    SCM = 2;
-    DATANODE = 3;
-}
-
-// Should we rename NodeState to DatanodeState?
-/**
- * Enum that represents the Node State. This is used in calls to getNodeList
- * and getNodeCount.
- */
-enum NodeState {
-    HEALTHY             = 1;
-    STALE               = 2;
-    DEAD                = 3;
-    DECOMMISSIONING     = 4;
-    DECOMMISSIONED      = 5;
-    RAFT_MEMBER         = 6;
-    FREE_NODE           = 7; // Not a member in raft.
-    UNKNOWN             = 8;
-}
-
-enum QueryScope {
-    CLUSTER = 1;
-    POOL = 2;
-}
-
-message Node {
-    required DatanodeIDProto nodeID = 1;
-    repeated NodeState nodeStates = 2;
-}
-
-message NodePool {
-    repeated Node nodes = 1;
-}
-
-/**
- * LifeCycleState for SCM object creation state machine:
- *    ->Allocated: allocated on SCM but clean has not started creating it yet.
- *    ->Creating: allocated and assigned to client to create but not ack-ed yet.
- *    ->Open: allocated on SCM and created on datanodes and ack-ed by a client.
- *    ->Close: container closed due to space all used or error?
- *    ->Timeout -> container failed to create on datanodes or ack-ed by client.
- *    ->Deleting(TBD) -> container will be deleted after timeout
- * 1. ALLOCATE-ed containers on SCM can't serve key/block related operation
- *    until ACK-ed explicitly which changes the state to OPEN.
- * 2. Only OPEN/CLOSED containers can serve key/block related operation.
- * 3. ALLOCATE-ed containers that are not ACK-ed timely will be TIMEOUT and
- *    CLEANUP asynchronously.
- */
-
-enum LifeCycleState {
-    ALLOCATED = 1;
-    CREATING = 2; // Used for container allocated/created by different client.
-    OPEN =3; // Mostly an update to SCM via HB or client call.
-    CLOSING = 4;
-    CLOSED = 5; // !!State after this has not been used yet.
-    DELETING = 6;
-    DELETED = 7; // object is deleted.
-}
-
-enum LifeCycleEvent {
-    CREATE = 1; // A request to client to create this object
-    CREATED = 2;
-    FINALIZE = 3;
-    CLOSE = 4; // !!Event after this has not been used yet.
-    UPDATE = 5;
-    TIMEOUT = 6; // creation has timed out from SCM's View.
-    DELETE = 7;
-    CLEANUP = 8;
-}
-
-message SCMContainerInfo {
-    // TODO : Remove the container name from pipeline.
-    required string containerName = 1;
-    required LifeCycleState state = 2;
-    required Pipeline pipeline = 3;
-    // This is not total size of container, but space allocated by SCM for
-    // clients to write blocks
-    required uint64 allocatedBytes = 4;
-    required uint64 usedBytes = 5;
-    required uint64 numberOfKeys = 6;
-    optional int64 stateEnterTime = 7;
-    required string owner = 8;
-    required int64 containerID = 9;
-}
-
-message GetScmInfoRequestProto {
-}
-
-message GetScmInfoRespsonseProto {
-    required string clusterId = 1;
-    required string scmId = 2;
-}
-
-
-enum ReplicationType {
-    RATIS = 1;
-    STAND_ALONE = 2;
-    CHAINED = 3;
-}
-
-enum ReplicationFactor {
-    ONE = 1;
-    THREE = 3;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ScmBlockLocationProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ScmBlockLocationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ScmBlockLocationProtocol.proto
deleted file mode 100644
index 99536ac..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ScmBlockLocationProtocol.proto
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are private and unstable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
- * for what changes are allowed for a *unstable* .proto interface.
- */
-
-option java_package = "org.apache.hadoop.ozone.protocol.proto";
-option java_outer_classname = "ScmBlockLocationProtocolProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.hdfs;
-
-import "hdfs.proto";
-import "Ozone.proto";
-
-
-// SCM Block protocol
-/**
- * keys - batch of block keys to find
- */
-message GetScmBlockLocationsRequestProto {
-  repeated string keys = 1;
-}
-
-/**
- * locatedBlocks - for each requested hash, nodes that currently host the
- *     container for that object key hash
- */
-message GetScmBlockLocationsResponseProto {
-  repeated ScmLocatedBlockProto locatedBlocks = 1;
-}
-
-/**
- * Holds the nodes that currently host the blocks for a key.
- */
-message ScmLocatedBlockProto {
-  required string key = 1;
-  required hadoop.hdfs.ozone.Pipeline pipeline = 2;
-}
-
-/**
-* Request send to SCM asking allocate block of specified size.
-*/
-message AllocateScmBlockRequestProto {
-  required uint64 size = 1;
-  required hadoop.hdfs.ozone.ReplicationType type = 2;
-  required hadoop.hdfs.ozone.ReplicationFactor factor = 3;
-  required string owner = 4;
-
-}
-
-/**
- * A delete key request sent by KSM to SCM, it contains
- * multiple number of keys (and their blocks).
- */
-message DeleteScmKeyBlocksRequestProto {
-  repeated KeyBlocks keyBlocks = 1;
-}
-
-/**
- * A object key and all its associated blocks.
- * We need to encapsulate object key name plus the blocks in this potocol
- * because SCM needs to response KSM with the keys it has deleted.
- * If the response only contains blocks, it will be very expensive for
- * KSM to figure out what keys have been deleted.
- */
-message KeyBlocks {
-  required string key = 1;
-  repeated string blocks = 2;
-}
-
-/**
- * A delete key response from SCM to KSM, it contains multiple child-results.
- * Each child-result represents a key deletion result, only if all blocks of
- * a key are successfully deleted, this key result is considered as succeed.
- */
-message DeleteScmKeyBlocksResponseProto {
-  repeated DeleteKeyBlocksResultProto results = 1;
-}
-
-/**
- * A key deletion result. It contains all the block deletion results.
- */
-message DeleteKeyBlocksResultProto {
-  required string objectKey = 1;
-  repeated DeleteScmBlockResult blockResults = 2;
-}
-
-message DeleteScmBlockResult {
-  enum Result {
-    success = 1;
-    chillMode = 2;
-    errorNotFound = 3;
-    unknownFailure = 4;
-  }
-  required Result result = 1;
-  required string key = 2;
-}
-
-/**
- * Reply from SCM indicating that the container.
- */
-message AllocateScmBlockResponseProto {
-  enum Error {
-    success = 1;
-    errorNotEnoughSpace = 2;
-    errorSizeTooBig = 3;
-    unknownFailure = 4;
-  }
-  required Error errorCode = 1;
-  required string key = 2;
-  required hadoop.hdfs.ozone.Pipeline pipeline = 3;
-  required bool createContainer = 4;
-  optional string errorMessage = 5;
-}
-
-/**
- * Protocol used from KeySpaceManager to StorageContainerManager.
- * See request and response messages for details of the RPC calls.
- */
-service ScmBlockLocationProtocolService {
-
-  /**
-   * Find the set of nodes that currently host the block, as
-   * identified by the key.  This method supports batch lookup by
-   * passing multiple keys.
-   */
-  rpc getScmBlockLocations(GetScmBlockLocationsRequestProto)
-      returns (GetScmBlockLocationsResponseProto);
-
-  /**
-   * Creates a block entry in SCM.
-   */
-  rpc allocateScmBlock(AllocateScmBlockRequestProto)
-      returns (AllocateScmBlockResponseProto);
-
-  /**
-   * Deletes blocks for a set of object keys from SCM.
-   */
-  rpc deleteScmKeyBlocks(DeleteScmKeyBlocksRequestProto)
-      returns (DeleteScmKeyBlocksResponseProto);
-
-  /**
-   * Gets the scmInfo from SCM.
-   */
-  rpc getScmInfo(hadoop.hdfs.ozone.GetScmInfoRequestProto)
-      returns (hadoop.hdfs.ozone.GetScmInfoRespsonseProto);
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto
deleted file mode 100644
index 512ec75..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto
+++ /dev/null
@@ -1,214 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are private and unstable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
- * for what changes are allowed for a *unstable* .proto interface.
- */
-
-option java_package = "org.apache.hadoop.ozone.protocol.proto";
-option java_outer_classname = "StorageContainerLocationProtocolProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.hdfs;
-
-import "hdfs.proto";
-import "Ozone.proto";
-
-/**
-* Request send to SCM asking where the container should be created.
-*/
-message ContainerRequestProto {
-  required string containerName = 1;
-  // Ozone only support replciation of either 1 or 3.
-  required hadoop.hdfs.ozone.ReplicationFactor replicationFactor = 2;
-  required hadoop.hdfs.ozone.ReplicationType  replicationType = 3;
-  required string owner = 4;
-
-}
-
-/**
- * Reply from SCM indicating that the container.
- */
-message ContainerResponseProto {
-  enum Error {
-    success = 1;
-    errorContainerAlreadyExists = 2;
-    errorContainerMissing = 3;
-  }
-  required Error errorCode = 1;
-  required hadoop.hdfs.ozone.Pipeline pipeline = 2;
-  optional string errorMessage = 3;
-}
-
-message GetContainerRequestProto {
-  required string containerName = 1;
-}
-
-message GetContainerResponseProto {
-  required hadoop.hdfs.ozone.Pipeline pipeline = 1;
-}
-
-message ListContainerRequestProto {
-  required uint32 count = 1;
-  optional string startName = 2;
-  optional string prefixName = 3;
-}
-
-message ListContainerResponseProto {
-  repeated hadoop.hdfs.ozone.SCMContainerInfo containers = 1;
-}
-
-message DeleteContainerRequestProto {
-  required string containerName = 1;
-}
-
-message DeleteContainerResponseProto {
-  // Empty response
-}
-
-message ObjectStageChangeRequestProto {
-  enum Type {
-    container = 1;
-    pipeline = 2;
-  }
-  // delete/copy operation may be added later
-  enum Op {
-    create = 1;
-    close = 2;
-  }
-  enum Stage {
-    begin = 1;
-    complete = 2;
-  }
-  required string name = 1;
-  required Type type = 2;
-  required Op op= 3;
-  required Stage stage = 4;
-}
-
-message ObjectStageChangeResponseProto {
-  // Empty response
-}
-
-/*
- NodeQueryRequest sends a request to SCM asking to send a list of nodes that
- match the NodeState that we are requesting.
-*/
-message NodeQueryRequestProto {
-
-
-  // Repeated, So we can specify more than one status type.
-  // These NodeState types are additive for now, in the sense that
-  // if you specify HEALTHY and FREE_NODE members --
-  // Then you get all healthy node which are not raft members.
-  //
-  // if you specify all healthy and dead nodes, you will get nothing
-  // back. Server is not going to dictate what combinations make sense,
-  // it is entirely up to the caller.
-  // TODO: Support operators like OR and NOT. Currently it is always an
-  // implied AND.
-
-  repeated hadoop.hdfs.ozone.NodeState query = 1;
-  required hadoop.hdfs.ozone.QueryScope scope = 2;
-  optional string poolName = 3; // if scope is pool, then pool name is needed.
-}
-
-message NodeQueryResponseProto {
-  required hadoop.hdfs.ozone.NodePool datanodes = 1;
-}
-
-/**
-  Request to create a replication pipeline.
- */
-message PipelineRequestProto {
-  required hadoop.hdfs.ozone.ReplicationType replicationType = 1;
-  required hadoop.hdfs.ozone.ReplicationFactor replicationFactor = 2;
-
-  // if datanodes are specified then pipelines are created using those
-  // datanodes.
-  optional hadoop.hdfs.ozone.NodePool nodePool = 3;
-  optional string pipelineID = 4;
-}
-
-message  PipelineResponseProto {
-  enum Error {
-    success = 1;
-    errorPipelineAlreadyExists = 2;
-  }
-  required Error errorCode = 1;
-  optional hadoop.hdfs.ozone.Pipeline  pipeline = 2;
-  optional string errorMessage = 3;
-}
-
-/**
- * Protocol used from an HDFS node to StorageContainerManager.  See the request
- * and response messages for details of the RPC calls.
- */
-service StorageContainerLocationProtocolService {
-
-  /**
-   * Creates a container entry in SCM.
-   */
-  rpc allocateContainer(ContainerRequestProto) returns (ContainerResponseProto);
-
-  /**
-   * Returns the pipeline for a given container.
-   */
-  rpc getContainer(GetContainerRequestProto) returns (GetContainerResponseProto);
-
-  rpc listContainer(ListContainerRequestProto) returns (ListContainerResponseProto);
-
-  /**
-   * Deletes a container in SCM.
-   */
-  rpc deleteContainer(DeleteContainerRequestProto) returns (DeleteContainerResponseProto);
-
-  /**
-  * Returns a set of Nodes that meet a criteria.
-  */
-  rpc queryNode(NodeQueryRequestProto)  returns (NodeQueryResponseProto);
-
-  /**
-  * Notify from client when begin or finish container or pipeline operations on datanodes.
-  */
-  rpc notifyObjectStageChange(ObjectStageChangeRequestProto) returns (ObjectStageChangeResponseProto);
-
-  /*
-  *  Apis that Manage Pipelines.
-  *
-  * Pipelines are abstractions offered by SCM and Datanode that allows users
-  * to create a replication pipeline.
-  *
-  *  These following APIs allow command line programs like SCM CLI to list
-  * and manage pipelines.
-  */
-
-  /**
-  *  Creates a replication pipeline.
-  */
-  rpc allocatePipeline(PipelineRequestProto)
-      returns (PipelineResponseProto);
-
-  /**
-  *  Returns information about SCM.
-  */
-  rpc getScmInfo(hadoop.hdfs.ozone.GetScmInfoRequestProto)
-      returns (hadoop.hdfs.ozone.GetScmInfoRespsonseProto);
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index e9b5c24..4db2d58 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -75,6 +75,7 @@ message DatanodeIDProto {
   optional uint32 infoSecurePort = 7 [default = 0]; // datanode https port
   optional uint32 containerPort = 8 [default = 0]; // Ozone stand_alone protocol
   optional uint32 ratisPort = 9 [default = 0]; //Ozone ratis port
+  optional uint32 ozoneRestPort = 10 [default = 0];
 }
 
 /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java
deleted file mode 100644
index 03c45c5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone;
-
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * This class is to test acl stoarge and retreival in ozone store.
- */
-public class TestOzoneAcls {
-
-  @Test
-  public void testAclParse() {
-    HashMap<String, Boolean> testMatrix;
-    testMatrix = new HashMap<>();
-
-    testMatrix.put("user:bilbo:r", Boolean.TRUE);
-    testMatrix.put("user:bilbo:w", Boolean.TRUE);
-    testMatrix.put("user:bilbo:rw", Boolean.TRUE);
-    testMatrix.put("user:bilbo:wr", Boolean.TRUE);
-    testMatrix.put("    user:bilbo:wr   ", Boolean.TRUE);
-
-
-    // ACLs makes no judgement on the quality of
-    // user names. it is for the userAuth interface
-    // to determine if a user name is really a name
-    testMatrix.put(" user:*:rw", Boolean.TRUE);
-    testMatrix.put(" user:~!:rw", Boolean.TRUE);
-
-
-    testMatrix.put("", Boolean.FALSE);
-    testMatrix.put(null, Boolean.FALSE);
-    testMatrix.put(" user:bilbo:", Boolean.FALSE);
-    testMatrix.put(" user:bilbo:rx", Boolean.FALSE);
-    testMatrix.put(" user:bilbo:mk", Boolean.FALSE);
-    testMatrix.put(" user::rw", Boolean.FALSE);
-    testMatrix.put("user11:bilbo:rw", Boolean.FALSE);
-    testMatrix.put(" user:::rw", Boolean.FALSE);
-
-    testMatrix.put(" group:hobbit:r", Boolean.TRUE);
-    testMatrix.put(" group:hobbit:w", Boolean.TRUE);
-    testMatrix.put(" group:hobbit:rw", Boolean.TRUE);
-    testMatrix.put(" group:hobbit:wr", Boolean.TRUE);
-    testMatrix.put(" group:*:rw", Boolean.TRUE);
-    testMatrix.put(" group:~!:rw", Boolean.TRUE);
-
-    testMatrix.put(" group:hobbit:", Boolean.FALSE);
-    testMatrix.put(" group:hobbit:rx", Boolean.FALSE);
-    testMatrix.put(" group:hobbit:mk", Boolean.FALSE);
-    testMatrix.put(" group::", Boolean.FALSE);
-    testMatrix.put(" group::rw", Boolean.FALSE);
-    testMatrix.put(" group22:hobbit:", Boolean.FALSE);
-    testMatrix.put(" group:::rw", Boolean.FALSE);
-
-    testMatrix.put("JUNK group:hobbit:r", Boolean.FALSE);
-    testMatrix.put("JUNK group:hobbit:w", Boolean.FALSE);
-    testMatrix.put("JUNK group:hobbit:rw", Boolean.FALSE);
-    testMatrix.put("JUNK group:hobbit:wr", Boolean.FALSE);
-    testMatrix.put("JUNK group:*:rw", Boolean.FALSE);
-    testMatrix.put("JUNK group:~!:rw", Boolean.FALSE);
-
-    testMatrix.put(" world::r", Boolean.TRUE);
-    testMatrix.put(" world::w", Boolean.TRUE);
-    testMatrix.put(" world::rw", Boolean.TRUE);
-    testMatrix.put(" world::wr", Boolean.TRUE);
-
-    testMatrix.put(" world:bilbo:w", Boolean.FALSE);
-    testMatrix.put(" world:bilbo:rw", Boolean.FALSE);
-
-    Set<String> keys = testMatrix.keySet();
-    for (String key : keys) {
-      if (testMatrix.get(key)) {
-        OzoneAcl.parseAcl(key);
-      } else {
-        try {
-          OzoneAcl.parseAcl(key);
-          // should never get here since parseAcl will throw
-          fail("An exception was expected but did not happen.");
-        } catch (IllegalArgumentException e) {
-          // nothing to do
-        }
-      }
-    }
-  }
-
-  @Test
-  public void testAclValues() {
-    OzoneAcl acl = OzoneAcl.parseAcl("user:bilbo:rw");
-    assertEquals(acl.getName(), "bilbo");
-    assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
-    assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
-
-    acl = OzoneAcl.parseAcl("user:bilbo:wr");
-    assertEquals("bilbo", acl.getName());
-    assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
-    assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
-
-    acl = OzoneAcl.parseAcl("user:bilbo:r");
-    assertEquals("bilbo", acl.getName());
-    assertEquals(OzoneAcl.OzoneACLRights.READ, acl.getRights());
-    assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
-
-    acl = OzoneAcl.parseAcl("user:bilbo:w");
-    assertEquals("bilbo", acl.getName());
-    assertEquals(OzoneAcl.OzoneACLRights.WRITE, acl.getRights());
-    assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
-
-    acl = OzoneAcl.parseAcl("group:hobbit:wr");
-    assertEquals(acl.getName(), "hobbit");
-    assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
-    assertEquals(OzoneAcl.OzoneACLType.GROUP, acl.getType());
-
-    acl = OzoneAcl.parseAcl("world::wr");
-    assertEquals(acl.getName(), "");
-    assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
-    assertEquals(OzoneAcl.OzoneACLType.WORLD, acl.getType());
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java
deleted file mode 100644
index d4c626e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.common;
-
-import org.apache.commons.collections.SetUtils;
-import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException;
-import org.apache.hadoop.ozone.common.statemachine.StateMachine;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.INIT;
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CREATING;
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.OPERATIONAL;
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CLOSED;
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CLEANUP;
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.FINAL;
-
-/**
- * This class is to test ozone common state machine.
- */
-public class TestStateMachine {
-
-  /**
-   * STATES used by the test state machine.
-   */
-  public enum STATES {INIT, CREATING, OPERATIONAL, CLOSED, CLEANUP, FINAL};
-
-  /**
-   * EVENTS used by the test state machine.
-   */
-  public enum EVENTS {ALLOCATE, CREATE, UPDATE, CLOSE, DELETE, TIMEOUT};
-
-  @Rule
-  public ExpectedException exception = ExpectedException.none();
-
-  @Test
-  public void testStateMachineStates() throws InvalidStateTransitionException {
-    Set<STATES> finals = new HashSet<>();
-    finals.add(FINAL);
-
-    StateMachine<STATES, EVENTS> stateMachine =
-        new StateMachine<>(INIT, finals);
-
-    stateMachine.addTransition(INIT, CREATING, EVENTS.ALLOCATE);
-    stateMachine.addTransition(CREATING, OPERATIONAL, EVENTS.CREATE);
-    stateMachine.addTransition(OPERATIONAL, OPERATIONAL, EVENTS.UPDATE);
-    stateMachine.addTransition(OPERATIONAL, CLEANUP, EVENTS.DELETE);
-    stateMachine.addTransition(OPERATIONAL, CLOSED, EVENTS.CLOSE);
-    stateMachine.addTransition(CREATING, CLEANUP, EVENTS.TIMEOUT);
-
-    // Initial and Final states
-    Assert.assertEquals("Initial State", INIT, stateMachine.getInitialState());
-    Assert.assertTrue("Final States", SetUtils.isEqualSet(finals,
-        stateMachine.getFinalStates()));
-
-    // Valid state transitions
-    Assert.assertEquals("STATE should be OPERATIONAL after being created",
-        OPERATIONAL, stateMachine.getNextState(CREATING, EVENTS.CREATE));
-    Assert.assertEquals("STATE should be OPERATIONAL after being updated",
-        OPERATIONAL, stateMachine.getNextState(OPERATIONAL, EVENTS.UPDATE));
-    Assert.assertEquals("STATE should be CLEANUP after being deleted",
-        CLEANUP, stateMachine.getNextState(OPERATIONAL, EVENTS.DELETE));
-    Assert.assertEquals("STATE should be CLEANUP after being timeout",
-        CLEANUP, stateMachine.getNextState(CREATING, EVENTS.TIMEOUT));
-    Assert.assertEquals("STATE should be CLOSED after being closed",
-        CLOSED, stateMachine.getNextState(OPERATIONAL, EVENTS.CLOSE));
-
-    // Negative cases: invalid transition
-    expectException();
-    stateMachine.getNextState(OPERATIONAL, EVENTS.CREATE);
-
-    expectException();
-    stateMachine.getNextState(CREATING, EVENTS.CLOSE);
-  }
-
-  /**
-   * We expect an InvalidStateTransitionException.
-   */
-  private void expectException() {
-    exception.expect(InvalidStateTransitionException.class);
-    exception.expectMessage("Invalid event");
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/scm/TestArchive.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/scm/TestArchive.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/scm/TestArchive.java
deleted file mode 100644
index 86adddb..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/scm/TestArchive.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.fs.FileUtil;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Iterator;
-import java.util.Random;
-import java.util.zip.Adler32;
-import java.util.zip.Checksum;
-
-/**
- * Test archive creation and unpacking.
- */
-public class TestArchive {
-  private static final int DIR_COUNT = 10;
-  private static final int SUB_DIR_COUNT = 3;
-  private static final int FILE_COUNT = 10;
-  private long checksumWrite = 0L;
-  private long checksumRead = 0L;
-  private long tmp = 0L;
-  private Checksum crc = new Adler32();
-
-  @Rule
-  public TemporaryFolder folder = new TemporaryFolder();
-
-  @Rule
-  public TemporaryFolder outputFolder = new TemporaryFolder();
-
-
-  @Before
-  public void setUp() throws Exception {
-    Random r = new Random();
-    final int megaByte = 1024 * 1024;
-
-    for (int x = 0; x < DIR_COUNT; x++) {
-      File subdir = folder.newFolder(String.format("dir%d", x));
-      for (int y = 0; y < SUB_DIR_COUNT; y++) {
-        File targetDir = new File(subdir.getPath().concat(File.separator)
-            .concat(String.format("subdir%d%d", x, y)));
-        if(!targetDir.mkdirs()) {
-          throw new IOException("Failed to create subdirectory. " +
-              targetDir.toString());
-        }
-        for (int z = 0; z < FILE_COUNT; z++) {
-          Path temp = Paths.get(targetDir.getPath().concat(File.separator)
-              .concat(String.format("File%d.txt", z)));
-          byte[] buf = RandomUtils.nextBytes(r.nextInt(megaByte));
-          Files.write(temp, buf);
-          crc.reset();
-          crc.update(buf, 0, buf.length);
-          tmp = crc.getValue();
-          checksumWrite +=tmp;
-        }
-      }
-    }
-  }
-
-  @Test
-  public void testArchive() throws Exception {
-    File archiveFile = new File(outputFolder.getRoot() + File.separator
-        + "test.container.zip");
-    long zipCheckSum = FileUtil.zip(folder.getRoot(), archiveFile);
-    Assert.assertTrue(zipCheckSum > 0);
-    File decomp = new File(outputFolder.getRoot() + File.separator +
-        "decompress");
-    if (!decomp.exists() && !decomp.mkdirs()) {
-      throw new IOException("Unable to create the destination directory. " +
-          decomp.getPath());
-    }
-
-    FileUtil.unZip(archiveFile, decomp);
-    String[] patterns = {"txt"};
-    Iterator<File> iter = FileUtils.iterateFiles(decomp, patterns, true);
-    int count = 0;
-    while (iter.hasNext()) {
-      count++;
-      byte[] buf = Files.readAllBytes(iter.next().toPath());
-      crc.reset();
-      crc.update(buf, 0, buf.length);
-      tmp = crc.getValue();
-      checksumRead += tmp;
-    }
-    Assert.assertEquals(DIR_COUNT * SUB_DIR_COUNT * FILE_COUNT, count);
-    Assert.assertEquals(checksumWrite, checksumRead);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/scm/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/scm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/scm/package-info.java
deleted file mode 100644
index 9c480d6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/scm/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm;
-/**
- Test cases for SCM client classes.
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
index b4da6ba7..a99ea75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
@@ -3,9 +3,6 @@
        <Package name="org.apache.hadoop.record.compiler.generated" />
      </Match>
      <Match>
-       <Package name="org.apache.hadoop.hdfs.ozone.protocol.proto" />
-     </Match>
-     <Match>
        <Package name="org.apache.hadoop.hdfs.protocol.proto" />
      </Match>
      <Match>
@@ -18,12 +15,6 @@
        <Package name="org.apache.hadoop.hdfs.qjournal.protocol" />
      </Match>
      <Match>
-       <Package name="org.apache.hadoop.ozone.protocol.proto" />
-     </Match>
-     <Match>
-       <Package name ="org.apache.hadoop.cblock.protocol.proto" />
-     </Match>
-     <Match>
        <Package name="org.apache.hadoop.hdfs.federation.protocol.proto" />
      </Match>
      <Match>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index c57ecc0..909e589 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -169,6 +169,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <scope>compile</scope>
     </dependency>
     <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-all</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
       <groupId>org.apache.htrace</groupId>
       <artifactId>htrace-core4</artifactId>
     </dependency>
@@ -187,17 +192,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <groupId>org.fusesource.leveldbjni</groupId>
       <artifactId>leveldbjni-all</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.rocksdb</groupId>
-      <artifactId>rocksdbjni</artifactId>
-      <version>5.8.0</version>
-    </dependency>
-    <dependency>
-      <groupId>io.swagger</groupId>
-      <artifactId>swagger-annotations</artifactId>
-      <version>1.5.9</version>
-      <scope>provided</scope>
-    </dependency>
     <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
     <dependency>
       <groupId>org.bouncycastle</groupId>
@@ -209,68 +203,15 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <artifactId>jackson-databind</artifactId>
     </dependency>
     <dependency>
-      <groupId>org.jscsi</groupId>
-      <artifactId>target</artifactId>
-      <version>2.6.0</version>
-      <optional>true</optional>
-      <exclusions>
-        <exclusion>
-          <groupId>ch.qos.logback</groupId>
-          <artifactId>logback-classic</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-    <dependency>
-      <groupId>org.jctools</groupId>
-      <artifactId>jctools-core</artifactId>
-      <optional>true</optional>
-    </dependency>
-    <dependency>
-      <groupId>org.xerial</groupId>
-      <artifactId>sqlite-jdbc</artifactId>
-      <version>3.8.7</version>
-    </dependency>
-    <dependency>
-      <groupId>io.kubernetes</groupId>
-      <artifactId>client-java</artifactId>
-      <version>1.0.0-beta1</version>
-      <exclusions>
-        <exclusion>
-          <groupId>io.swagger</groupId>
-          <artifactId>swagger-annotations</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.github.stefanbirkner</groupId>
-          <artifactId>system-rules</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-    <dependency>
       <groupId>org.apache.curator</groupId>
       <artifactId>curator-test</artifactId>
       <scope>test</scope>
     </dependency>
-      <dependency>
-          <groupId>io.dropwizard.metrics</groupId>
-          <artifactId>metrics-core</artifactId>
-      </dependency>
     <dependency>
         <groupId>org.assertj</groupId>
         <artifactId>assertj-core</artifactId>
         <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.openjdk.jmh</groupId>
-      <artifactId>jmh-core</artifactId>
-      <version>1.19</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.openjdk.jmh</groupId>
-      <artifactId>jmh-generator-annprocess</artifactId>
-      <version>1.19</version>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
 
   <build>
@@ -366,7 +307,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
             <configuration>
               <tasks>
                 <copy file="src/main/resources/hdfs-default.xml" todir="src/site/resources"/>
-                <copy file="src/main/resources/ozone-default.xml" todir="src/site/resources"/>
                 <copy file="src/main/xsl/configuration.xsl" todir="src/site/resources"/>
               </tasks>
             </configuration>
@@ -403,9 +343,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
                   <include>QJournalProtocol.proto</include>
                   <include>editlog.proto</include>
                   <include>fsimage.proto</include>
-                  <include>StorageContainerDatanodeProtocol.proto</include>
-                  <include>CBlockServiceProtocol.proto</include>
-                  <include>CBlockClientServerProtocol.proto</include>
                   <include>FederationProtocol.proto</include>
                   <include>RouterProtocol.proto</include>
                   <include>AliasMapProtocol.proto</include>
@@ -472,14 +409,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
             <exclude>src/main/webapps/static/dataTables.bootstrap.js</exclude>
             <exclude>src/main/webapps/static/d3-v4.1.1.min.js</exclude>
             <exclude>src/test/resources/diskBalancer/data-cluster-3node-3disk.json</exclude>
-            <exclude>src/main/webapps/static/nvd3-1.8.5.min.css.map</exclude>
-            <exclude>src/main/webapps/static/nvd3-1.8.5.min.js</exclude>
-            <exclude>src/main/webapps/static/angular-route-1.6.4.min.js</exclude>
-            <exclude>src/main/webapps/static/nvd3-1.8.5.min.css</exclude>
-            <exclude>src/main/webapps/static/angular-nvd3-1.0.9.min.js</exclude>
-            <exclude>src/main/webapps/static/nvd3-1.8.5.min.js.map</exclude>
-            <exclude>src/main/webapps/static/angular-1.6.4.min.js</exclude>
-            <exclude>src/main/webapps/static/d3-3.5.17.min.js</exclude>
           </excludes>
         </configuration>
       </plugin>
@@ -492,55 +421,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
               <includes>
                 <include>configuration.xsl</include>
                 <include>hdfs-default.xml</include>
-                <include>ozone-default.xml</include>
               </includes>
               <followSymlinks>false</followSymlinks>
             </fileset>
           </filesets>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>com.github.kongchen</groupId>
-        <artifactId>swagger-maven-plugin</artifactId>
-        <version>3.1.5</version>
-        <executions>
-          <execution>
-            <phase>compile</phase>
-            <goals>
-              <goal>generate</goal>
-            </goals>
-          </execution>
-        </executions>
-        <configuration>
-          <apiSources>
-            <apiSource>
-              <springmvc>false</springmvc>
-              <swaggerDirectory>target/webapps/static</swaggerDirectory>
-              <swaggerFileName>ozone.swagger</swaggerFileName>
-              <schemes>
-                <scheme>http</scheme>
-              </schemes>
-              <host>localhost:9864</host>
-              <basePath>/</basePath>
-              <locations>
-                <location>org.apache.hadoop.ozone.web.interfaces</location>
-              </locations>
-              <info>
-                <title>HDFS Ozone REST Api</title>
-                <version>${project.version}</version>
-                <contact>
-                  <name>Apache Hadoop project</name>
-                  <url>https://hadoop.apache.org</url>
-                </contact>
-                <license>
-                  <url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
-                  <name>Apache 2.0</name>
-                </license>
-              </info>
-            </apiSource>
-          </apiSources>
-        </configuration>
-      </plugin>
     </plugins>
   </build>
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 4be674b..38be348b 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -32,11 +32,8 @@ function hadoop_usage
   hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
   hadoop_add_option "--workers" "turn on worker mode"
 
-
   hadoop_add_subcommand "balancer" daemon "run a cluster balancing utility"
   hadoop_add_subcommand "cacheadmin" admin "configure the HDFS cache"
-  hadoop_add_subcommand "cblock" admin "cblock CLI"
-  hadoop_add_subcommand "cblockserver" daemon "run cblock server"
   hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries"
   hadoop_add_subcommand "crypto" admin "configure HDFS encryption zones"
   hadoop_add_subcommand "datanode" daemon "run a DFS datanode"
@@ -49,15 +46,12 @@ function hadoop_usage
   hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables"
   hadoop_add_subcommand "ec" admin "run a HDFS ErasureCoding CLI"
   hadoop_add_subcommand "fetchdt" client "fetch a delegation token from the NameNode"
-  hadoop_add_subcommand "freon" client "runs an ozone data generator"
   hadoop_add_subcommand "fsck" admin "run a DFS filesystem checking utility"
   hadoop_add_subcommand "getconf" client "get config values from configuration"
   hadoop_add_subcommand "groups" client "get the groups which users belong to"
   hadoop_add_subcommand "haadmin" admin "run a DFS HA admin client"
   hadoop_add_subcommand "jmxget" admin "get JMX exported values from NameNode or DataNode."
   hadoop_add_subcommand "journalnode" daemon "run the DFS journalnode"
-  hadoop_add_subcommand "jscsi" daemon "run cblock jscsi server"
-  hadoop_add_subcommand "ksm" daemon "Ozone keyspace manager"
   hadoop_add_subcommand "lsSnapshottableDir" client "list all snapshottable dirs owned by the current user"
   hadoop_add_subcommand "mover" daemon "run a utility to move block replicas across storage types"
   hadoop_add_subcommand "namenode" daemon "run the DFS namenode"
@@ -65,17 +59,12 @@ function hadoop_usage
   hadoop_add_subcommand "oev" admin "apply the offline edits viewer to an edits file"
   hadoop_add_subcommand "oiv" admin "apply the offline fsimage viewer to an fsimage"
   hadoop_add_subcommand "oiv_legacy" admin "apply the offline fsimage viewer to a legacy fsimage"
-  hadoop_add_subcommand "oz" client "command line interface for ozone"
-  hadoop_add_subcommand "oz_debug" client "ozone debug tool, convert ozone metadata into relational data"
   hadoop_add_subcommand "portmap" daemon "run a portmap service"
-  hadoop_add_subcommand "scm" daemon "run the Storage Container Manager service"
-  hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager "
   hadoop_add_subcommand "secondarynamenode" daemon "run the DFS secondary namenode"
   hadoop_add_subcommand "snapshotDiff" client "diff two snapshots of a directory or diff the current directory contents with a snapshot"
   hadoop_add_subcommand "storagepolicies" admin "list/get/set block storage policies"
   hadoop_add_subcommand "version" client "print the version"
   hadoop_add_subcommand "zkfc" daemon "run the ZK Failover Controller daemon"
-
   hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false
 }
 
@@ -97,13 +86,6 @@ function hdfscmd_case
     cacheadmin)
       HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.CacheAdmin
     ;;
-    cblock)
-      HADOOP_CLASSNAME=org.apache.hadoop.cblock.cli.CBlockCli
-    ;;
-    cblockserver)
-      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
-      HADOOP_CLASSNAME=org.apache.hadoop.cblock.CBlockManager
-    ;;
     classpath)
       hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
     ;;
@@ -157,9 +139,6 @@ function hdfscmd_case
     fetchdt)
       HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
     ;;
-    freon)
-      HADOOP_CLASSNAME=org.apache.hadoop.ozone.tools.Freon
-    ;;
     fsck)
       HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSck
     ;;
@@ -176,17 +155,9 @@ function hdfscmd_case
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
       HADOOP_CLASSNAME='org.apache.hadoop.hdfs.qjournal.server.JournalNode'
     ;;
-    jscsi)
-      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
-      HADOOP_CLASSNAME=org.apache.hadoop.cblock.jscsiHelper.SCSITargetDaemon
-    ;;
     jmxget)
       HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.JMXGet
     ;;
-    ksm)
-      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
-      HADOOP_CLASSNAME=org.apache.hadoop.ozone.ksm.KeySpaceManager
-    ;;
     lsSnapshottableDir)
       HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir
     ;;
@@ -215,28 +186,10 @@ function hdfscmd_case
     oiv_legacy)
       HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
     ;;
-    oz)
-      HADOOP_CLASSNAME=org.apache.hadoop.ozone.web.ozShell.Shell
-    ;;
-    oz_debug)
-      HADOOP_CLASSNAME=org.apache.hadoop.ozone.scm.cli.SQLCLI
-    ;;
     portmap)
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
       HADOOP_CLASSNAME=org.apache.hadoop.portmap.Portmap
     ;;
-    scmcli)
-       HADOOP_CLASSNAME=org.apache.hadoop.ozone.scm.cli.SCMCLI
-    ;;
-    scm)
-      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
-      HADOOP_CLASSNAME='org.apache.hadoop.ozone.scm.StorageContainerManager'
-      hadoop_debug "Appending HDFS_STORAGECONTAINERMANAGER_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_STORAGECONTAINERMANAGER_OPTS}"
-    ;;
-    scmcli)
-      HADOOP_CLASSNAME=org.apache.hadoop.ozone.scm.cli.SCMCLI
-    ;;
     secondarynamenode)
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
       HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh
index bde4b3a..df044fe 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh
@@ -17,10 +17,6 @@
 
 MYNAME="${BASH_SOURCE-$0}"
 
-## @description  usage info
-## @audience     private
-## @stability    evolving
-## @replaceable  no
 function hadoop_usage
 {
   hadoop_add_option "--buildpaths" "attempt to add class files from build tree"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-ozone.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-ozone.sh
deleted file mode 100644
index 097490e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-ozone.sh
+++ /dev/null
@@ -1,105 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Start hadoop hdfs and ozone daemons.
-# Run this on master node.
-## @description  usage info
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function hadoop_usage
-{
-  echo "Usage: start-ozone.sh"
-}
-
-this="${BASH_SOURCE-$0}"
-bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
-
-# let's locate libexec...
-if [[ -n "${HADOOP_HOME}" ]]; then
-  HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
-else
-  HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
-fi
-
-HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
-# shellcheck disable=SC2034
-HADOOP_NEW_CONFIG=true
-if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
-  # shellcheck disable=SC1090
-  . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
-else
-  echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
-  exit 1
-fi
-
-SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
-SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
-
-if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
-  echo "Ozone is not supported in a security enabled cluster."
-  exit 1
-fi
-
-#---------------------------------------------------------
-# Check if ozone is enabled
-OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
-if [[ "${OZONE_ENABLED}" != "true" ]]; then
-  echo "Operation is not supported because ozone is not enabled."
-  exit -1
-fi
-
-#---------------------------------------------------------
-# Start hdfs before starting ozone daemons
-if [[ -f "${bin}/start-dfs.sh" ]]; then
-  "${bin}/start-dfs.sh"
-else
-  echo "ERROR: Cannot execute ${bin}/start-dfs.sh." 2>&1
-  exit 1
-fi
-
-#---------------------------------------------------------
-# Ozone keyspacemanager nodes
-KSM_NODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -keyspacemanagers 2>/dev/null)
-echo "Starting key space manager nodes [${KSM_NODES}]"
-if [[ "${KSM_NODES}" == "0.0.0.0" ]]; then
-  KSM_NODES=$(hostname)
-fi
-
-hadoop_uservar_su hdfs ksm "${HADOOP_HDFS_HOME}/bin/hdfs" \
-  --workers \
-  --config "${HADOOP_CONF_DIR}" \
-  --hostnames "${KSM_NODES}" \
-  --daemon start \
-  ksm
-
-HADOOP_JUMBO_RETCOUNTER=$?
-
-#---------------------------------------------------------
-# Ozone storagecontainermanager nodes
-SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -storagecontainermanagers 2>/dev/null)
-echo "Starting storage container manager nodes [${SCM_NODES}]"
-hadoop_uservar_su hdfs scm "${HADOOP_HDFS_HOME}/bin/hdfs" \
-  --workers \
-  --config "${HADOOP_CONF_DIR}" \
-  --hostnames "${SCM_NODES}" \
-  --daemon start \
-  scm
-
-(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? ))
-
-exit ${HADOOP_JUMBO_RETCOUNTER}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh
index 487aee8..ec94080 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh
@@ -17,10 +17,6 @@
 
 MYNAME="${BASH_SOURCE-$0}"
 
-## @description  usage info
-## @audience     private
-## @stability    evolving
-## @replaceable  no
 function hadoop_usage
 {
   hadoop_add_option "--buildpaths" "attempt to add class files from build tree"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-ozone.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-ozone.sh
deleted file mode 100644
index bf7309d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-ozone.sh
+++ /dev/null
@@ -1,99 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Stop hdfs and ozone daemons.
-# Run this on master node.
-## @description  usage info
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function hadoop_usage
-{
-  echo "Usage: stop-ozone.sh"
-}
-
-this="${BASH_SOURCE-$0}"
-bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
-
-# let's locate libexec...
-if [[ -n "${HADOOP_HOME}" ]]; then
-  HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
-else
-  HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
-fi
-
-HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
-# shellcheck disable=SC2034
-HADOOP_NEW_CONFIG=true
-if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
-  # shellcheck disable=SC1090
-  . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
-else
-  echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
-  exit 1
-fi
-
-SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
-SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
-
-if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
-  echo "Ozone is not supported in a security enabled cluster."
-  exit 1
-fi
-
-#---------------------------------------------------------
-# Check if ozone is enabled
-OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
-if [[ "${OZONE_ENABLED}" != "true" ]]; then
-  echo "Operation is not supported because ozone is not enabled."
-  exit -1
-fi
-
-#---------------------------------------------------------
-# Start hdfs before starting ozone daemons
-if [[ -f "${bin}/stop-dfs.sh" ]]; then
-  "${bin}/stop-dfs.sh"
-else
-  echo "ERROR: Cannot execute ${bin}/stop-dfs.sh." 2>&1
-  exit 1
-fi
-
-#---------------------------------------------------------
-# Ozone keyspacemanager nodes
-KSM_NODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -keyspacemanagers 2>/dev/null)
-echo "Stopping key space manager nodes [${KSM_NODES}]"
-if [[ "${KSM_NODES}" == "0.0.0.0" ]]; then
-  KSM_NODES=$(hostname)
-fi
-
-hadoop_uservar_su hdfs ksm "${HADOOP_HDFS_HOME}/bin/hdfs" \
-  --workers \
-  --config "${HADOOP_CONF_DIR}" \
-  --hostnames "${KSM_NODES}" \
-  --daemon stop \
-  ksm
-
-#---------------------------------------------------------
-# Ozone storagecontainermanager nodes
-SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -storagecontainermanagers 2>/dev/null)
-echo "Stopping storage container manager nodes [${SCM_NODES}]"
-hadoop_uservar_su hdfs scm "${HADOOP_HDFS_HOME}/bin/hdfs" \
-  --workers \
-  --config "${HADOOP_CONF_DIR}" \
-  --hostnames "${SCM_NODES}" \
-  --daemon stop \
-  scm
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/ozone-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/ozone-site.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/ozone-site.xml
deleted file mode 100644
index 77dd7ef..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/ozone-site.xml
+++ /dev/null
@@ -1,24 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-
-</configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
deleted file mode 100644
index 62e89f0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
+++ /dev/null
@@ -1,421 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.BlockingService;
-import org.apache.hadoop.cblock.kubernetes.DynamicProvisioner;
-import org.apache.hadoop.cblock.meta.VolumeDescriptor;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.proto.CBlockClientProtocol;
-import org.apache.hadoop.cblock.proto.CBlockServiceProtocol;
-import org.apache.hadoop.cblock.proto.MountVolumeResponse;
-import org.apache.hadoop.cblock.protocol.proto
-    .CBlockClientServerProtocolProtos;
-import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
-import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB;
-import org.apache.hadoop.cblock.protocolPB
-    .CBlockClientServerProtocolServerSideTranslatorPB;
-import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
-import org.apache.hadoop.cblock.protocolPB
-    .CBlockServiceProtocolServerSideTranslatorPB;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.client.ContainerOperationClient;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.cblock.storage.StorageManager;
-import org.apache.hadoop.cblock.util.KeyUtil;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.scm.protocolPB
-    .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.utils.LevelDBStore;
-import org.iq80.leveldb.DBIterator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_CONTAINER_SIZE_GB_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SCM_IPADDRESS_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SCM_IPADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SCM_PORT_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SCM_PORT_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED_DEFAULT;
-
-
-/**
- * The main entry point of CBlock operations, ALL the CBlock operations
- * will go through this class. But NOTE that:
- *
- * volume operations (create/
- * delete/info) are:
- *    client -> CBlockManager -> StorageManager -> CBlock client
- *
- * IO operations (put/get block) are;
- *    client -> CBlock client -> container
- *
- */
-public class CBlockManager implements CBlockServiceProtocol,
-    CBlockClientProtocol {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(CBlockManager.class);
-
-  private final RPC.Server cblockService;
-  private final RPC.Server cblockServer;
-
-  private final StorageManager storageManager;
-
-  private final LevelDBStore levelDBStore;
-  private final String dbPath;
-
-  private final DynamicProvisioner kubernetesDynamicProvisioner;
-
-  private Charset encoding = Charset.forName("UTF-8");
-
-  public CBlockManager(OzoneConfiguration conf,
-      ScmClient storageClient) throws IOException {
-    // Fix the cBlockManagerId generattion code here. Should support
-    // cBlockManager --init command which will generate a cBlockManagerId and
-    // persist it locally.
-    storageManager =
-        new StorageManager(storageClient, conf, "CBLOCK");
-
-    dbPath = conf.getTrimmed(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY,
-        DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT);
-    levelDBStore = new LevelDBStore(new File(dbPath), true);
-    LOG.info("Try to load exising volume information");
-    readFromPersistentStore();
-
-    RPC.setProtocolEngine(conf, CBlockServiceProtocolPB.class,
-        ProtobufRpcEngine.class);
-    RPC.setProtocolEngine(conf, CBlockClientServerProtocolPB.class,
-        ProtobufRpcEngine.class);
-    // start service for client command-to-cblock server service
-    InetSocketAddress serviceRpcAddr =
-        OzoneClientUtils.getCblockServiceRpcAddr(conf);
-    BlockingService cblockProto =
-        CBlockServiceProtocolProtos
-            .CBlockServiceProtocolService
-            .newReflectiveBlockingService(
-                new CBlockServiceProtocolServerSideTranslatorPB(this)
-            );
-    cblockService = startRpcServer(conf, CBlockServiceProtocolPB.class,
-        cblockProto, serviceRpcAddr,
-        DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY,
-        DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY,
-        DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT);
-    InetSocketAddress cblockServiceRpcAddress =
-        OzoneClientUtils.updateRPCListenAddress(conf,
-            DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, serviceRpcAddr, cblockService);
-    LOG.info("CBlock manager listening for client commands on: {}",
-        cblockServiceRpcAddress);
-    // now start service for cblock client-to-cblock server communication
-
-    InetSocketAddress serverRpcAddr =
-        OzoneClientUtils.getCblockServerRpcAddr(conf);
-    BlockingService serverProto =
-        CBlockClientServerProtocolProtos
-            .CBlockClientServerProtocolService
-            .newReflectiveBlockingService(
-                new CBlockClientServerProtocolServerSideTranslatorPB(this)
-            );
-    cblockServer = startRpcServer(
-        conf, CBlockClientServerProtocolPB.class,
-        serverProto, serverRpcAddr,
-        DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY,
-        DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY,
-        DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT);
-    InetSocketAddress cblockServerRpcAddress =
-        OzoneClientUtils.updateRPCListenAddress(conf,
-            DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, serverRpcAddr, cblockServer);
-    LOG.info("CBlock server listening for client commands on: {}",
-        cblockServerRpcAddress);
-
-    if (conf.getBoolean(DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED,
-        DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED_DEFAULT)) {
-
-      kubernetesDynamicProvisioner =
-          new DynamicProvisioner(conf, storageManager);
-      kubernetesDynamicProvisioner.init();
-
-    } else {
-      kubernetesDynamicProvisioner = null;
-    }
-  }
-
-  public void start() {
-    cblockService.start();
-    cblockServer.start();
-    if (kubernetesDynamicProvisioner != null) {
-      kubernetesDynamicProvisioner.start();
-    }
-    LOG.info("CBlock manager started!");
-  }
-
-  public void stop() {
-    cblockService.stop();
-    cblockServer.stop();
-    if (kubernetesDynamicProvisioner != null) {
-      kubernetesDynamicProvisioner.stop();
-    }
-  }
-
-  public void join() {
-    try {
-      cblockService.join();
-      cblockServer.join();
-    } catch (InterruptedException e) {
-      LOG.error("Interrupted during join");
-      Thread.currentThread().interrupt();
-    }
-  }
-
-  /**
-   * Starts an RPC server, if configured.
-   *
-   * @param conf configuration
-   * @param protocol RPC protocol provided by RPC server
-   * @param instance RPC protocol implementation instance
-   * @param addr configured address of RPC server
-   * @param bindHostKey configuration key for setting explicit bind host.  If
-   *     the property is not configured, then the bind host is taken from addr.
-   * @param handlerCountKey configuration key for RPC server handler count
-   * @param handlerCountDefault default RPC server handler count if unconfigured
-   * @return RPC server, or null if addr is null
-   * @throws IOException if there is an I/O error while creating RPC server
-   */
-  private static RPC.Server startRpcServer(OzoneConfiguration conf,
-      Class<?> protocol, BlockingService instance,
-      InetSocketAddress addr, String bindHostKey,
-      String handlerCountKey, int handlerCountDefault) throws IOException {
-    if (addr == null) {
-      return null;
-    }
-    String bindHost = conf.getTrimmed(bindHostKey);
-    if (bindHost == null || bindHost.isEmpty()) {
-      bindHost = addr.getHostName();
-    }
-    int numHandlers = conf.getInt(handlerCountKey, handlerCountDefault);
-    RPC.Server rpcServer = new RPC.Builder(conf)
-        .setProtocol(protocol)
-        .setInstance(instance)
-        .setBindAddress(bindHost)
-        .setPort(addr.getPort())
-        .setNumHandlers(numHandlers)
-        .setVerbose(false)
-        .setSecretManager(null)
-        .build();
-    return rpcServer;
-  }
-
-  @Override
-  public synchronized MountVolumeResponse mountVolume(
-      String userName, String volumeName) throws IOException {
-    return storageManager.isVolumeValid(userName, volumeName);
-  }
-
-  @Override
-  public List<VolumeInfo> listVolumes() throws IOException {
-    return listVolume(null);
-  }
-
-  @Override
-  public synchronized void createVolume(String userName, String volumeName,
-      long volumeSize, int blockSize) throws IOException {
-    LOG.info("Create volume received: userName: {} volumeName: {} " +
-            "volumeSize: {} blockSize: {}", userName, volumeName,
-        volumeSize, blockSize);
-    // It is important to create in-memory representation of the
-    // volume first, then writes to persistent storage (levelDB)
-    // such that it is guaranteed that when there is an entry in
-    // levelDB, the volume is allocated. (more like a UNDO log fashion)
-    // TODO: what if creation failed? we allocated containers but lost
-    // the reference to the volume and all it's containers. How to release
-    // the containers?
-    storageManager.createVolume(userName, volumeName, volumeSize, blockSize);
-    VolumeDescriptor volume = storageManager.getVolume(userName, volumeName);
-    if (volume == null) {
-      throw new IOException("Volume creation failed!");
-    }
-    String volumeKey = KeyUtil.getVolumeKey(userName, volumeName);
-    writeToPersistentStore(volumeKey.getBytes(encoding),
-        volume.toProtobuf().toByteArray());
-  }
-
-  @Override
-  public synchronized void deleteVolume(String userName,
-      String volumeName, boolean force) throws IOException {
-    LOG.info("Delete volume received: volume: {} {} ", volumeName, force);
-    storageManager.deleteVolume(userName, volumeName, force);
-    // being here means volume is successfully deleted now
-    String volumeKey = KeyUtil.getVolumeKey(userName, volumeName);
-    removeFromPersistentStore(volumeKey.getBytes(encoding));
-  }
-
-  // No need to synchronize on the following three methods, since write and
-  // remove's caller are synchronized. read's caller is the constructor and
-  // no other method call can happen at that time.
-  @VisibleForTesting
-  public void writeToPersistentStore(byte[] key, byte[] value) {
-    levelDBStore.put(key, value);
-  }
-
-  @VisibleForTesting
-  public void removeFromPersistentStore(byte[] key) {
-    levelDBStore.delete(key);
-  }
-
-  public void readFromPersistentStore() throws IOException {
-    try (DBIterator iter = levelDBStore.getIterator()) {
-      iter.seekToFirst();
-      while (iter.hasNext()) {
-        Map.Entry<byte[], byte[]> entry = iter.next();
-        String volumeKey = new String(entry.getKey(), encoding);
-        try {
-          VolumeDescriptor volumeDescriptor =
-              VolumeDescriptor.fromProtobuf(entry.getValue());
-          storageManager.addVolume(volumeDescriptor);
-        } catch (IOException e) {
-          LOG.error("Loading volume " + volumeKey + " error " + e);
-        }
-      }
-    }
-  }
-
-  @Override
-  public synchronized VolumeInfo infoVolume(String userName, String volumeName
-  ) throws IOException {
-    LOG.info("Info volume received: volume: {}", volumeName);
-    return storageManager.infoVolume(userName, volumeName);
-  }
-
-  @VisibleForTesting
-  public synchronized List<VolumeDescriptor> getAllVolumes() {
-    return storageManager.getAllVolume(null);
-  }
-
-  public synchronized List<VolumeDescriptor> getAllVolumes(String userName) {
-    return storageManager.getAllVolume(userName);
-  }
-
-  public synchronized void close() {
-    try {
-      levelDBStore.close();
-    } catch (IOException e) {
-      LOG.error("Error when closing levelDB " + e);
-    }
-  }
-
-  public synchronized void clean() {
-    try {
-      levelDBStore.close();
-      levelDBStore.destroy();
-    } catch (IOException e) {
-      LOG.error("Error when deleting levelDB " + e);
-    }
-  }
-
-  @Override
-  public synchronized List<VolumeInfo> listVolume(String userName)
-      throws IOException {
-    ArrayList<VolumeInfo> response = new ArrayList<>();
-    List<VolumeDescriptor> allVolumes =
-        storageManager.getAllVolume(userName);
-    for (VolumeDescriptor volume : allVolumes) {
-      VolumeInfo info =
-          new VolumeInfo(volume.getUserName(), volume.getVolumeName(),
-              volume.getVolumeSize(), volume.getBlockSize());
-      response.add(info);
-    }
-    return response;
-  }
-
-  public static void main(String[] args) throws Exception {
-    long version = RPC.getProtocolVersion(
-        StorageContainerLocationProtocolPB.class);
-    OzoneConfiguration ozoneConf = new OzoneConfiguration();
-    String scmAddress = ozoneConf.get(DFS_CBLOCK_SCM_IPADDRESS_KEY,
-        DFS_CBLOCK_SCM_IPADDRESS_DEFAULT);
-    int scmPort = ozoneConf.getInt(DFS_CBLOCK_SCM_PORT_KEY,
-        DFS_CBLOCK_SCM_PORT_DEFAULT);
-    int containerSizeGB = ozoneConf.getInt(DFS_CBLOCK_CONTAINER_SIZE_GB_KEY,
-        DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT);
-    ContainerOperationClient.setContainerSizeB(containerSizeGB* OzoneConsts.GB);
-    InetSocketAddress address = new InetSocketAddress(scmAddress, scmPort);
-
-    ozoneConf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
-        OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
-    LOG.info(
-        "Creating StorageContainerLocationProtocol RPC client with address {}",
-        address);
-    RPC.setProtocolEngine(ozoneConf, StorageContainerLocationProtocolPB.class,
-        ProtobufRpcEngine.class);
-    StorageContainerLocationProtocolClientSideTranslatorPB client =
-        new StorageContainerLocationProtocolClientSideTranslatorPB(
-            RPC.getProxy(StorageContainerLocationProtocolPB.class, version,
-                address, UserGroupInformation.getCurrentUser(), ozoneConf,
-                NetUtils.getDefaultSocketFactory(ozoneConf),
-                Client.getRpcTimeout(ozoneConf)));
-    ScmClient storageClient = new ContainerOperationClient(
-        client, new XceiverClientManager(ozoneConf));
-    CBlockManager cbm = new CBlockManager(ozoneConf, storageClient);
-    cbm.start();
-    cbm.join();
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[32/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
deleted file mode 100644
index 23ba00b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.annotation.Metrics;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-import org.apache.hadoop.metrics2.lib.MutableRate;
-import org.apache.hadoop.metrics2.lib.MetricsRegistry;
-import org.apache.hadoop.metrics2.lib.MutableQuantiles;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-
-/**
- *
- * This class is for maintaining  the various Storage Container
- * DataNode statistics and publishing them through the metrics interfaces.
- * This also registers the JMX MBean for RPC.
- * <p>
- * This class has a number of metrics variables that are publicly accessible;
- * these variables (objects) have methods to update their values;
- *  for example:
- *  <p> {@link #numOps}.inc()
- *
- */
-@InterfaceAudience.Private
-@Metrics(about="Storage Container DataNode Metrics", context="dfs")
-public class ContainerMetrics {
-  @Metric private MutableCounterLong numOps;
-  private MutableCounterLong[] numOpsArray;
-  private MutableCounterLong[] opsBytesArray;
-  private MutableRate[] opsLatency;
-  private MutableQuantiles[][] opsLatQuantiles;
-  private MetricsRegistry registry = null;
-
-  public ContainerMetrics(int[] intervals) {
-    int numEnumEntries = ContainerProtos.Type.values().length;
-    final int len = intervals.length;
-    this.numOpsArray = new MutableCounterLong[numEnumEntries];
-    this.opsBytesArray = new MutableCounterLong[numEnumEntries];
-    this.opsLatency = new MutableRate[numEnumEntries];
-    this.opsLatQuantiles = new MutableQuantiles[numEnumEntries][len];
-    this.registry = new MetricsRegistry("StorageContainerMetrics");
-    for (int i = 0; i < numEnumEntries; i++) {
-      numOpsArray[i] = registry.newCounter(
-          "num" + ContainerProtos.Type.valueOf(i + 1),
-          "number of " + ContainerProtos.Type.valueOf(i + 1) + " ops",
-          (long) 0);
-      opsBytesArray[i] = registry.newCounter(
-          "bytes" + ContainerProtos.Type.valueOf(i + 1),
-          "bytes used by " + ContainerProtos.Type.valueOf(i + 1) + "op",
-          (long) 0);
-      opsLatency[i] = registry.newRate(
-          "latency" + ContainerProtos.Type.valueOf(i + 1),
-          ContainerProtos.Type.valueOf(i + 1) + " op");
-
-      for (int j = 0; j < len; j++) {
-        int interval = intervals[j];
-        String quantileName = ContainerProtos.Type.valueOf(i + 1) + "Nanos"
-            + interval + "s";
-        opsLatQuantiles[i][j] = registry.newQuantiles(quantileName,
-            "latency of Container ops", "ops", "latency", interval);
-      }
-    }
-  }
-
-  public static ContainerMetrics create(Configuration conf) {
-    MetricsSystem ms = DefaultMetricsSystem.instance();
-    // Percentile measurement is off by default, by watching no intervals
-    int[] intervals =
-             conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
-    return ms.register("StorageContainerMetrics",
-                       "Storage Container Node Metrics",
-                       new ContainerMetrics(intervals));
-  }
-
-  public void incContainerOpcMetrics(ContainerProtos.Type type){
-    numOps.incr();
-    numOpsArray[type.ordinal()].incr();
-  }
-
-  public long getContainerOpsMetrics(ContainerProtos.Type type){
-    return numOpsArray[type.ordinal()].value();
-  }
-
-  public void incContainerOpsLatencies(ContainerProtos.Type type,
-                                       long latencyNanos) {
-    opsLatency[type.ordinal()].add(latencyNanos);
-    for (MutableQuantiles q: opsLatQuantiles[type.ordinal()]) {
-      q.add(latencyNanos);
-    }
-  }
-
-  public void incContainerBytesStats(ContainerProtos.Type type, long bytes) {
-    opsBytesArray[type.ordinal()].incr(bytes);
-  }
-
-  public long getContainerBytesMetrics(ContainerProtos.Type type){
-    return opsBytesArray[type.ordinal()].value();
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
deleted file mode 100644
index 81c41bb..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerInfo;
-
-/**
- * Container Report iterates the closed containers and sends a container report
- * to SCM.
- */
-public class ContainerReport {
-  private static final int UNKNOWN = -1;
-  private final String containerName;
-  private final String finalhash;
-  private long size;
-  private long keyCount;
-  private long bytesUsed;
-  private long readCount;
-  private long writeCount;
-  private long readBytes;
-  private long writeBytes;
-  private long containerID;
-
-  public long getContainerID() {
-    return containerID;
-  }
-
-  public void setContainerID(long containerID) {
-    this.containerID = containerID;
-  }
-
-
-
-
-  /**
-   * Constructs the ContainerReport.
-   *
-   * @param containerName - Container Name.
-   * @param finalhash - Final Hash.
-   */
-  public ContainerReport(String containerName, String finalhash) {
-    this.containerName = containerName;
-    this.finalhash = finalhash;
-    this.size = UNKNOWN;
-    this.keyCount = UNKNOWN;
-    this.bytesUsed = 0L;
-    this.readCount = 0L;
-    this.readBytes = 0L;
-    this.writeCount = 0L;
-    this.writeBytes = 0L;
-  }
-
-  /**
-   * Gets a containerReport from protobuf class.
-   *
-   * @param info - ContainerInfo.
-   * @return - ContainerReport.
-   */
-  public static ContainerReport getFromProtoBuf(ContainerInfo info) {
-    Preconditions.checkNotNull(info);
-    ContainerReport report = new ContainerReport(info.getContainerName(),
-        info.getFinalhash());
-    if (info.hasSize()) {
-      report.setSize(info.getSize());
-    }
-    if (info.hasKeyCount()) {
-      report.setKeyCount(info.getKeyCount());
-    }
-    if (info.hasUsed()) {
-      report.setBytesUsed(info.getUsed());
-    }
-    if (info.hasReadCount()) {
-      report.setReadCount(info.getReadCount());
-    }
-    if (info.hasReadBytes()) {
-      report.setReadBytes(info.getReadBytes());
-    }
-    if (info.hasWriteCount()) {
-      report.setWriteCount(info.getWriteCount());
-    }
-    if (info.hasWriteBytes()) {
-      report.setWriteBytes(info.getWriteBytes());
-    }
-
-    report.setContainerID(info.getContainerID());
-    return report;
-  }
-
-  /**
-   * Gets the container name.
-   *
-   * @return - Name
-   */
-  public String getContainerName() {
-    return containerName;
-  }
-
-  /**
-   * Returns the final signature for this container.
-   *
-   * @return - hash
-   */
-  public String getFinalhash() {
-    return finalhash;
-  }
-
-  /**
-   * Returns a positive number it is a valid number, -1 if not known.
-   *
-   * @return size or -1
-   */
-  public long getSize() {
-    return size;
-  }
-
-  /**
-   * Sets the size of the container on disk.
-   *
-   * @param size - int
-   */
-  public void setSize(long size) {
-    this.size = size;
-  }
-
-  /**
-   * Gets number of keys in the container if known.
-   *
-   * @return - Number of keys or -1 for not known.
-   */
-  public long getKeyCount() {
-    return keyCount;
-  }
-
-  /**
-   * Sets the key count.
-   *
-   * @param keyCount - Key Count
-   */
-  public void setKeyCount(long keyCount) {
-    this.keyCount = keyCount;
-  }
-
-  public long getReadCount() {
-    return readCount;
-  }
-
-  public void setReadCount(long readCount) {
-    this.readCount = readCount;
-  }
-
-  public long getWriteCount() {
-    return writeCount;
-  }
-
-  public void setWriteCount(long writeCount) {
-    this.writeCount = writeCount;
-  }
-
-  public long getReadBytes() {
-    return readBytes;
-  }
-
-  public void setReadBytes(long readBytes) {
-    this.readBytes = readBytes;
-  }
-
-  public long getWriteBytes() {
-    return writeBytes;
-  }
-
-  public void setWriteBytes(long writeBytes) {
-    this.writeBytes = writeBytes;
-  }
-
-  public long getBytesUsed() {
-    return bytesUsed;
-  }
-
-  public void setBytesUsed(long bytesUsed) {
-    this.bytesUsed = bytesUsed;
-  }
-
-  /**
-   * Gets a containerInfo protobuf message from ContainerReports.
-   *
-   * @return ContainerInfo
-   */
-  public ContainerInfo getProtoBufMessage() {
-    return ContainerInfo.newBuilder()
-        .setContainerName(this.getContainerName())
-        .setKeyCount(this.getKeyCount())
-        .setSize(this.getSize())
-        .setUsed(this.getBytesUsed())
-        .setReadCount(this.getReadCount())
-        .setReadBytes(this.getReadBytes())
-        .setWriteCount(this.getWriteCount())
-        .setWriteBytes(this.getWriteBytes())
-        .setFinalhash(this.getFinalhash())
-        .setContainerID(this.getContainerID())
-        .build();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
deleted file mode 100644
index f78e646..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
+++ /dev/null
@@ -1,477 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.impl.ContainerManagerImpl;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.apache.commons.io.FilenameUtils.removeExtension;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.INVALID_ARGUMENT;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.UNABLE_TO_FIND_DATA_DIR;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_EXTENSION;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_META;
-
-/**
- * A set of helper functions to create proper responses.
- */
-public final class ContainerUtils {
-
-  private ContainerUtils() {
-    //never constructed.
-  }
-
-  /**
-   * Returns a CreateContainer Response. This call is used by create and delete
-   * containers which have null success responses.
-   *
-   * @param msg Request
-   * @return Response.
-   */
-  public static ContainerProtos.ContainerCommandResponseProto
-      getContainerResponse(ContainerProtos.ContainerCommandRequestProto msg) {
-    ContainerProtos.ContainerCommandResponseProto.Builder builder =
-        getContainerResponse(msg, ContainerProtos.Result.SUCCESS, "");
-    return builder.build();
-  }
-
-  /**
-   * Returns a ReadContainer Response.
-   *
-   * @param msg Request
-   * @param containerData - data
-   * @return Response.
-   */
-  public static ContainerProtos.ContainerCommandResponseProto
-      getReadContainerResponse(ContainerProtos.ContainerCommandRequestProto msg,
-      ContainerData containerData) {
-    Preconditions.checkNotNull(containerData);
-
-    ContainerProtos.ReadContainerResponseProto.Builder response =
-        ContainerProtos.ReadContainerResponseProto.newBuilder();
-    response.setContainerData(containerData.getProtoBufMessage());
-
-    ContainerProtos.ContainerCommandResponseProto.Builder builder =
-        getContainerResponse(msg, ContainerProtos.Result.SUCCESS, "");
-    builder.setReadContainer(response);
-    return builder.build();
-  }
-
-  /**
-   * We found a command type but no associated payload for the command. Hence
-   * return malformed Command as response.
-   *
-   * @param msg - Protobuf message.
-   * @param result - result
-   * @param message - Error message.
-   * @return ContainerCommandResponseProto - MALFORMED_REQUEST.
-   */
-  public static ContainerProtos.ContainerCommandResponseProto.Builder
-      getContainerResponse(ContainerProtos.ContainerCommandRequestProto msg,
-      ContainerProtos.Result result, String message) {
-    return
-        ContainerProtos.ContainerCommandResponseProto.newBuilder()
-            .setCmdType(msg.getCmdType())
-            .setTraceID(msg.getTraceID())
-            .setResult(result)
-            .setMessage(message);
-  }
-
-  /**
-   * Logs the error and returns a response to the caller.
-   *
-   * @param log - Logger
-   * @param ex - Exception
-   * @param msg - Request Object
-   * @return Response
-   */
-  public static ContainerProtos.ContainerCommandResponseProto logAndReturnError(
-      Logger log, StorageContainerException ex,
-      ContainerProtos.ContainerCommandRequestProto msg) {
-    log.info("Operation: {} : Trace ID: {} : Message: {} : Result: {}",
-        msg.getCmdType().name(), msg.getTraceID(),
-        ex.getMessage(), ex.getResult().getValueDescriptor().getName());
-    return getContainerResponse(msg, ex.getResult(), ex.getMessage()).build();
-  }
-
-  /**
-   * Logs the error and returns a response to the caller.
-   *
-   * @param log - Logger
-   * @param ex - Exception
-   * @param msg - Request Object
-   * @return Response
-   */
-  public static ContainerProtos.ContainerCommandResponseProto logAndReturnError(
-      Logger log, RuntimeException ex,
-      ContainerProtos.ContainerCommandRequestProto msg) {
-    log.info("Operation: {} : Trace ID: {} : Message: {} ",
-        msg.getCmdType().name(), msg.getTraceID(), ex.getMessage());
-    return getContainerResponse(msg, INVALID_ARGUMENT, ex.getMessage()).build();
-  }
-
-  /**
-   * We found a command type but no associated payload for the command. Hence
-   * return malformed Command as response.
-   *
-   * @param msg - Protobuf message.
-   * @return ContainerCommandResponseProto - MALFORMED_REQUEST.
-   */
-  public static ContainerProtos.ContainerCommandResponseProto
-      malformedRequest(ContainerProtos.ContainerCommandRequestProto msg) {
-    return getContainerResponse(msg, ContainerProtos.Result.MALFORMED_REQUEST,
-        "Cmd type does not match the payload.").build();
-  }
-
-  /**
-   * We found a command type that is not supported yet.
-   *
-   * @param msg - Protobuf message.
-   * @return ContainerCommandResponseProto - MALFORMED_REQUEST.
-   */
-  public static ContainerProtos.ContainerCommandResponseProto
-      unsupportedRequest(ContainerProtos.ContainerCommandRequestProto msg) {
-    return getContainerResponse(msg, ContainerProtos.Result.UNSUPPORTED_REQUEST,
-        "Server does not support this command yet.").build();
-  }
-
-  /**
-   * get containerName from a container file.
-   *
-   * @param containerFile - File
-   * @return Name of the container.
-   */
-  public static String getContainerNameFromFile(File containerFile) {
-    Preconditions.checkNotNull(containerFile);
-    return Paths.get(containerFile.getParent()).resolve(
-        removeExtension(containerFile.getName())).toString();
-  }
-
-  /**
-   * Verifies that this in indeed a new container.
-   *
-   * @param containerFile - Container File to verify
-   * @param metadataFile - metadata File to verify
-   * @throws IOException
-   */
-  public static void verifyIsNewContainer(File containerFile, File metadataFile)
-      throws IOException {
-    Logger log = LoggerFactory.getLogger(ContainerManagerImpl.class);
-    if (containerFile.exists()) {
-      log.error("container already exists on disk. File: {}",
-          containerFile.toPath());
-      throw new FileAlreadyExistsException("container already exists on " +
-          "disk.");
-    }
-
-    if (metadataFile.exists()) {
-      log.error("metadata found on disk, but missing container. Refusing to" +
-          " write this container. File: {} ", metadataFile.toPath());
-      throw new FileAlreadyExistsException(("metadata found on disk, but " +
-          "missing container. Refusing to write this container."));
-    }
-
-    File parentPath = new File(containerFile.getParent());
-
-    if (!parentPath.exists() && !parentPath.mkdirs()) {
-      log.error("Unable to create parent path. Path: {}",
-          parentPath.toString());
-      throw new IOException("Unable to create container directory.");
-    }
-
-    if (!containerFile.createNewFile()) {
-      log.error("creation of a new container file failed. File: {}",
-          containerFile.toPath());
-      throw new IOException("creation of a new container file failed.");
-    }
-
-    if (!metadataFile.createNewFile()) {
-      log.error("creation of the metadata file failed. File: {}",
-          metadataFile.toPath());
-      throw new IOException("creation of a new container file failed.");
-    }
-  }
-
-  public static String getContainerDbFileName(String containerName) {
-    return containerName + OzoneConsts.DN_CONTAINER_DB;
-  }
-
-  /**
-   * creates a Metadata DB for the specified container.
-   *
-   * @param containerPath - Container Path.
-   * @throws IOException
-   */
-  public static Path createMetadata(Path containerPath, String containerName,
-      Configuration conf)
-      throws IOException {
-    Logger log = LoggerFactory.getLogger(ContainerManagerImpl.class);
-    Preconditions.checkNotNull(containerPath);
-    Path metadataPath = containerPath.resolve(OzoneConsts.CONTAINER_META_PATH);
-    if (!metadataPath.toFile().mkdirs()) {
-      log.error("Unable to create directory for metadata storage. Path: {}",
-          metadataPath);
-      throw new IOException("Unable to create directory for metadata storage." +
-          " Path: " + metadataPath);
-    }
-    MetadataStore store = MetadataStoreBuilder.newBuilder()
-        .setConf(conf)
-        .setCreateIfMissing(true)
-        .setDbFile(metadataPath
-            .resolve(getContainerDbFileName(containerName)).toFile())
-        .build();
-
-    // we close since the SCM pre-creates containers.
-    // we will open and put Db handle into a cache when keys are being created
-    // in a container.
-
-    store.close();
-
-    Path dataPath = containerPath.resolve(OzoneConsts.CONTAINER_DATA_PATH);
-    if (!dataPath.toFile().mkdirs()) {
-
-      // If we failed to create data directory, we cleanup the
-      // metadata directory completely. That is, we will delete the
-      // whole directory including LevelDB file.
-      log.error("Unable to create directory for data storage. cleaning up the" +
-              " container path: {} dataPath: {}",
-          containerPath, dataPath);
-      FileUtils.deleteDirectory(containerPath.toFile());
-      throw new IOException("Unable to create directory for data storage." +
-          " Path: " + dataPath);
-    }
-    return metadataPath;
-  }
-
-  /**
-   * Returns Metadata location.
-   *
-   * @param containerData - Data
-   * @param location - Path
-   * @return Path
-   */
-  public static File getMetadataFile(ContainerData containerData,
-      Path location) {
-    return location.resolve(containerData
-        .getContainerName().concat(CONTAINER_META))
-        .toFile();
-  }
-
-  /**
-   * Returns container file location.
-   *
-   * @param containerData - Data
-   * @param location - Root path
-   * @return Path
-   */
-  public static File getContainerFile(ContainerData containerData,
-      Path location) {
-    return location.resolve(containerData
-        .getContainerName().concat(CONTAINER_EXTENSION))
-        .toFile();
-  }
-
-  /**
-   * Container metadata directory -- here is where the level DB lives.
-   *
-   * @param cData - cData.
-   * @return Path to the parent directory where the DB lives.
-   */
-  public static Path getMetadataDirectory(ContainerData cData) {
-    Path dbPath = Paths.get(cData.getDBPath());
-    Preconditions.checkNotNull(dbPath);
-    Preconditions.checkState(dbPath.toString().length() > 0);
-    return dbPath.getParent();
-  }
-
-  /**
-   * Returns the path where data or chunks live for a given container.
-   *
-   * @param cData - cData container
-   * @return - Path
-   * @throws StorageContainerException
-   */
-  public static Path getDataDirectory(ContainerData cData)
-      throws StorageContainerException {
-    Path path = getMetadataDirectory(cData);
-    Preconditions.checkNotNull(path);
-    Path parentPath = path.getParent();
-    if (parentPath == null) {
-      throw new StorageContainerException("Unable to get Data directory."
-          + path, UNABLE_TO_FIND_DATA_DIR);
-    }
-    return parentPath.resolve(OzoneConsts.CONTAINER_DATA_PATH);
-  }
-
-  /**
-   * remove Container if it is empty.
-   * <p/>
-   * There are three things we need to delete.
-   * <p/>
-   * 1. Container file and metadata file. 2. The Level DB file 3. The path that
-   * we created on the data location.
-   *
-   * @param containerData - Data of the container to remove.
-   * @param conf - configuration of the cluster.
-   * @param forceDelete - whether this container should be deleted forcibly.
-   * @throws IOException
-   */
-  public static void removeContainer(ContainerData containerData,
-      Configuration conf, boolean forceDelete) throws IOException {
-    Preconditions.checkNotNull(containerData);
-    Path dbPath = Paths.get(containerData.getDBPath());
-
-    MetadataStore db = KeyUtils.getDB(containerData, conf);
-    // If the container is not empty and cannot be deleted forcibly,
-    // then throw a SCE to stop deleting.
-    if(!forceDelete && !db.isEmpty()) {
-      throw new StorageContainerException(
-          "Container cannot be deleted because it is not empty.",
-          ContainerProtos.Result.ERROR_CONTAINER_NOT_EMPTY);
-    }
-    // Close the DB connection and remove the DB handler from cache
-    KeyUtils.removeDB(containerData, conf);
-
-    // Delete the DB File.
-    FileUtils.forceDelete(dbPath.toFile());
-    dbPath = dbPath.getParent();
-
-    // Delete all Metadata in the Data directories for this containers.
-    if (dbPath != null) {
-      FileUtils.deleteDirectory(dbPath.toFile());
-      dbPath = dbPath.getParent();
-    }
-
-    // now delete the container directory, this means that all key data dirs
-    // will be removed too.
-    if (dbPath != null) {
-      FileUtils.deleteDirectory(dbPath.toFile());
-    }
-
-    // Delete the container metadata from the metadata locations.
-    String rootPath = getContainerNameFromFile(new File(containerData
-        .getContainerPath()));
-    Path containerPath = Paths.get(rootPath.concat(CONTAINER_EXTENSION));
-    Path metaPath = Paths.get(rootPath.concat(CONTAINER_META));
-
-    FileUtils.forceDelete(containerPath.toFile());
-    FileUtils.forceDelete(metaPath.toFile());
-  }
-
-  /**
-   * Write datanode ID protobuf messages to an ID file.
-   * The old ID file will be overwritten.
-   *
-   * @param ids A set of {@link DatanodeID}
-   * @param path Local ID file path
-   * @throws IOException When read/write error occurs
-   */
-  private synchronized static void writeDatanodeIDs(List<DatanodeID> ids,
-      File path) throws IOException {
-    if (path.exists()) {
-      if (!path.delete() || !path.createNewFile()) {
-        throw new IOException("Unable to overwrite the datanode ID file.");
-      }
-    } else {
-      if(!path.getParentFile().exists() &&
-          !path.getParentFile().mkdirs()) {
-        throw new IOException("Unable to create datanode ID directories.");
-      }
-    }
-    try (FileOutputStream out = new FileOutputStream(path)) {
-      for (DatanodeID id : ids) {
-        HdfsProtos.DatanodeIDProto dnId = id.getProtoBufMessage();
-        dnId.writeDelimitedTo(out);
-      }
-    }
-  }
-
-  /**
-   * Persistent a {@link DatanodeID} to a local file.
-   * It reads the IDs first and append a new entry only if the ID is new.
-   * This is to avoid on some dirty environment, this file gets too big.
-   *
-   * @throws IOException when read/write error occurs
-   */
-  public synchronized static void writeDatanodeIDTo(DatanodeID dnID,
-      File path) throws IOException {
-    List<DatanodeID> ids = ContainerUtils.readDatanodeIDsFrom(path);
-    // Only create or overwrite the file
-    // if the ID doesn't exist in the ID file
-    for (DatanodeID id : ids) {
-      if (id.getProtoBufMessage()
-          .equals(dnID.getProtoBufMessage())) {
-        return;
-      }
-    }
-    ids.add(dnID);
-    writeDatanodeIDs(ids, path);
-  }
-
-  /**
-   * Read {@link DatanodeID} from a local ID file and return a set of
-   * datanode IDs. If the ID file doesn't exist, an empty set is returned.
-   *
-   * @param path ID file local path
-   * @return A set of {@link DatanodeID}
-   * @throws IOException If the id file is malformed or other I/O exceptions
-   */
-  public synchronized static List<DatanodeID> readDatanodeIDsFrom(File path)
-      throws IOException {
-    List<DatanodeID> ids = new ArrayList<DatanodeID>();
-    if (!path.exists()) {
-      return ids;
-    }
-    try(FileInputStream in = new FileInputStream(path)) {
-      while(in.available() > 0) {
-        try {
-          HdfsProtos.DatanodeIDProto id =
-              HdfsProtos.DatanodeIDProto.parseDelimitedFrom(in);
-          ids.add(DatanodeID.getFromProtoBuf(id));
-        } catch (IOException e) {
-          throw new IOException("Failed to parse Datanode ID from "
-              + path.getAbsolutePath(), e);
-        }
-      }
-    }
-    return ids;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DeletedContainerBlocksSummary.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DeletedContainerBlocksSummary.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DeletedContainerBlocksSummary.java
deleted file mode 100644
index de5e2d0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DeletedContainerBlocksSummary.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import com.google.common.collect.Maps;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-/**
- * A helper class to wrap the info about under deletion container blocks.
- */
-public final class DeletedContainerBlocksSummary {
-
-  private final List<DeletedBlocksTransaction> blocks;
-  // key : txID
-  // value : times of this tx has been processed
-  private final Map<Long, Integer> txSummary;
-  // key : container name
-  // value : the number of blocks need to be deleted in this container
-  // if the message contains multiple entries for same block,
-  // blocks will be merged
-  private final Map<String, Integer> blockSummary;
-  // total number of blocks in this message
-  private int numOfBlocks;
-
-  private DeletedContainerBlocksSummary(List<DeletedBlocksTransaction> blocks) {
-    this.blocks = blocks;
-    txSummary = Maps.newHashMap();
-    blockSummary = Maps.newHashMap();
-    blocks.forEach(entry -> {
-      txSummary.put(entry.getTxID(), entry.getCount());
-      if (blockSummary.containsKey(entry.getContainerName())) {
-        blockSummary.put(entry.getContainerName(),
-            blockSummary.get(entry.getContainerName())
-                + entry.getBlockIDCount());
-      } else {
-        blockSummary.put(entry.getContainerName(), entry.getBlockIDCount());
-      }
-      numOfBlocks += entry.getBlockIDCount();
-    });
-  }
-
-  public static DeletedContainerBlocksSummary getFrom(
-      List<DeletedBlocksTransaction> blocks) {
-    return new DeletedContainerBlocksSummary(blocks);
-  }
-
-  public int getNumOfBlocks() {
-    return numOfBlocks;
-  }
-
-  public int getNumOfContainers() {
-    return blockSummary.size();
-  }
-
-  public String getTXIDs() {
-    return String.join(",", txSummary.keySet()
-        .stream().map(String::valueOf).collect(Collectors.toList()));
-  }
-
-  public String getTxIDSummary() {
-    List<String> txSummaryEntry = txSummary.entrySet().stream()
-        .map(entry -> entry.getKey() + "(" + entry.getValue() + ")")
-        .collect(Collectors.toList());
-    return "[" + String.join(",", txSummaryEntry) + "]";
-  }
-
-  @Override public String toString() {
-    StringBuffer sb = new StringBuffer();
-    for (DeletedBlocksTransaction blks : blocks) {
-      sb.append(" ")
-          .append("TXID=")
-          .append(blks.getTxID())
-          .append(", ")
-          .append("TimesProceed=")
-          .append(blks.getCount())
-          .append(", ")
-          .append(blks.getContainerName())
-          .append(" : [")
-          .append(String.join(",", blks.getBlockIDList())).append("]")
-          .append("\n");
-    }
-    return sb.toString();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/FileUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/FileUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/FileUtils.java
deleted file mode 100644
index 27731c7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/FileUtils.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import com.google.common.base.Preconditions;
-import com.google.protobuf.ByteString;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-
-/**
- * File Utils are helper routines used by putSmallFile and getSmallFile
- * RPCs.
- */
-public final class FileUtils {
-  /**
-   * Never Constructed.
-   */
-  private FileUtils() {
-  }
-
-  /**
-   * Gets a response for the putSmallFile RPC.
-   * @param msg - ContainerCommandRequestProto
-   * @return - ContainerCommandResponseProto
-   */
-  public static ContainerProtos.ContainerCommandResponseProto
-      getPutFileResponse(ContainerProtos.ContainerCommandRequestProto msg) {
-    ContainerProtos.PutSmallFileResponseProto.Builder getResponse =
-        ContainerProtos.PutSmallFileResponseProto.newBuilder();
-    ContainerProtos.ContainerCommandResponseProto.Builder builder =
-        ContainerUtils.getContainerResponse(msg, ContainerProtos.Result
-            .SUCCESS, "");
-    builder.setCmdType(ContainerProtos.Type.PutSmallFile);
-    builder.setPutSmallFile(getResponse);
-    return  builder.build();
-  }
-
-  /**
-   * Gets a response to the read small file call.
-   * @param msg - Msg
-   * @param data  - Data
-   * @param info  - Info
-   * @return    Response.
-   */
-  public static ContainerProtos.ContainerCommandResponseProto
-      getGetSmallFileResponse(ContainerProtos.ContainerCommandRequestProto msg,
-      byte[] data, ChunkInfo info) {
-    Preconditions.checkNotNull(msg);
-
-    ContainerProtos.ReadChunkResponseProto.Builder readChunkresponse =
-        ContainerProtos.ReadChunkResponseProto.newBuilder();
-    readChunkresponse.setChunkData(info.getProtoBufMessage());
-    readChunkresponse.setData(ByteString.copyFrom(data));
-    readChunkresponse.setPipeline(msg.getGetSmallFile().getKey().getPipeline());
-
-    ContainerProtos.GetSmallFileResponseProto.Builder getSmallFile =
-        ContainerProtos.GetSmallFileResponseProto.newBuilder();
-    getSmallFile.setData(readChunkresponse.build());
-    ContainerProtos.ContainerCommandResponseProto.Builder builder =
-        ContainerUtils.getContainerResponse(msg, ContainerProtos.Result
-            .SUCCESS, "");
-    builder.setCmdType(ContainerProtos.Type.GetSmallFile);
-    builder.setGetSmallFile(getSmallFile);
-    return builder.build();
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java
deleted file mode 100644
index 639de16..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-/**
- * Helper class to convert Protobuf to Java classes.
- */
-public class KeyData {
-  private final String containerName;
-  private final String keyName;
-  private final Map<String, String> metadata;
-
-  /**
-   * Please note : when we are working with keys, we don't care what they point
-   * to. So we We don't read chunkinfo nor validate them. It is responsibility
-   * of higher layer like ozone. We just read and write data from network.
-   */
-  private List<ContainerProtos.ChunkInfo> chunks;
-
-  /**
-   * Constructs a KeyData Object.
-   *
-   * @param containerName
-   * @param keyName
-   */
-  public KeyData(String containerName, String keyName) {
-    this.containerName = containerName;
-    this.keyName = keyName;
-    this.metadata = new TreeMap<>();
-  }
-
-  /**
-   * Returns a keyData object from the protobuf data.
-   *
-   * @param data - Protobuf data.
-   * @return - KeyData
-   * @throws IOException
-   */
-  public static KeyData getFromProtoBuf(ContainerProtos.KeyData data) throws
-      IOException {
-    KeyData keyData = new KeyData(data.getContainerName(), data.getName());
-    for (int x = 0; x < data.getMetadataCount(); x++) {
-      keyData.addMetadata(data.getMetadata(x).getKey(),
-          data.getMetadata(x).getValue());
-    }
-    keyData.setChunks(data.getChunksList());
-    return keyData;
-  }
-
-  /**
-   * Returns a Protobuf message from KeyData.
-   * @return Proto Buf Message.
-   */
-  public ContainerProtos.KeyData getProtoBufMessage() {
-    ContainerProtos.KeyData.Builder builder =
-        ContainerProtos.KeyData.newBuilder();
-    builder.setContainerName(this.containerName);
-    builder.setName(this.getKeyName());
-    builder.addAllChunks(this.chunks);
-    for (Map.Entry<String, String> entry : metadata.entrySet()) {
-      OzoneProtos.KeyValue.Builder keyValBuilder =
-          OzoneProtos.KeyValue.newBuilder();
-      builder.addMetadata(keyValBuilder.setKey(entry.getKey())
-          .setValue(entry.getValue()).build());
-    }
-    return builder.build();
-  }
-
-  /**
-   * Adds metadata.
-   *
-   * @param key   - Key
-   * @param value - Value
-   * @throws IOException
-   */
-  public synchronized void addMetadata(String key, String value) throws
-      IOException {
-    if (this.metadata.containsKey(key)) {
-      throw new IOException("This key already exists. Key " + key);
-    }
-    metadata.put(key, value);
-  }
-
-  public synchronized Map<String, String> getMetadata() {
-    return Collections.unmodifiableMap(this.metadata);
-  }
-
-  /**
-   * Returns value of a key.
-   */
-  public synchronized String getValue(String key) {
-    return metadata.get(key);
-  }
-
-  /**
-   * Deletes a metadata entry from the map.
-   *
-   * @param key - Key
-   */
-  public synchronized void deleteKey(String key) {
-    metadata.remove(key);
-  }
-
-  /**
-   * Returns chunks list.
-   *
-   * @return list of chunkinfo.
-   */
-  public List<ContainerProtos.ChunkInfo> getChunks() {
-    return chunks;
-  }
-
-  /**
-   * Returns container Name.
-   * @return String.
-   */
-  public String getContainerName() {
-    return containerName;
-  }
-
-  /**
-   * Returns KeyName.
-   * @return String.
-   */
-  public String getKeyName() {
-    return keyName;
-  }
-
-  /**
-   * Sets Chunk list.
-   *
-   * @param chunks - List of chunks.
-   */
-  public void setChunks(List<ContainerProtos.ChunkInfo> chunks) {
-    this.chunks = chunks;
-  }
-
-  /**
-   * Get the total size of chunks allocated for the key.
-   * @return total size of the key.
-   */
-  public long getSize() {
-    return chunks.parallelStream().mapToLong(e->e.getLen()).sum();
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyUtils.java
deleted file mode 100644
index b1c2b00..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyUtils.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
-import org.apache.hadoop.utils.MetadataStore;
-
-import java.io.IOException;
-import java.nio.charset.Charset;
-
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.UNABLE_TO_READ_METADATA_DB;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.NO_SUCH_KEY;
-
-/**
- * Utils functions to help key functions.
- */
-public final class KeyUtils {
-  public static final String ENCODING_NAME = "UTF-8";
-  public static final Charset ENCODING = Charset.forName(ENCODING_NAME);
-
-  /**
-   * Never Constructed.
-   */
-  private KeyUtils() {
-  }
-
-  /**
-   * Get a DB handler for a given container.
-   * If the handler doesn't exist in cache yet, first create one and
-   * add into cache. This function is called with containerManager
-   * ReadLock held.
-   *
-   * @param container container.
-   * @param conf configuration.
-   * @return MetadataStore handle.
-   * @throws StorageContainerException
-   */
-  public static MetadataStore getDB(ContainerData container,
-      Configuration conf) throws StorageContainerException {
-    Preconditions.checkNotNull(container);
-    ContainerCache cache = ContainerCache.getInstance(conf);
-    Preconditions.checkNotNull(cache);
-    try {
-      return cache.getDB(container.getContainerName(), container.getDBPath());
-    } catch (IOException ex) {
-      String message =
-          String.format("Unable to open DB. DB Name: %s, Path: %s. ex: %s",
-          container.getContainerName(), container.getDBPath(), ex.getMessage());
-      throw new StorageContainerException(message, UNABLE_TO_READ_METADATA_DB);
-    }
-  }
-
-  /**
-   * Remove a DB handler from cache.
-   *
-   * @param container - Container data.
-   * @param conf - Configuration.
-   */
-  public static void removeDB(ContainerData container,
-      Configuration conf) {
-    Preconditions.checkNotNull(container);
-    ContainerCache cache = ContainerCache.getInstance(conf);
-    Preconditions.checkNotNull(cache);
-    cache.removeDB(container.getContainerName());
-  }
-  /**
-   * Shutdown all DB Handles.
-   *
-   * @param cache - Cache for DB Handles.
-   */
-  @SuppressWarnings("unchecked")
-  public static void shutdownCache(ContainerCache cache)  {
-    cache.shutdownCache();
-  }
-
-  /**
-   * Returns successful keyResponse.
-   * @param msg - Request.
-   * @return Response.
-   */
-  public static ContainerProtos.ContainerCommandResponseProto
-      getKeyResponse(ContainerProtos.ContainerCommandRequestProto msg) {
-    return ContainerUtils.getContainerResponse(msg);
-  }
-
-
-  public static ContainerProtos.ContainerCommandResponseProto
-      getKeyDataResponse(ContainerProtos.ContainerCommandRequestProto msg,
-      KeyData data) {
-    ContainerProtos.GetKeyResponseProto.Builder getKey = ContainerProtos
-        .GetKeyResponseProto.newBuilder();
-    getKey.setKeyData(data.getProtoBufMessage());
-    ContainerProtos.ContainerCommandResponseProto.Builder builder =
-        ContainerUtils.getContainerResponse(msg, ContainerProtos.Result
-            .SUCCESS, "");
-    builder.setGetKey(getKey);
-    return  builder.build();
-  }
-
-  /**
-   * Parses the key name from a bytes array.
-   * @param bytes key name in bytes.
-   * @return key name string.
-   */
-  public static String getKeyName(byte[] bytes) {
-    return new String(bytes, ENCODING);
-  }
-
-  /**
-   * Parses the {@link KeyData} from a bytes array.
-   *
-   * @param bytes key data in bytes.
-   * @return key data.
-   * @throws IOException if the bytes array is malformed or invalid.
-   */
-  public static KeyData getKeyData(byte[] bytes) throws IOException {
-    try {
-      ContainerProtos.KeyData kd = ContainerProtos.KeyData.parseFrom(bytes);
-      KeyData data = KeyData.getFromProtoBuf(kd);
-      return data;
-    } catch (IOException e) {
-      throw new StorageContainerException("Failed to parse key data from the" +
-              " bytes array.", NO_SUCH_KEY);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/package-info.java
deleted file mode 100644
index 21f31e1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.helpers;
-/**
- Contains protocol buffer helper classes and utilites used in
- impl.
- **/
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkManagerImpl.java
deleted file mode 100644
index 9052df7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkManagerImpl.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.ChunkManager;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.nio.file.StandardCopyOption;
-import java.security.NoSuchAlgorithmException;
-import java.util.concurrent.ExecutionException;
-
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.CONTAINER_INTERNAL_ERROR;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.UNSUPPORTED_REQUEST;
-
-/**
- * An implementation of ChunkManager that is used by default in ozone.
- */
-public class ChunkManagerImpl implements ChunkManager {
-  static final Logger LOG =
-      LoggerFactory.getLogger(ChunkManagerImpl.class);
-
-  private final ContainerManager containerManager;
-
-  /**
-   * Constructs a ChunkManager.
-   *
-   * @param manager - ContainerManager.
-   */
-  public ChunkManagerImpl(ContainerManager manager) {
-    this.containerManager = manager;
-  }
-
-  /**
-   * writes a given chunk.
-   *
-   * @param pipeline - Name and the set of machines that make this container.
-   * @param keyName - Name of the Key.
-   * @param info - ChunkInfo.
-   * @throws StorageContainerException
-   */
-  @Override
-  public void writeChunk(Pipeline pipeline, String keyName, ChunkInfo info,
-      byte[] data, ContainerProtos.Stage stage)
-      throws StorageContainerException {
-    // we don't want container manager to go away while we are writing chunks.
-    containerManager.readLock();
-
-    // TODO : Take keyManager Write lock here.
-    try {
-      Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
-      String containerName = pipeline.getContainerName();
-      Preconditions.checkNotNull(containerName,
-          "Container name cannot be null");
-      ContainerData container =
-          containerManager.readContainer(containerName);
-      File chunkFile = ChunkUtils.validateChunk(pipeline, container, info);
-      File tmpChunkFile = getTmpChunkFile(chunkFile, info);
-
-      LOG.debug("writing chunk:{} chunk stage:{} chunk file:{} tmp chunk file",
-          info.getChunkName(), stage, chunkFile, tmpChunkFile);
-      switch (stage) {
-      case WRITE_DATA:
-        ChunkUtils.writeData(tmpChunkFile, info, data);
-        break;
-      case COMMIT_DATA:
-        commitChunk(tmpChunkFile, chunkFile, containerName, info.getLen());
-        break;
-      case COMBINED:
-        // directly write to the chunk file
-        long oldSize = chunkFile.length();
-        ChunkUtils.writeData(chunkFile, info, data);
-        long newSize = chunkFile.length();
-        containerManager.incrBytesUsed(containerName, newSize - oldSize);
-        containerManager.incrWriteCount(containerName);
-        containerManager.incrWriteBytes(containerName, info.getLen());
-        break;
-      }
-    } catch (ExecutionException | NoSuchAlgorithmException | IOException e) {
-      LOG.error("write data failed. error: {}", e);
-      throw new StorageContainerException("Internal error: ", e,
-          CONTAINER_INTERNAL_ERROR);
-    } catch (InterruptedException e) {
-      Thread.currentThread().interrupt();
-      LOG.error("write data failed. error: {}", e);
-      throw new StorageContainerException("Internal error: ", e,
-          CONTAINER_INTERNAL_ERROR);
-    } finally {
-      containerManager.readUnlock();
-    }
-  }
-
-  // Create a temporary file in the same container directory
-  // in the format "<chunkname>.tmp"
-  private static File getTmpChunkFile(File chunkFile, ChunkInfo info)
-      throws StorageContainerException {
-    return new File(chunkFile.getParent(),
-        chunkFile.getName() +
-            OzoneConsts.CONTAINER_CHUNK_NAME_DELIMITER +
-            OzoneConsts.CONTAINER_TEMPORARY_CHUNK_PREFIX);
-  }
-
-  // Commit the chunk by renaming the temporary chunk file to chunk file
-  private void commitChunk(File tmpChunkFile, File chunkFile,
-      String containerName, long chunkLen) throws IOException {
-    long sizeDiff = tmpChunkFile.length() - chunkFile.length();
-    // It is safe to replace here as the earlier chunk if existing should be
-    // caught as part of validateChunk
-    Files.move(tmpChunkFile.toPath(), chunkFile.toPath(),
-        StandardCopyOption.REPLACE_EXISTING);
-    containerManager.incrBytesUsed(containerName, sizeDiff);
-    containerManager.incrWriteCount(containerName);
-    containerManager.incrWriteBytes(containerName, chunkLen);
-  }
-
-  /**
-   * reads the data defined by a chunk.
-   *
-   * @param pipeline - container pipeline.
-   * @param keyName - Name of the Key
-   * @param info - ChunkInfo.
-   * @return byte array
-   * @throws StorageContainerException
-   * TODO: Right now we do not support partial reads and writes of chunks.
-   * TODO: Explore if we need to do that for ozone.
-   */
-  @Override
-  public byte[] readChunk(Pipeline pipeline, String keyName, ChunkInfo info)
-      throws StorageContainerException {
-    containerManager.readLock();
-    try {
-      Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
-      String containerName = pipeline.getContainerName();
-      Preconditions.checkNotNull(containerName,
-          "Container name cannot be null");
-      ContainerData container =
-          containerManager.readContainer(containerName);
-      File chunkFile = ChunkUtils.getChunkFile(pipeline, container, info);
-      ByteBuffer data =  ChunkUtils.readData(chunkFile, info);
-      containerManager.incrReadCount(containerName);
-      containerManager.incrReadBytes(containerName, chunkFile.length());
-      return data.array();
-    } catch (ExecutionException | NoSuchAlgorithmException e) {
-      LOG.error("read data failed. error: {}", e);
-      throw new StorageContainerException("Internal error: ",
-          e, CONTAINER_INTERNAL_ERROR);
-    } catch (InterruptedException e) {
-      Thread.currentThread().interrupt();
-      LOG.error("read data failed. error: {}", e);
-      throw new StorageContainerException("Internal error: ",
-          e, CONTAINER_INTERNAL_ERROR);
-    } finally {
-      containerManager.readUnlock();
-    }
-  }
-
-  /**
-   * Deletes a given chunk.
-   *
-   * @param pipeline - Pipeline.
-   * @param keyName - Key Name
-   * @param info - Chunk Info
-   * @throws StorageContainerException
-   */
-  @Override
-  public void deleteChunk(Pipeline pipeline, String keyName, ChunkInfo info)
-      throws StorageContainerException {
-    containerManager.readLock();
-    try {
-      Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
-      String containerName = pipeline.getContainerName();
-      Preconditions.checkNotNull(containerName,
-          "Container name cannot be null");
-      File chunkFile = ChunkUtils.getChunkFile(pipeline, containerManager
-          .readContainer(containerName), info);
-      if ((info.getOffset() == 0) && (info.getLen() == chunkFile.length())) {
-        FileUtil.fullyDelete(chunkFile);
-        containerManager.decrBytesUsed(containerName, chunkFile.length());
-      } else {
-        LOG.error("Not Supported Operation. Trying to delete a " +
-            "chunk that is in shared file. chunk info : " + info.toString());
-        throw new StorageContainerException("Not Supported Operation. " +
-            "Trying to delete a chunk that is in shared file. chunk info : "
-            + info.toString(), UNSUPPORTED_REQUEST);
-      }
-    } finally {
-      containerManager.readUnlock();
-    }
-  }
-
-  /**
-   * Shutdown the chunkManager.
-   *
-   * In the chunkManager we haven't acquired any resources, so nothing to do
-   * here. This call is made with containerManager Writelock held.
-   */
-  @Override
-  public void shutdown() {
-    Preconditions.checkState(this.containerManager.hasWriteLock());
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
deleted file mode 100644
index b842493..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.impl;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.interfaces
-    .ContainerLocationManager;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerLocationManagerMXBean;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.ObjectName;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * A class that tells the ContainerManager where to place the containers.
- * Please note : There is *no* one-to-one correlation between metadata
- * Locations and data Locations.
- *
- *  For example : A user could map all container files to a
- *  SSD but leave data/metadata on bunch of other disks.
- */
-public class ContainerLocationManagerImpl implements ContainerLocationManager,
-    ContainerLocationManagerMXBean {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ContainerLocationManagerImpl.class);
-
-  private final List<ContainerStorageLocation> dataLocations;
-  private int currentIndex;
-  private final List<StorageLocation> metadataLocations;
-  private final ObjectName jmxbean;
-
-  /**
-   * Constructs a Location Manager.
-   * @param metadataLocations  - Refers to the metadataLocations
-   * where we store the container metadata.
-   * @param dataDirs - metadataLocations where we store the actual
-   * data or chunk files.
-   * @param conf - configuration.
-   * @throws IOException
-   */
-  public ContainerLocationManagerImpl(List<StorageLocation> metadataLocations,
-      List<StorageLocation> dataDirs, Configuration conf)
-      throws IOException {
-    dataLocations = new LinkedList<>();
-    for (StorageLocation dataDir : dataDirs) {
-      dataLocations.add(new ContainerStorageLocation(dataDir, conf));
-    }
-    this.metadataLocations = metadataLocations;
-    jmxbean = MBeans.register("OzoneDataNode",
-        ContainerLocationManager.class.getSimpleName(), this);
-  }
-
-  /**
-   * Returns the path where the container should be placed from a set of
-   * metadataLocations.
-   *
-   * @return A path where we should place this container and metadata.
-   * @throws IOException
-   */
-  @Override
-  public Path getContainerPath()
-      throws IOException {
-    Preconditions.checkState(metadataLocations.size() > 0);
-    int index = currentIndex % metadataLocations.size();
-    return Paths.get(metadataLocations.get(index).getNormalizedUri());
-  }
-
-  /**
-   * Returns the path where the container Data file are stored.
-   *
-   * @return  a path where we place the LevelDB and data files of a container.
-   * @throws IOException
-   */
-  @Override
-  public Path getDataPath(String containerName) throws IOException {
-    Path currentPath = Paths.get(
-        dataLocations.get(currentIndex++ % dataLocations.size())
-            .getNormalizedUri());
-    currentPath = currentPath.resolve(OzoneConsts.CONTAINER_PREFIX);
-    return currentPath.resolve(containerName);
-  }
-
-  @Override
-  public StorageLocationReport[] getLocationReport() throws IOException {
-    StorageLocationReport[] reports =
-        new StorageLocationReport[dataLocations.size()];
-    for (int idx = 0; idx < dataLocations.size(); idx++) {
-      ContainerStorageLocation loc = dataLocations.get(idx);
-      long scmUsed = 0;
-      long remaining = 0;
-      try {
-        scmUsed = loc.getScmUsed();
-        remaining = loc.getAvailable();
-      } catch (IOException ex) {
-        LOG.warn("Failed to get scmUsed and remaining for container " +
-            "storage location {}", loc.getNormalizedUri());
-        // reset scmUsed and remaining if df/du failed.
-        scmUsed = 0;
-        remaining = 0;
-      }
-
-      // TODO: handle failed storage
-      // For now, include storage report for location that failed to get df/du.
-      StorageLocationReport r = new StorageLocationReport(
-          loc.getStorageUuId(), false, loc.getCapacity(),
-          scmUsed, remaining);
-      reports[idx] = r;
-    }
-    return reports;
-  }
-
-  /**
-   * Supports clean shutdown of container location du threads.
-   *
-   * @throws IOException
-   */
-  @Override
-  public void shutdown() throws IOException {
-    for (ContainerStorageLocation loc: dataLocations) {
-      loc.shutdown();
-    }
-    MBeans.unregister(jmxbean);
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[40/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolPB.java
deleted file mode 100644
index 8acca8a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolPB.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.protocolPB;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.KeySpaceManagerService;
-
-/**
- * Protocol used to communicate with KSM.
- */
-@ProtocolInfo(protocolName =
-    "org.apache.hadoop.ozone.protocol.KeySpaceManagerProtocol",
-    protocolVersion = 1)
-@InterfaceAudience.Private
-public interface KeySpaceManagerProtocolPB
-    extends KeySpaceManagerService.BlockingInterface {
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/package-info.java
deleted file mode 100644
index 67f9f7b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.ksm.protocolPB;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/package-info.java
deleted file mode 100644
index 764ff3c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone;
-
-/**
- This package contains ozone client side libraries.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/KSMPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/KSMPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/KSMPBHelper.java
deleted file mode 100644
index fdc3ce7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/KSMPBHelper.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocolPB;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.OzoneAclInfo;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclType;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclRights;
-
-/**
- * Utilities for converting protobuf classes.
- */
-public final class KSMPBHelper {
-
-  private KSMPBHelper() {
-    /** Hidden constructor */
-  }
-
-  /**
-   * Converts OzoneAcl into protobuf's OzoneAclInfo.
-   * @return OzoneAclInfo
-   */
-  public static OzoneAclInfo convertOzoneAcl(OzoneAcl acl) {
-    OzoneAclInfo.OzoneAclType aclType;
-    switch(acl.getType()) {
-    case USER:
-      aclType = OzoneAclType.USER;
-      break;
-    case GROUP:
-      aclType = OzoneAclType.GROUP;
-      break;
-    case WORLD:
-      aclType = OzoneAclType.WORLD;
-      break;
-    default:
-      throw new IllegalArgumentException("ACL type is not recognized");
-    }
-    OzoneAclInfo.OzoneAclRights aclRights;
-    switch(acl.getRights()) {
-    case READ:
-      aclRights = OzoneAclRights.READ;
-      break;
-    case WRITE:
-      aclRights = OzoneAclRights.WRITE;
-      break;
-    case READ_WRITE:
-      aclRights = OzoneAclRights.READ_WRITE;
-      break;
-    default:
-      throw new IllegalArgumentException("ACL right is not recognized");
-    }
-
-    return OzoneAclInfo.newBuilder().setType(aclType)
-        .setName(acl.getName())
-        .setRights(aclRights)
-        .build();
-  }
-
-  /**
-   * Converts protobuf's OzoneAclInfo into OzoneAcl.
-   * @return OzoneAcl
-   */
-  public static OzoneAcl convertOzoneAcl(OzoneAclInfo aclInfo) {
-    OzoneAcl.OzoneACLType aclType;
-    switch(aclInfo.getType()) {
-    case USER:
-      aclType = OzoneAcl.OzoneACLType.USER;
-      break;
-    case GROUP:
-      aclType = OzoneAcl.OzoneACLType.GROUP;
-      break;
-    case WORLD:
-      aclType = OzoneAcl.OzoneACLType.WORLD;
-      break;
-    default:
-      throw new IllegalArgumentException("ACL type is not recognized");
-    }
-    OzoneAcl.OzoneACLRights aclRights;
-    switch(aclInfo.getRights()) {
-    case READ:
-      aclRights = OzoneAcl.OzoneACLRights.READ;
-      break;
-    case WRITE:
-      aclRights = OzoneAcl.OzoneACLRights.WRITE;
-      break;
-    case READ_WRITE:
-      aclRights = OzoneAcl.OzoneACLRights.READ_WRITE;
-      break;
-    default:
-      throw new IllegalArgumentException("ACL right is not recognized");
-    }
-
-    return new OzoneAcl(aclType, aclInfo.getName(), aclRights);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java
deleted file mode 100644
index 860386d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.protocolPB;
-
-/**
- * This package contains classes for the Protocol Buffers binding of Ozone
- * protocols.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerID.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerID.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerID.java
deleted file mode 100644
index a51d3b7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerID.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.ozone.scm.container.ContainerStates;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.math3.util.MathUtils;
-
-/**
- * Container ID is an integer that is a value between 1..MAX_CONTAINER ID.
- * <p>
- * We are creating a specific type for this to avoid mixing this with
- * normal integers in code.
- */
-public class ContainerID implements Comparable {
-
-  private final long id;
-
-  /**
-   * Constructs ContainerID.
-   *
-   * @param id int
-   */
-  public ContainerID(long id) {
-    Preconditions.checkState(id > 0,
-        "Container ID should be a positive int");
-    this.id = id;
-  }
-
-  /**
-   * Factory method for creation of ContainerID.
-   * @param containerID  long
-   * @return ContainerID.
-   */
-  public static ContainerID valueof(long containerID) {
-    Preconditions.checkState(containerID > 0);
-    return new ContainerID(containerID);
-  }
-
-  /**
-   * Returns int representation of ID.
-   *
-   * @return int
-   */
-  public long getId() {
-    return id;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-
-    ContainerID that = (ContainerID) o;
-
-    return id == that.id;
-  }
-
-  @Override
-  public int hashCode() {
-    return MathUtils.hash(id);
-  }
-
-  @Override
-  public int compareTo(Object o) {
-    Preconditions.checkNotNull(o);
-    if (o instanceof ContainerID) {
-      return Long.compare(((ContainerID) o).getId(), this.getId());
-    }
-    throw new IllegalArgumentException("Object O, should be an instance " +
-        "of ContainerID");
-  }
-
-  @Override
-  public String toString() {
-    return "id=" + id;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java
deleted file mode 100644
index 61f5609..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-/**
- * Container States.
- */
-package org.apache.hadoop.ozone.scm.container.ContainerStates;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java
deleted file mode 100644
index fbe9637..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java
+++ /dev/null
@@ -1,260 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.scm;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * This class contains constants for configuration keys used in SCM.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public final class ScmConfigKeys {
-
-  public static final String SCM_CONTAINER_CLIENT_STALE_THRESHOLD_KEY =
-      "scm.container.client.idle.threshold";
-  public static final String SCM_CONTAINER_CLIENT_STALE_THRESHOLD_DEFAULT =
-      "10s";
-
-  public static final String SCM_CONTAINER_CLIENT_MAX_SIZE_KEY =
-      "scm.container.client.max.size";
-  public static final int SCM_CONTAINER_CLIENT_MAX_SIZE_DEFAULT =
-      256;
-
-  public static final String SCM_CONTAINER_CLIENT_MAX_OUTSTANDING_REQUESTS =
-      "scm.container.client.max.outstanding.requests";
-  public static final int SCM_CONTAINER_CLIENT_MAX_OUTSTANDING_REQUESTS_DEFAULT
-      = 100;
-
-  public static final String DFS_CONTAINER_RATIS_ENABLED_KEY
-      = "dfs.container.ratis.enabled";
-  public static final boolean DFS_CONTAINER_RATIS_ENABLED_DEFAULT
-      = false;
-  public static final String DFS_CONTAINER_RATIS_RPC_TYPE_KEY
-      = "dfs.container.ratis.rpc.type";
-  public static final String DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT
-      = "GRPC";
-  public static final String DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY
-      = "dfs.container.ratis.num.write.chunk.threads";
-  public static final int DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT
-      = 60;
-  public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY =
-      "dfs.container.ratis.segment.size";
-  public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
-      1 * 1024 * 1024 * 1024;
-  public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY =
-      "dfs.container.ratis.segment.preallocated.size";
-  public static final int
-      DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = 128 * 1024 * 1024;
-
-  // TODO : this is copied from OzoneConsts, may need to move to a better place
-  public static final String OZONE_SCM_CHUNK_SIZE_KEY = "ozone.scm.chunk.size";
-  // 16 MB by default
-  public static final int OZONE_SCM_CHUNK_SIZE_DEFAULT = 16 * 1024 * 1024;
-  public static final int OZONE_SCM_CHUNK_MAX_SIZE = 32 * 1024 * 1024;
-
-  public static final String OZONE_SCM_CLIENT_PORT_KEY =
-      "ozone.scm.client.port";
-  public static final int OZONE_SCM_CLIENT_PORT_DEFAULT = 9860;
-
-  public static final String OZONE_SCM_DATANODE_PORT_KEY =
-      "ozone.scm.datanode.port";
-  public static final int OZONE_SCM_DATANODE_PORT_DEFAULT = 9861;
-
-  // OZONE_KSM_PORT_DEFAULT = 9862
-  public static final String OZONE_SCM_BLOCK_CLIENT_PORT_KEY =
-      "ozone.scm.block.client.port";
-  public static final int OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT = 9863;
-
-  // Container service client
-  public static final String OZONE_SCM_CLIENT_ADDRESS_KEY =
-      "ozone.scm.client.address";
-  public static final String OZONE_SCM_CLIENT_BIND_HOST_KEY =
-      "ozone.scm.client.bind.host";
-  public static final String OZONE_SCM_CLIENT_BIND_HOST_DEFAULT =
-      "0.0.0.0";
-
-  // Block service client
-  public static final String OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY =
-      "ozone.scm.block.client.address";
-  public static final String OZONE_SCM_BLOCK_CLIENT_BIND_HOST_KEY =
-      "ozone.scm.block.client.bind.host";
-  public static final String OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT =
-      "0.0.0.0";
-
-  public static final String OZONE_SCM_DATANODE_ADDRESS_KEY =
-      "ozone.scm.datanode.address";
-  public static final String OZONE_SCM_DATANODE_BIND_HOST_KEY =
-      "ozone.scm.datanode.bind.host";
-  public static final String OZONE_SCM_DATANODE_BIND_HOST_DEFAULT =
-      "0.0.0.0";
-
-  public static final String OZONE_SCM_HTTP_ENABLED_KEY =
-      "ozone.scm.http.enabled";
-  public static final String OZONE_SCM_HTTP_BIND_HOST_KEY =
-      "ozone.scm.http-bind-host";
-  public static final String OZONE_SCM_HTTPS_BIND_HOST_KEY =
-      "ozone.scm.https-bind-host";
-  public static final String OZONE_SCM_HTTP_ADDRESS_KEY =
-      "ozone.scm.http-address";
-  public static final String OZONE_SCM_HTTPS_ADDRESS_KEY =
-      "ozone.scm.https-address";
-  public static final String OZONE_SCM_KEYTAB_FILE =
-      "ozone.scm.keytab.file";
-  public static final String OZONE_SCM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
-  public static final int OZONE_SCM_HTTP_BIND_PORT_DEFAULT = 9876;
-  public static final int OZONE_SCM_HTTPS_BIND_PORT_DEFAULT = 9877;
-
-
-  public static final String OZONE_SCM_HANDLER_COUNT_KEY =
-      "ozone.scm.handler.count.key";
-  public static final int OZONE_SCM_HANDLER_COUNT_DEFAULT = 10;
-
-  public static final String OZONE_SCM_HEARTBEAT_INTERVAL =
-      "ozone.scm.heartbeat.interval";
-  public static final String OZONE_SCM_HEARBEAT_INTERVAL_DEFAULT =
-      "30s";
-
-  public static final String OZONE_SCM_DEADNODE_INTERVAL =
-      "ozone.scm.dead.node.interval";
-  public static final String OZONE_SCM_DEADNODE_INTERVAL_DEFAULT =
-      "10m";
-
-  public static final String OZONE_SCM_MAX_HB_COUNT_TO_PROCESS =
-      "ozone.scm.max.hb.count.to.process";
-  public static final int OZONE_SCM_MAX_HB_COUNT_TO_PROCESS_DEFAULT = 5000;
-
-  public static final String OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL =
-      "ozone.scm.heartbeat.thread.interval";
-  public static final String OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_DEFAULT =
-      "3s";
-
-  public static final String OZONE_SCM_STALENODE_INTERVAL =
-      "ozone.scm.stale.node.interval";
-  public static final String OZONE_SCM_STALENODE_INTERVAL_DEFAULT =
-      "90s";
-
-  public static final String OZONE_SCM_HEARTBEAT_RPC_TIMEOUT =
-      "ozone.scm.heartbeat.rpc-timeout";
-  public static final long OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT =
-      1000;
-
-  /**
-   * Defines how frequently we will log the missing of heartbeat to a specific
-   * SCM. In the default case we will write a warning message for each 10
-   * sequential heart beats that we miss to a specific SCM. This is to avoid
-   * overrunning the log with lots of HB missed Log statements.
-   */
-  public static final String OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT =
-      "ozone.scm.heartbeat.log.warn.interval.count";
-  public static final int OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT =
-      10;
-
-  // ozone.scm.names key is a set of DNS | DNS:PORT | IP Address | IP:PORT.
-  // Written as a comma separated string. e.g. scm1, scm2:8020, 7.7.7.7:7777
-  //
-  // If this key is not specified datanodes will not be able to find
-  // SCM. The SCM membership can be dynamic, so this key should contain
-  // all possible SCM names. Once the SCM leader is discovered datanodes will
-  // get the right list of SCMs to heartbeat to from the leader.
-  // While it is good for the datanodes to know the names of all SCM nodes,
-  // it is sufficient to actually know the name of on working SCM. That SCM
-  // will be able to return the information about other SCMs that are part of
-  // the SCM replicated Log.
-  //
-  //In case of a membership change, any one of the SCM machines will be
-  // able to send back a new list to the datanodes.
-  public static final String OZONE_SCM_NAMES = "ozone.scm.names";
-
-  public static final int OZONE_SCM_DEFAULT_PORT =
-      OZONE_SCM_DATANODE_PORT_DEFAULT;
-  // File Name and path where datanode ID is to written to.
-  // if this value is not set then container startup will fail.
-  public static final String OZONE_SCM_DATANODE_ID = "ozone.scm.datanode.id";
-
-  public static final String OZONE_SCM_DATANODE_ID_PATH_DEFAULT = "datanode.id";
-
-  public static final String OZONE_SCM_DB_CACHE_SIZE_MB =
-      "ozone.scm.db.cache.size.mb";
-  public static final int OZONE_SCM_DB_CACHE_SIZE_DEFAULT = 128;
-
-  public static final String OZONE_SCM_CONTAINER_SIZE_GB =
-      "ozone.scm.container.size.gb";
-  public static final int OZONE_SCM_CONTAINER_SIZE_DEFAULT = 5;
-
-  public static final String OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY =
-      "ozone.scm.container.placement.impl";
-
-  public static final String OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE =
-      "ozone.scm.container.provision_batch_size";
-  public static final int OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE_DEFAULT = 20;
-
-  public static final String OZONE_SCM_CONTAINER_DELETION_CHOOSING_POLICY =
-      "ozone.scm.container.deletion-choosing.policy";
-
-  public static final String OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT =
-      "ozone.scm.container.creation.lease.timeout";
-
-  public static final String
-      OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT_DEFAULT = "60s";
-
-  /**
-   * Don't start processing a pool if we have not had a minimum number of
-   * seconds from the last processing.
-   */
-  public static final String OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL =
-      "ozone.scm.container.report.processing.interval";
-  public static final String
-      OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT = "60s";
-
-  /**
-   * This determines the total number of pools to be processed in parallel.
-   */
-  public static final String OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS =
-      "ozone.scm.max.nodepool.processing.threads";
-  public static final int OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT = 1;
-  /**
-   * These 2 settings control the number of threads in executor pool and time
-   * outs for thw container reports from all nodes.
-   */
-  public static final String OZONE_SCM_MAX_CONTAINER_REPORT_THREADS =
-      "ozone.scm.max.container.report.threads";
-  public static final int OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT = 100;
-  public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT =
-      "ozone.scm.container.reports.wait.timeout";
-  public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT =
-      "5m";
-
-  public static final String OZONE_SCM_BLOCK_DELETION_MAX_RETRY =
-      "ozone.scm.block.deletion.max.retry";
-  public static final int OZONE_SCM_BLOCK_DELETION_MAX_RETRY_DEFAULT = 4096;
-
-  // Once a container usage crosses this threshold, it is eligible for
-  // closing.
-  public static final String OZONE_SCM_CONTAINER_CLOSE_THRESHOLD =
-      "ozone.scm.container.close.threshold";
-  public static final float OZONE_SCM_CONTAINER_CLOSE_THRESHOLD_DEFAULT = 0.9f;
-
-  /**
-   * Never constructed.
-   */
-  private ScmConfigKeys() {
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmInfo.java
deleted file mode 100644
index e442fe2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmInfo.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.scm;
-
-/**
- * ScmInfo wraps the result returned from SCM#getScmInfo which
- * contains clusterId and the SCM Id.
- */
-public final class ScmInfo {
-  private String clusterId;
-  private String scmId;
-
-  /**
-   * Builder for ScmInfo.
-   */
-  public static class Builder {
-    private String clusterId;
-    private String scmId;
-
-    /**
-     * sets the cluster id.
-     * @param cid clusterId to be set
-     * @return Builder for ScmInfo
-     */
-    public Builder setClusterId(String cid) {
-      this.clusterId = cid;
-      return this;
-    }
-
-    /**
-     * sets the scmId.
-     * @param id scmId
-     * @return Builder for scmInfo
-     */
-    public Builder setScmId(String id) {
-      this.scmId = id;
-      return this;
-    }
-
-    public ScmInfo build() {
-      return new ScmInfo(clusterId, scmId);
-    }
-  }
-
-  private ScmInfo(String clusterId, String scmId) {
-    this.clusterId = clusterId;
-    this.scmId = scmId;
-  }
-
-  /**
-   * Gets the clusterId from the Version file.
-   * @return ClusterId
-   */
-  public String getClusterId() {
-    return clusterId;
-  }
-
-  /**
-   * Gets the SCM Id from the Version file.
-   * @return SCM Id
-   */
-  public String getScmId() {
-    return scmId;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClient.java
deleted file mode 100644
index bde9064..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClient.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.scm;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import io.netty.bootstrap.Bootstrap;
-import io.netty.channel.Channel;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.socket.nio.NioSocketChannel;
-import io.netty.handler.logging.LogLevel;
-import io.netty.handler.logging.LoggingHandler;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-import java.io.IOException;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.List;
-import java.util.concurrent.Semaphore;
-
-/**
- * A Client for the storageContainer protocol.
- */
-public class XceiverClient extends XceiverClientSpi {
-  static final Logger LOG = LoggerFactory.getLogger(XceiverClient.class);
-  private final Pipeline pipeline;
-  private final Configuration config;
-  private Channel channel;
-  private Bootstrap b;
-  private EventLoopGroup group;
-  private final Semaphore semaphore;
-
-  /**
-   * Constructs a client that can communicate with the Container framework on
-   * data nodes.
-   *
-   * @param pipeline - Pipeline that defines the machines.
-   * @param config -- Ozone Config
-   */
-  public XceiverClient(Pipeline pipeline, Configuration config) {
-    super();
-    Preconditions.checkNotNull(pipeline);
-    Preconditions.checkNotNull(config);
-    this.pipeline = pipeline;
-    this.config = config;
-    this.semaphore =
-        new Semaphore(OzoneClientUtils.getMaxOutstandingRequests(config));
-  }
-
-  @Override
-  public void connect() throws Exception {
-    if (channel != null && channel.isActive()) {
-      throw new IOException("This client is already connected to a host.");
-    }
-
-    group = new NioEventLoopGroup();
-    b = new Bootstrap();
-    b.group(group)
-        .channel(NioSocketChannel.class)
-        .handler(new LoggingHandler(LogLevel.INFO))
-        .handler(new XceiverClientInitializer(this.pipeline, semaphore));
-    DatanodeID leader = this.pipeline.getLeader();
-
-    // read port from the data node, on failure use default configured
-    // port.
-    int port = leader.getContainerPort();
-    if (port == 0) {
-      port = config.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
-          OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
-    }
-    LOG.debug("Connecting to server Port : " + port);
-    channel = b.connect(leader.getHostName(), port).sync().channel();
-  }
-
-  /**
-   * Returns if the exceiver client connects to a server.
-   *
-   * @return True if the connection is alive, false otherwise.
-   */
-  @VisibleForTesting
-  public boolean isConnected() {
-    return channel.isActive();
-  }
-
-  @Override
-  public void close() {
-    if (group != null) {
-      group.shutdownGracefully().awaitUninterruptibly();
-    }
-  }
-
-  @Override
-  public Pipeline getPipeline() {
-    return pipeline;
-  }
-
-  @Override
-  public ContainerProtos.ContainerCommandResponseProto sendCommand(
-      ContainerProtos.ContainerCommandRequestProto request) throws IOException {
-    try {
-      if ((channel == null) || (!channel.isActive())) {
-        throw new IOException("This channel is not connected.");
-      }
-      XceiverClientHandler handler =
-          channel.pipeline().get(XceiverClientHandler.class);
-
-      return handler.sendCommand(request);
-    } catch (ExecutionException | InterruptedException e) {
-      /**
-       * In case the netty channel handler throws an exception,
-       * the exception thrown will be wrapped within {@link ExecutionException}.
-       * Unwarpping here so that original exception gets passed
-       * to to the client.
-       */
-      if (e instanceof ExecutionException) {
-        Throwable cause = e.getCause();
-        if (cause instanceof IOException) {
-          throw (IOException) cause;
-        }
-      }
-      throw new IOException(
-          "Unexpected exception during execution:" + e.getMessage());
-    }
-  }
-
-  /**
-   * Sends a given command to server gets a waitable future back.
-   *
-   * @param request Request
-   * @return Response to the command
-   * @throws IOException
-   */
-  @Override
-  public CompletableFuture<ContainerProtos.ContainerCommandResponseProto>
-      sendCommandAsync(ContainerProtos.ContainerCommandRequestProto request)
-      throws IOException, ExecutionException, InterruptedException {
-    if ((channel == null) || (!channel.isActive())) {
-      throw new IOException("This channel is not connected.");
-    }
-    XceiverClientHandler handler =
-        channel.pipeline().get(XceiverClientHandler.class);
-    return handler.sendCommandAsync(request);
-  }
-
-  /**
-   * Create a pipeline.
-   *
-   * @param pipelineID - Name of the pipeline.
-   * @param datanodes - Datanodes
-   */
-  @Override
-  public void createPipeline(String pipelineID, List<DatanodeID> datanodes)
-      throws IOException {
-    // For stand alone pipeline, there is no notion called setup pipeline.
-    return;
-  }
-
-  /**
-   * Returns pipeline Type.
-   *
-   * @return - Stand Alone as the type.
-   */
-  @Override
-  public OzoneProtos.ReplicationType getPipelineType() {
-    return OzoneProtos.ReplicationType.STAND_ALONE;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientHandler.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientHandler.java
deleted file mode 100644
index ac2cf1a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientHandler.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.scm;
-
-import com.google.common.base.Preconditions;
-
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.SimpleChannelInboundHandler;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .ContainerCommandResponseProto;
-
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Iterator;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.Semaphore;
-
-/**
- * Netty client handler.
- */
-public class XceiverClientHandler extends
-    SimpleChannelInboundHandler<ContainerCommandResponseProto> {
-
-  static final Logger LOG = LoggerFactory.getLogger(XceiverClientHandler.class);
-  private final ConcurrentMap<String, ResponseFuture> responses =
-      new ConcurrentHashMap<>();
-
-  private final Pipeline pipeline;
-  private volatile Channel channel;
-  private XceiverClientMetrics metrics;
-  private final Semaphore semaphore;
-
-  /**
-   * Constructs a client that can communicate to a container server.
-   */
-  public XceiverClientHandler(Pipeline pipeline, Semaphore semaphore) {
-    super(false);
-    Preconditions.checkNotNull(pipeline);
-    this.pipeline = pipeline;
-    this.metrics = XceiverClientManager.getXceiverClientMetrics();
-    this.semaphore = semaphore;
-  }
-
-  /**
-   * <strong>Please keep in mind that this method will be renamed to {@code
-   * messageReceived(ChannelHandlerContext, I)} in 5.0.</strong>
-   * <p>
-   * Is called for each message of type {@link ContainerProtos
-   * .ContainerCommandResponseProto}.
-   *
-   * @param ctx the {@link ChannelHandlerContext} which this {@link
-   * SimpleChannelInboundHandler} belongs to
-   * @param msg the message to handle
-   * @throws Exception is thrown if an error occurred
-   */
-  @Override
-  public void channelRead0(ChannelHandlerContext ctx,
-      ContainerProtos.ContainerCommandResponseProto msg)
-      throws Exception {
-    Preconditions.checkNotNull(msg);
-    metrics.decrPendingContainerOpsMetrics(msg.getCmdType());
-
-    String key = msg.getTraceID();
-    ResponseFuture response = responses.remove(key);
-    semaphore.release();
-
-    if (response != null) {
-      response.getFuture().complete(msg);
-
-      long requestTime = response.getRequestTime();
-      metrics.addContainerOpsLatency(msg.getCmdType(),
-          Time.monotonicNowNanos() - requestTime);
-    } else {
-      LOG.error("A reply received for message that was not queued. trace " +
-          "ID: {}", msg.getTraceID());
-    }
-  }
-
-  @Override
-  public void channelRegistered(ChannelHandlerContext ctx) {
-    LOG.debug("channelRegistered: Connected to ctx");
-    channel = ctx.channel();
-  }
-
-  @Override
-  public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
-    LOG.info("Exception in client " + cause.toString());
-    Iterator<String> keyIterator = responses.keySet().iterator();
-    while (keyIterator.hasNext()) {
-      ResponseFuture response = responses.remove(keyIterator.next());
-      response.getFuture().completeExceptionally(cause);
-      semaphore.release();
-    }
-    ctx.close();
-  }
-
-  /**
-   * Since netty is async, we send a work request and then wait until a response
-   * appears in the reply queue. This is simple sync interface for clients. we
-   * should consider building async interfaces for client if this turns out to
-   * be a performance bottleneck.
-   *
-   * @param request - request.
-   * @return -- response
-   */
-
-  public ContainerCommandResponseProto sendCommand(
-      ContainerProtos.ContainerCommandRequestProto request)
-      throws ExecutionException, InterruptedException {
-    Future<ContainerCommandResponseProto> future = sendCommandAsync(request);
-    return future.get();
-  }
-
-  /**
-   * SendCommandAsyc queues a command to the Netty Subsystem and returns a
-   * CompletableFuture. This Future is marked compeleted in the channelRead0
-   * when the call comes back.
-   * @param request - Request to execute
-   * @return CompletableFuture
-   */
-  public CompletableFuture<ContainerCommandResponseProto> sendCommandAsync(
-      ContainerProtos.ContainerCommandRequestProto request)
-      throws InterruptedException {
-
-    // Throw an exception of request doesn't have traceId
-    if (StringUtils.isEmpty(request.getTraceID())) {
-      throw new IllegalArgumentException("Invalid trace ID");
-    }
-
-    // Setting the datanode ID in the commands, so that we can distinguish
-    // commands when the cluster simulator is running.
-    if(!request.hasDatanodeID()) {
-      throw new IllegalArgumentException("Invalid Datanode ID");
-    }
-
-    metrics.incrPendingContainerOpsMetrics(request.getCmdType());
-
-    CompletableFuture<ContainerCommandResponseProto> future
-        = new CompletableFuture<>();
-    ResponseFuture response = new ResponseFuture(future,
-        Time.monotonicNowNanos());
-    semaphore.acquire();
-    ResponseFuture previous = responses.putIfAbsent(
-        request.getTraceID(), response);
-    if (previous != null) {
-      LOG.error("Command with Trace already exists. Ignoring this command. " +
-              "{}. Previous Command: {}", request.getTraceID(),
-          previous.toString());
-      throw new IllegalStateException("Duplicate trace ID. Command with this " +
-          "trace ID is already executing. Please ensure that " +
-          "trace IDs are not reused. ID: " + request.getTraceID());
-    }
-
-    channel.writeAndFlush(request);
-    return response.getFuture();
-  }
-
-  /**
-   * Class wraps response future info.
-   */
-  static class ResponseFuture {
-    private final long requestTime;
-    private final CompletableFuture<ContainerCommandResponseProto> future;
-
-    ResponseFuture(CompletableFuture<ContainerCommandResponseProto> future,
-        long requestTime) {
-      this.future = future;
-      this.requestTime = requestTime;
-    }
-
-    public long getRequestTime() {
-      return requestTime;
-    }
-
-    public CompletableFuture<ContainerCommandResponseProto> getFuture() {
-      return future;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientInitializer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientInitializer.java
deleted file mode 100644
index 6aac960..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientInitializer.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.scm;
-
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.ChannelPipeline;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.handler.codec.protobuf.ProtobufDecoder;
-import io.netty.handler.codec.protobuf.ProtobufEncoder;
-import io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder;
-import io.netty.handler.codec.protobuf.ProtobufVarint32LengthFieldPrepender;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.util.concurrent.Semaphore;
-
-/**
- * Setup the netty pipeline.
- */
-public class XceiverClientInitializer extends
-    ChannelInitializer<SocketChannel> {
-  private final Pipeline pipeline;
-  private final Semaphore semaphore;
-
-  /**
-   * Constructs an Initializer for the client pipeline.
-   * @param pipeline  - Pipeline.
-   */
-  public XceiverClientInitializer(Pipeline pipeline, Semaphore semaphore) {
-    this.pipeline = pipeline;
-    this.semaphore = semaphore;
-  }
-
-  /**
-   * This method will be called once when the Channel is registered. After
-   * the method returns this instance will be removed from the
-   * ChannelPipeline of the Channel.
-   *
-   * @param ch   Channel which was registered.
-   * @throws Exception is thrown if an error occurs. In that case the
-   *                   Channel will be closed.
-   */
-  @Override
-  protected void initChannel(SocketChannel ch) throws Exception {
-    ChannelPipeline p = ch.pipeline();
-
-    p.addLast(new ProtobufVarint32FrameDecoder());
-    p.addLast(new ProtobufDecoder(ContainerProtos
-        .ContainerCommandResponseProto.getDefaultInstance()));
-
-    p.addLast(new ProtobufVarint32LengthFieldPrepender());
-    p.addLast(new ProtobufEncoder());
-
-    p.addLast(new XceiverClientHandler(this.pipeline, this.semaphore));
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientManager.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientManager.java
deleted file mode 100644
index 161cdce..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientManager.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.scm;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.Callable;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .SCM_CONTAINER_CLIENT_STALE_THRESHOLD_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .SCM_CONTAINER_CLIENT_STALE_THRESHOLD_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .SCM_CONTAINER_CLIENT_MAX_SIZE_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .SCM_CONTAINER_CLIENT_MAX_SIZE_DEFAULT;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos
-    .ReplicationType.RATIS;
-
-/**
- * XceiverClientManager is responsible for the lifecycle of XceiverClient
- * instances.  Callers use this class to acquire an XceiverClient instance
- * connected to the desired container pipeline.  When done, the caller also uses
- * this class to release the previously acquired XceiverClient instance.
- *
- *
- * This class caches connection to container for reuse purpose, such that
- * accessing same container frequently will be through the same connection
- * without reestablishing connection. But the connection will be closed if
- * not being used for a period of time.
- */
-public class XceiverClientManager implements Closeable {
-
-  //TODO : change this to SCM configuration class
-  private final Configuration conf;
-  private final Cache<String, XceiverClientSpi> clientCache;
-  private final boolean useRatis;
-
-  private static XceiverClientMetrics metrics;
-  /**
-   * Creates a new XceiverClientManager.
-   *
-   * @param conf configuration
-   */
-  public XceiverClientManager(Configuration conf) {
-    Preconditions.checkNotNull(conf);
-    int maxSize = conf.getInt(SCM_CONTAINER_CLIENT_MAX_SIZE_KEY,
-        SCM_CONTAINER_CLIENT_MAX_SIZE_DEFAULT);
-    long staleThresholdMs = conf.getTimeDuration(
-        SCM_CONTAINER_CLIENT_STALE_THRESHOLD_KEY,
-        SCM_CONTAINER_CLIENT_STALE_THRESHOLD_DEFAULT, TimeUnit.MILLISECONDS);
-    this.useRatis = conf.getBoolean(
-        ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY,
-        ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT);
-    this.conf = conf;
-    this.clientCache = CacheBuilder.newBuilder()
-        .expireAfterAccess(staleThresholdMs, TimeUnit.MILLISECONDS)
-        .maximumSize(maxSize)
-        .removalListener(
-            new RemovalListener<String, XceiverClientSpi>() {
-            @Override
-            public void onRemoval(
-                RemovalNotification<String, XceiverClientSpi>
-                  removalNotification) {
-              synchronized (clientCache) {
-                // Mark the entry as evicted
-                XceiverClientSpi info = removalNotification.getValue();
-                info.setEvicted();
-              }
-            }
-          }).build();
-  }
-
-  @VisibleForTesting
-  public Cache<String, XceiverClientSpi> getClientCache() {
-    return clientCache;
-  }
-
-  /**
-   * Acquires a XceiverClientSpi connected to a container capable of
-   * storing the specified key.
-   *
-   * If there is already a cached XceiverClientSpi, simply return
-   * the cached otherwise create a new one.
-   *
-   * @param pipeline the container pipeline for the client connection
-   * @return XceiverClientSpi connected to a container
-   * @throws IOException if a XceiverClientSpi cannot be acquired
-   */
-  public XceiverClientSpi acquireClient(Pipeline pipeline)
-      throws IOException {
-    Preconditions.checkNotNull(pipeline);
-    Preconditions.checkArgument(pipeline.getMachines() != null);
-    Preconditions.checkArgument(!pipeline.getMachines().isEmpty());
-
-    synchronized (clientCache) {
-      XceiverClientSpi info = getClient(pipeline);
-      info.incrementReference();
-      return info;
-    }
-  }
-
-  /**
-   * Releases a XceiverClientSpi after use.
-   *
-   * @param client client to release
-   */
-  public void releaseClient(XceiverClientSpi client) {
-    Preconditions.checkNotNull(client);
-    synchronized (clientCache) {
-      client.decrementReference();
-    }
-  }
-
-  private XceiverClientSpi getClient(Pipeline pipeline)
-      throws IOException {
-    String containerName = pipeline.getContainerName();
-    try {
-      return clientCache.get(containerName,
-          new Callable<XceiverClientSpi>() {
-          @Override
-          public XceiverClientSpi call() throws Exception {
-            XceiverClientSpi client = pipeline.getType() == RATIS ?
-                    XceiverClientRatis.newXceiverClientRatis(pipeline, conf)
-                    : new XceiverClient(pipeline, conf);
-            client.connect();
-            return client;
-          }
-        });
-    } catch (Exception e) {
-      throw new IOException(
-          "Exception getting XceiverClient: " + e.toString(), e);
-    }
-  }
-
-  /**
-   * Close and remove all the cached clients.
-   */
-  public void close() {
-    //closing is done through RemovalListener
-    clientCache.invalidateAll();
-    clientCache.cleanUp();
-
-    if (metrics != null) {
-      metrics.unRegister();
-    }
-  }
-
-  /**
-   * Tells us if Ratis is enabled for this cluster.
-   * @return True if Ratis is enabled.
-   */
-  public boolean isUseRatis() {
-    return useRatis;
-  }
-
-  /**
-   * Returns hard coded 3 as replication factor.
-   * @return 3
-   */
-  public  OzoneProtos.ReplicationFactor getFactor() {
-    if(isUseRatis()) {
-      return OzoneProtos.ReplicationFactor.THREE;
-    }
-    return OzoneProtos.ReplicationFactor.ONE;
-  }
-
-  /**
-   * Returns the default replication type.
-   * @return Ratis or Standalone
-   */
-  public OzoneProtos.ReplicationType getType() {
-    // TODO : Fix me and make Ratis default before release.
-    // TODO: Remove this as replication factor and type are pipeline properties
-    if(isUseRatis()) {
-      return OzoneProtos.ReplicationType.RATIS;
-    }
-    return OzoneProtos.ReplicationType.STAND_ALONE;
-  }
-
-  /**
-   * Get xceiver client metric.
-   */
-  public synchronized static XceiverClientMetrics getXceiverClientMetrics() {
-    if (metrics == null) {
-      metrics = XceiverClientMetrics.create();
-    }
-
-    return metrics;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientMetrics.java
deleted file mode 100644
index 6359db1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientMetrics.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.annotation.Metrics;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MetricsRegistry;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-import org.apache.hadoop.metrics2.lib.MutableRate;
-
-/**
- * The client metrics for the Storage Container protocol.
- */
-@InterfaceAudience.Private
-@Metrics(about = "Storage Container Client Metrics", context = "dfs")
-public class XceiverClientMetrics {
-  public static final String SOURCE_NAME = XceiverClientMetrics.class
-      .getSimpleName();
-
-  private @Metric MutableCounterLong pendingOps;
-  private MutableCounterLong[] pendingOpsArray;
-  private MutableRate[] containerOpsLatency;
-  private MetricsRegistry registry;
-
-  public XceiverClientMetrics() {
-    int numEnumEntries = ContainerProtos.Type.values().length;
-    this.registry = new MetricsRegistry(SOURCE_NAME);
-
-    this.pendingOpsArray = new MutableCounterLong[numEnumEntries];
-    this.containerOpsLatency = new MutableRate[numEnumEntries];
-    for (int i = 0; i < numEnumEntries; i++) {
-      pendingOpsArray[i] = registry.newCounter(
-          "numPending" + ContainerProtos.Type.valueOf(i + 1),
-          "number of pending" + ContainerProtos.Type.valueOf(i + 1) + " ops",
-          (long) 0);
-
-      containerOpsLatency[i] = registry.newRate(
-          ContainerProtos.Type.valueOf(i + 1) + "Latency",
-          "latency of " + ContainerProtos.Type.valueOf(i + 1)
-          + " ops");
-    }
-  }
-
-  public static XceiverClientMetrics create() {
-    MetricsSystem ms = DefaultMetricsSystem.instance();
-    return ms.register(SOURCE_NAME, "Storage Container Client Metrics",
-        new XceiverClientMetrics());
-  }
-
-  public void incrPendingContainerOpsMetrics(ContainerProtos.Type type) {
-    pendingOps.incr();
-    pendingOpsArray[type.ordinal()].incr();
-  }
-
-  public void decrPendingContainerOpsMetrics(ContainerProtos.Type type) {
-    pendingOps.incr(-1);
-    pendingOpsArray[type.ordinal()].incr(-1);
-  }
-
-  public void addContainerOpsLatency(ContainerProtos.Type type,
-      long latencyNanos) {
-    containerOpsLatency[type.ordinal()].add(latencyNanos);
-  }
-
-  public long getContainerOpsMetrics(ContainerProtos.Type type) {
-    return pendingOpsArray[type.ordinal()].value();
-  }
-
-  public void unRegister() {
-    MetricsSystem ms = DefaultMetricsSystem.instance();
-    ms.unregisterSource(SOURCE_NAME);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientRatis.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientRatis.java
deleted file mode 100644
index 3bc70ed..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientRatis.java
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.scm;
-
-import com.google.common.base.Preconditions;
-import com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandResponseProto;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.ratis.RatisHelper;
-import org.apache.ratis.client.RaftClient;
-import org.apache.ratis.protocol.RaftClientReply;
-import org.apache.ratis.protocol.RaftGroup;
-import org.apache.ratis.protocol.RaftPeer;
-import org.apache.ratis.rpc.RpcType;
-import org.apache.ratis.rpc.SupportedRpcType;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
-import org.apache.ratis.shaded.com.google.protobuf.ShadedProtoUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Objects;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.atomic.AtomicReference;
-
-/**
- * An abstract implementation of {@link XceiverClientSpi} using Ratis.
- * The underlying RPC mechanism can be chosen via the constructor.
- */
-public final class XceiverClientRatis extends XceiverClientSpi {
-  static final Logger LOG = LoggerFactory.getLogger(XceiverClientRatis.class);
-
-  public static XceiverClientRatis newXceiverClientRatis(
-      Pipeline pipeline, Configuration ozoneConf) {
-    final String rpcType = ozoneConf.get(
-        ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
-        ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT);
-    final int maxOutstandingRequests =
-        OzoneClientUtils.getMaxOutstandingRequests(ozoneConf);
-    return new XceiverClientRatis(pipeline,
-        SupportedRpcType.valueOfIgnoreCase(rpcType), maxOutstandingRequests);
-  }
-
-  private final Pipeline pipeline;
-  private final RpcType rpcType;
-  private final AtomicReference<RaftClient> client = new AtomicReference<>();
-  private final int maxOutstandingRequests;
-
-  /**
-   * Constructs a client.
-   */
-  private XceiverClientRatis(Pipeline pipeline, RpcType rpcType,
-      int maxOutStandingChunks) {
-    super();
-    this.pipeline = pipeline;
-    this.rpcType = rpcType;
-    this.maxOutstandingRequests = maxOutStandingChunks;
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  public void createPipeline(String clusterId, List<DatanodeID> datanodes)
-      throws IOException {
-    RaftGroup group = RatisHelper.newRaftGroup(datanodes);
-    LOG.debug("initializing pipeline:{} with nodes:{}", clusterId,
-        group.getPeers());
-    reinitialize(datanodes, group);
-  }
-
-  /**
-   * Returns Ratis as pipeline Type.
-   *
-   * @return - Ratis
-   */
-  @Override
-  public OzoneProtos.ReplicationType getPipelineType() {
-    return OzoneProtos.ReplicationType.RATIS;
-  }
-
-  private void reinitialize(List<DatanodeID> datanodes, RaftGroup group)
-      throws IOException {
-    if (datanodes.isEmpty()) {
-      return;
-    }
-
-    IOException exception = null;
-    for (DatanodeID d : datanodes) {
-      try {
-        reinitialize(d, group);
-      } catch (IOException ioe) {
-        if (exception == null) {
-          exception = new IOException(
-              "Failed to reinitialize some of the RaftPeer(s)", ioe);
-        } else {
-          exception.addSuppressed(ioe);
-        }
-      }
-    }
-    if (exception != null) {
-      throw exception;
-    }
-  }
-
-  /**
-   * Adds a new peers to the Ratis Ring.
-   *
-   * @param datanode - new datanode
-   * @param group    - Raft group
-   * @throws IOException - on Failure.
-   */
-  private void reinitialize(DatanodeID datanode, RaftGroup group)
-      throws IOException {
-    final RaftPeer p = RatisHelper.toRaftPeer(datanode);
-    try (RaftClient client = RatisHelper.newRaftClient(rpcType, p)) {
-      client.reinitialize(group, p.getId());
-    } catch (IOException ioe) {
-      LOG.error("Failed to reinitialize RaftPeer:{} datanode: {}  ",
-          p, datanode, ioe);
-      throw new IOException("Failed to reinitialize RaftPeer " + p
-          + "(datanode=" + datanode + ")", ioe);
-    }
-  }
-
-  @Override
-  public Pipeline getPipeline() {
-    return pipeline;
-  }
-
-  @Override
-  public void connect() throws Exception {
-    LOG.debug("Connecting to pipeline:{} leader:{}",
-        getPipeline().getPipelineName(),
-        RatisHelper.toRaftPeerId(pipeline.getLeader()));
-    // TODO : XceiverClient ratis should pass the config value of
-    // maxOutstandingRequests so as to set the upper bound on max no of async
-    // requests to be handled by raft client
-    if (!client.compareAndSet(null,
-        RatisHelper.newRaftClient(rpcType, getPipeline()))) {
-      throw new IllegalStateException("Client is already connected.");
-    }
-  }
-
-  @Override
-  public void close() {
-    final RaftClient c = client.getAndSet(null);
-    if (c != null) {
-      try {
-        c.close();
-      } catch (IOException e) {
-        throw new IllegalStateException(e);
-      }
-    }
-  }
-
-  private RaftClient getClient() {
-    return Objects.requireNonNull(client.get(), "client is null");
-  }
-
-  private boolean isReadOnly(ContainerCommandRequestProto proto) {
-    switch (proto.getCmdType()) {
-    case ReadContainer:
-    case ReadChunk:
-    case ListKey:
-    case GetKey:
-    case GetSmallFile:
-    case ListContainer:
-    case ListChunk:
-      return true;
-    case CloseContainer:
-    case WriteChunk:
-    case UpdateContainer:
-    case CompactChunk:
-    case CreateContainer:
-    case DeleteChunk:
-    case DeleteContainer:
-    case DeleteKey:
-    case PutKey:
-    case PutSmallFile:
-    default:
-      return false;
-    }
-  }
-
-  private RaftClientReply sendRequest(ContainerCommandRequestProto request)
-      throws IOException {
-    boolean isReadOnlyRequest = isReadOnly(request);
-    ByteString byteString =
-        ShadedProtoUtil.asShadedByteString(request.toByteArray());
-    LOG.debug("sendCommand {} {}", isReadOnlyRequest, request);
-    final RaftClientReply reply =  isReadOnlyRequest ?
-        getClient().sendReadOnly(() -> byteString) :
-        getClient().send(() -> byteString);
-    LOG.debug("reply {} {}", isReadOnlyRequest, reply);
-    return reply;
-  }
-
-  private CompletableFuture<RaftClientReply> sendRequestAsync(
-      ContainerCommandRequestProto request) throws IOException {
-    boolean isReadOnlyRequest = isReadOnly(request);
-    ByteString byteString =
-        ShadedProtoUtil.asShadedByteString(request.toByteArray());
-    LOG.debug("sendCommandAsync {} {}", isReadOnlyRequest, request);
-    return isReadOnlyRequest ? getClient().sendReadOnlyAsync(() -> byteString) :
-        getClient().sendAsync(() -> byteString);
-  }
-
-  @Override
-  public ContainerCommandResponseProto sendCommand(
-      ContainerCommandRequestProto request) throws IOException {
-    final RaftClientReply reply = sendRequest(request);
-    Preconditions.checkState(reply.isSuccess());
-    return ContainerCommandResponseProto.parseFrom(
-        ShadedProtoUtil.asByteString(reply.getMessage().getContent()));
-  }
-
-  /**
-   * Sends a given command to server gets a waitable future back.
-   *
-   * @param request Request
-   * @return Response to the command
-   * @throws IOException
-   */
-  @Override
-  public CompletableFuture<ContainerCommandResponseProto> sendCommandAsync(
-      ContainerCommandRequestProto request)
-      throws IOException, ExecutionException, InterruptedException {
-    return sendRequestAsync(request).whenComplete((reply, e) ->
-          LOG.debug("received reply {} for request: {} exception: {}", request,
-              reply, e))
-        .thenApply(reply -> {
-          try {
-            return ContainerCommandResponseProto.parseFrom(
-                ShadedProtoUtil.asByteString(reply.getMessage().getContent()));
-          } catch (InvalidProtocolBufferException e) {
-            throw new CompletionException(e);
-          }
-        });
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientSpi.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientSpi.java
deleted file mode 100644
index 3711a51..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientSpi.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.scm;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .ContainerCommandRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .ContainerCommandResponseProto;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * A Client for the storageContainer protocol.
- */
-public abstract class XceiverClientSpi implements Closeable {
-
-  final private AtomicInteger referenceCount;
-  private boolean isEvicted;
-
-  XceiverClientSpi() {
-    this.referenceCount = new AtomicInteger(0);
-    this.isEvicted = false;
-  }
-
-  void incrementReference() {
-    this.referenceCount.incrementAndGet();
-  }
-
-  void decrementReference() {
-    this.referenceCount.decrementAndGet();
-    cleanup();
-  }
-
-  void setEvicted() {
-    isEvicted = true;
-    cleanup();
-  }
-
-  // close the xceiverClient only if,
-  // 1) there is no refcount on the client
-  // 2) it has been evicted from the cache.
-  private void cleanup() {
-    if (referenceCount.get() == 0 && isEvicted) {
-      close();
-    }
-  }
-
-  @VisibleForTesting
-  public int getRefcount() {
-    return referenceCount.get();
-  }
-
-  /**
-   * Connects to the leader in the pipeline.
-   */
-  public abstract void connect() throws Exception;
-
-  @Override
-  public abstract void close();
-
-  /**
-   * Returns the pipeline of machines that host the container used by this
-   * client.
-   *
-   * @return pipeline of machines that host the container
-   */
-  public abstract Pipeline getPipeline();
-
-  /**
-   * Sends a given command to server and gets the reply back.
-   * @param request Request
-   * @return Response to the command
-   * @throws IOException
-   */
-  public abstract ContainerCommandResponseProto sendCommand(
-      ContainerCommandRequestProto request) throws IOException;
-
-  /**
-   * Sends a given command to server gets a waitable future back.
-   *
-   * @param request Request
-   * @return Response to the command
-   * @throws IOException
-   */
-  public abstract CompletableFuture<ContainerCommandResponseProto>
-      sendCommandAsync(ContainerCommandRequestProto request)
-      throws IOException, ExecutionException, InterruptedException;
-
-  /**
-   * Create a pipeline.
-   *
-   * @param pipelineID - Name of the pipeline.
-   * @param datanodes - Datanodes
-   */
-  public abstract void createPipeline(String pipelineID,
-      List<DatanodeID> datanodes) throws IOException;
-
-  /**
-   * Returns pipeline Type.
-   *
-   * @return - {Stand_Alone, Ratis or Chained}
-   */
-  public abstract OzoneProtos.ReplicationType getPipelineType();
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[27/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java
deleted file mode 100644
index 957a6d9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java
+++ /dev/null
@@ -1,315 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.BucketInfo;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.util.Time;
-import org.iq80.leveldb.DBException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * KSM bucket manager.
- */
-public class BucketManagerImpl implements BucketManager {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(BucketManagerImpl.class);
-
-  /**
-   * KSMMetadataManager is used for accessing KSM MetadataDB and ReadWriteLock.
-   */
-  private final KSMMetadataManager metadataManager;
-
-  /**
-   * Constructs BucketManager.
-   * @param metadataManager
-   */
-  public BucketManagerImpl(KSMMetadataManager metadataManager){
-    this.metadataManager = metadataManager;
-  }
-
-  /**
-   * MetadataDB is maintained in MetadataManager and shared between
-   * BucketManager and VolumeManager. (and also by KeyManager)
-   *
-   * BucketManager uses MetadataDB to store bucket level information.
-   *
-   * Keys used in BucketManager for storing data into MetadataDB
-   * for BucketInfo:
-   * {volume/bucket} -> bucketInfo
-   *
-   * Work flow of create bucket:
-   *
-   * -> Check if the Volume exists in metadataDB, if not throw
-   * VolumeNotFoundException.
-   * -> Else check if the Bucket exists in metadataDB, if so throw
-   * BucketExistException
-   * -> Else update MetadataDB with VolumeInfo.
-   */
-
-  /**
-   * Creates a bucket.
-   * @param bucketInfo - KsmBucketInfo.
-   */
-  @Override
-  public void createBucket(KsmBucketInfo bucketInfo) throws IOException {
-    Preconditions.checkNotNull(bucketInfo);
-    metadataManager.writeLock().lock();
-    String volumeName = bucketInfo.getVolumeName();
-    String bucketName = bucketInfo.getBucketName();
-    try {
-      byte[] volumeKey = metadataManager.getVolumeKey(volumeName);
-      byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
-
-      //Check if the volume exists
-      if (metadataManager.get(volumeKey) == null) {
-        LOG.debug("volume: {} not found ", volumeName);
-        throw new KSMException("Volume doesn't exist",
-            KSMException.ResultCodes.FAILED_VOLUME_NOT_FOUND);
-      }
-      //Check if bucket already exists
-      if (metadataManager.get(bucketKey) != null) {
-        LOG.debug("bucket: {} already exists ", bucketName);
-        throw new KSMException("Bucket already exist",
-            KSMException.ResultCodes.FAILED_BUCKET_ALREADY_EXISTS);
-      }
-
-      KsmBucketInfo ksmBucketInfo = KsmBucketInfo.newBuilder()
-          .setVolumeName(bucketInfo.getVolumeName())
-          .setBucketName(bucketInfo.getBucketName())
-          .setAcls(bucketInfo.getAcls())
-          .setStorageType(bucketInfo.getStorageType())
-          .setIsVersionEnabled(bucketInfo.getIsVersionEnabled())
-          .setCreationTime(Time.now())
-          .build();
-      metadataManager.put(bucketKey, ksmBucketInfo.getProtobuf().toByteArray());
-
-      LOG.debug("created bucket: {} in volume: {}", bucketName, volumeName);
-    } catch (IOException | DBException ex) {
-      if (!(ex instanceof KSMException)) {
-        LOG.error("Bucket creation failed for bucket:{} in volume:{}",
-            bucketName, volumeName, ex);
-      }
-      throw ex;
-    } finally {
-      metadataManager.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Returns Bucket Information.
-   *
-   * @param volumeName - Name of the Volume.
-   * @param bucketName - Name of the Bucket.
-   */
-  @Override
-  public KsmBucketInfo getBucketInfo(String volumeName, String bucketName)
-      throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(bucketName);
-    metadataManager.readLock().lock();
-    try {
-      byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
-      byte[] value = metadataManager.get(bucketKey);
-      if (value == null) {
-        LOG.debug("bucket: {} not found in volume: {}.", bucketName,
-            volumeName);
-        throw new KSMException("Bucket not found",
-            KSMException.ResultCodes.FAILED_BUCKET_NOT_FOUND);
-      }
-      return KsmBucketInfo.getFromProtobuf(BucketInfo.parseFrom(value));
-    } catch (IOException | DBException ex) {
-      if (!(ex instanceof KSMException)) {
-        LOG.error("Exception while getting bucket info for bucket: {}",
-            bucketName, ex);
-      }
-      throw ex;
-    } finally {
-      metadataManager.readLock().unlock();
-    }
-  }
-
-  /**
-   * Sets bucket property from args.
-   * @param args - BucketArgs.
-   * @throws IOException
-   */
-  @Override
-  public void setBucketProperty(KsmBucketArgs args) throws IOException {
-    Preconditions.checkNotNull(args);
-    metadataManager.writeLock().lock();
-    String volumeName = args.getVolumeName();
-    String bucketName = args.getBucketName();
-    try {
-      byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
-      //Check if volume exists
-      if(metadataManager.get(metadataManager.getVolumeKey(volumeName)) ==
-          null) {
-        LOG.debug("volume: {} not found ", volumeName);
-        throw new KSMException("Volume doesn't exist",
-            KSMException.ResultCodes.FAILED_VOLUME_NOT_FOUND);
-      }
-      byte[] value = metadataManager.get(bucketKey);
-      //Check if bucket exist
-      if(value == null) {
-        LOG.debug("bucket: {} not found ", bucketName);
-        throw new KSMException("Bucket doesn't exist",
-            KSMException.ResultCodes.FAILED_BUCKET_NOT_FOUND);
-      }
-      KsmBucketInfo oldBucketInfo = KsmBucketInfo.getFromProtobuf(
-          BucketInfo.parseFrom(value));
-      KsmBucketInfo.Builder bucketInfoBuilder = KsmBucketInfo.newBuilder();
-      bucketInfoBuilder.setVolumeName(oldBucketInfo.getVolumeName())
-          .setBucketName(oldBucketInfo.getBucketName());
-
-      //Check ACLs to update
-      if(args.getAddAcls() != null || args.getRemoveAcls() != null) {
-        bucketInfoBuilder.setAcls(getUpdatedAclList(oldBucketInfo.getAcls(),
-            args.getRemoveAcls(), args.getAddAcls()));
-        LOG.debug("Updating ACLs for bucket: {} in volume: {}",
-            bucketName, volumeName);
-      } else {
-        bucketInfoBuilder.setAcls(oldBucketInfo.getAcls());
-      }
-
-      //Check StorageType to update
-      StorageType storageType = args.getStorageType();
-      if (storageType != null) {
-        bucketInfoBuilder.setStorageType(storageType);
-        LOG.debug("Updating bucket storage type for bucket: {} in volume: {}",
-            bucketName, volumeName);
-      } else {
-        bucketInfoBuilder.setStorageType(oldBucketInfo.getStorageType());
-      }
-
-      //Check Versioning to update
-      Boolean versioning = args.getIsVersionEnabled();
-      if (versioning != null) {
-        bucketInfoBuilder.setIsVersionEnabled(versioning);
-        LOG.debug("Updating bucket versioning for bucket: {} in volume: {}",
-            bucketName, volumeName);
-      } else {
-        bucketInfoBuilder
-            .setIsVersionEnabled(oldBucketInfo.getIsVersionEnabled());
-      }
-      bucketInfoBuilder.setCreationTime(oldBucketInfo.getCreationTime());
-
-      metadataManager.put(bucketKey,
-          bucketInfoBuilder.build().getProtobuf().toByteArray());
-    } catch (IOException | DBException ex) {
-      if (!(ex instanceof KSMException)) {
-        LOG.error("Setting bucket property failed for bucket:{} in volume:{}",
-            bucketName, volumeName, ex);
-      }
-      throw ex;
-    } finally {
-      metadataManager.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Updates the existing ACL list with remove and add ACLs that are passed.
-   * Remove is done before Add.
-   *
-   * @param existingAcls - old ACL list.
-   * @param removeAcls - ACLs to be removed.
-   * @param addAcls - ACLs to be added.
-   * @return updated ACL list.
-   */
-  private List<OzoneAcl> getUpdatedAclList(List<OzoneAcl> existingAcls,
-      List<OzoneAcl> removeAcls, List<OzoneAcl> addAcls) {
-    if(removeAcls != null && !removeAcls.isEmpty()) {
-      existingAcls.removeAll(removeAcls);
-    }
-    if(addAcls != null && !addAcls.isEmpty()) {
-      addAcls.stream().filter(acl -> !existingAcls.contains(acl)).forEach(
-          existingAcls::add);
-    }
-    return existingAcls;
-  }
-
-  /**
-   * Deletes an existing empty bucket from volume.
-   * @param volumeName - Name of the volume.
-   * @param bucketName - Name of the bucket.
-   * @throws IOException
-   */
-  public void deleteBucket(String volumeName, String bucketName)
-      throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(bucketName);
-    metadataManager.writeLock().lock();
-    try {
-      byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
-      //Check if volume exists
-      if (metadataManager.get(metadataManager.getVolumeKey(volumeName))
-          == null) {
-        LOG.debug("volume: {} not found ", volumeName);
-        throw new KSMException("Volume doesn't exist",
-            KSMException.ResultCodes.FAILED_VOLUME_NOT_FOUND);
-      }
-      //Check if bucket exist
-      if (metadataManager.get(bucketKey) == null) {
-        LOG.debug("bucket: {} not found ", bucketName);
-        throw new KSMException("Bucket doesn't exist",
-            KSMException.ResultCodes.FAILED_BUCKET_NOT_FOUND);
-      }
-      //Check if bucket is empty
-      if (!metadataManager.isBucketEmpty(volumeName, bucketName)) {
-        LOG.debug("bucket: {} is not empty ", bucketName);
-        throw new KSMException("Bucket is not empty",
-            KSMException.ResultCodes.FAILED_BUCKET_NOT_EMPTY);
-      }
-      metadataManager.delete(bucketKey);
-    } catch (IOException ex) {
-      if (!(ex instanceof KSMException)) {
-        LOG.error("Delete bucket failed for bucket:{} in volume:{}", bucketName,
-            volumeName, ex);
-      }
-      throw ex;
-    } finally {
-      metadataManager.writeLock().unlock();
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public List<KsmBucketInfo> listBuckets(String volumeName,
-      String startBucket, String bucketPrefix, int maxNumOfBuckets)
-      throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    metadataManager.readLock().lock();
-    try {
-      return metadataManager.listBuckets(
-          volumeName, startBucket, bucketPrefix, maxNumOfBuckets);
-    } finally {
-      metadataManager.readLock().unlock();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMXBean.java
deleted file mode 100644
index 42331f6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMXBean.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.jmx.ServiceRuntimeInfo;
-
-/**
- * This is the JMX management interface for ksm information.
- */
-@InterfaceAudience.Private
-public interface KSMMXBean extends ServiceRuntimeInfo {
-
-  String getRpcPort();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManager.java
deleted file mode 100644
index f5a2d5b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManager.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.utils.BatchOperation;
-import org.apache.hadoop.utils.MetadataStore;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.locks.Lock;
-
-/**
- * KSM metadata manager interface.
- */
-public interface KSMMetadataManager {
-  /**
-   * Start metadata manager.
-   */
-  void start();
-
-  /**
-   * Stop metadata manager.
-   */
-  void stop() throws IOException;
-
-  /**
-   * Get metadata store.
-   * @return metadata store.
-   */
-  @VisibleForTesting
-  MetadataStore getStore();
-
-  /**
-   * Returns the read lock used on Metadata DB.
-   * @return readLock
-   */
-  Lock readLock();
-
-  /**
-   * Returns the write lock used on Metadata DB.
-   * @return writeLock
-   */
-  Lock writeLock();
-
-  /**
-   * Returns the value associated with this key.
-   * @param key - key
-   * @return value
-   */
-  byte[] get(byte[] key) throws IOException;
-
-  /**
-   * Puts a Key into Metadata DB.
-   * @param key   - key
-   * @param value - value
-   */
-  void put(byte[] key, byte[] value) throws IOException;
-
-  /**
-   * Deletes a Key from Metadata DB.
-   * @param key   - key
-   */
-  void delete(byte[] key) throws IOException;
-
-  /**
-   * Atomic write a batch of operations.
-   * @param batch
-   * @throws IOException
-   */
-  void writeBatch(BatchOperation batch) throws IOException;
-
-  /**
-   * Given a volume return the corresponding DB key.
-   * @param volume - Volume name
-   */
-  byte[] getVolumeKey(String volume);
-
-  /**
-   * Given a user return the corresponding DB key.
-   * @param user - User name
-   */
-  byte[] getUserKey(String user);
-
-  /**
-   * Given a volume and bucket, return the corresponding DB key.
-   * @param volume - User name
-   * @param bucket - Bucket name
-   */
-  byte[] getBucketKey(String volume, String bucket);
-
-  /**
-   * Given a volume, bucket and a key, return the corresponding DB key.
-   * @param volume - volume name
-   * @param bucket - bucket name
-   * @param key - key name
-   * @return bytes of DB key.
-   */
-  byte[] getDBKeyBytes(String volume, String bucket, String key);
-
-  /**
-   * Returns the DB key name of a deleted key in KSM metadata store.
-   * The name for a deleted key has prefix #deleting# followed by
-   * the actual key name.
-   * @param keyName - key name
-   * @return bytes of DB key.
-   */
-  byte[] getDeletedKeyName(byte[] keyName);
-
-  /**
-   * Returns the DB key name of a open key in KSM metadata store.
-   * Should be #open# prefix followed by actual key name.
-   * @param keyName - key name
-   * @param id - the id for this open
-   * @return bytes of DB key.
-   */
-  byte[] getOpenKeyNameBytes(String keyName, int id);
-
-  /**
-   * Returns the full name of a key given volume name, bucket name and key name.
-   * Generally done by padding certain delimiters.
-   *
-   * @param volumeName - volume name
-   * @param bucketName - bucket name
-   * @param keyName - key name
-   * @return the full key name.
-   */
-  String getKeyWithDBPrefix(String volumeName, String bucketName,
-      String keyName);
-
-  /**
-   * Given a volume, check if it is empty,
-   * i.e there are no buckets inside it.
-   * @param volume - Volume name
-   */
-  boolean isVolumeEmpty(String volume) throws IOException;
-
-  /**
-   * Given a volume/bucket, check if it is empty,
-   * i.e there are no keys inside it.
-   * @param volume - Volume name
-   * @param  bucket - Bucket name
-   * @return true if the bucket is empty
-   */
-  boolean isBucketEmpty(String volume, String bucket) throws IOException;
-
-  /**
-   * Returns a list of buckets represented by {@link KsmBucketInfo}
-   * in the given volume.
-   *
-   * @param volumeName
-   *   the name of the volume. This argument is required,
-   *   this method returns buckets in this given volume.
-   * @param startBucket
-   *   the start bucket name. Only the buckets whose name is
-   *   after this value will be included in the result.
-   *   This key is excluded from the result.
-   * @param bucketPrefix
-   *   bucket name prefix. Only the buckets whose name has
-   *   this prefix will be included in the result.
-   * @param maxNumOfBuckets
-   *   the maximum number of buckets to return. It ensures
-   *   the size of the result will not exceed this limit.
-   * @return a list of buckets.
-   * @throws IOException
-   */
-  List<KsmBucketInfo> listBuckets(String volumeName, String startBucket,
-      String bucketPrefix, int maxNumOfBuckets) throws IOException;
-
-  /**
-   * Returns a list of keys represented by {@link KsmKeyInfo}
-   * in the given bucket.
-   *
-   * @param volumeName
-   *   the name of the volume.
-   * @param bucketName
-   *   the name of the bucket.
-   * @param startKey
-   *   the start key name, only the keys whose name is
-   *   after this value will be included in the result.
-   *   This key is excluded from the result.
-   * @param keyPrefix
-   *   key name prefix, only the keys whose name has
-   *   this prefix will be included in the result.
-   * @param maxKeys
-   *   the maximum number of keys to return. It ensures
-   *   the size of the result will not exceed this limit.
-   * @return a list of keys.
-   * @throws IOException
-   */
-  List<KsmKeyInfo> listKeys(String volumeName,
-      String bucketName, String startKey, String keyPrefix, int maxKeys)
-      throws IOException;
-
-  /**
-   * Returns a list of volumes owned by a given user; if user is null,
-   * returns all volumes.
-   *
-   * @param userName
-   *   volume owner
-   * @param prefix
-   *   the volume prefix used to filter the listing result.
-   * @param startKey
-   *   the start volume name determines where to start listing from,
-   *   this key is excluded from the result.
-   * @param maxKeys
-   *   the maximum number of volumes to return.
-   * @return a list of {@link KsmVolumeArgs}
-   * @throws IOException
-   */
-  List<KsmVolumeArgs> listVolumes(String userName, String prefix,
-      String startKey, int maxKeys) throws IOException;
-
-  /**
-   * Returns a list of pending deletion key info that ups to the given count.
-   * Each entry is a {@link BlockGroup}, which contains the info about the
-   * key name and all its associated block IDs. A pending deletion key is
-   * stored with #deleting# prefix in KSM DB.
-   *
-   * @param count max number of keys to return.
-   * @return a list of {@link BlockGroup} represent keys and blocks.
-   * @throws IOException
-   */
-  List<BlockGroup> getPendingDeletionKeys(int count) throws IOException;
-
-  /**
-   * Returns a list of all still open key info. Which contains the info about
-   * the key name and all its associated block IDs. A pending open key has
-   * prefix #open# in KSM DB.
-   *
-   * @return a list of {@link BlockGroup} representing keys and blocks.
-   * @throws IOException
-   */
-  List<BlockGroup> getExpiredOpenKeys() throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManagerImpl.java
deleted file mode 100644
index df02182..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManagerImpl.java
+++ /dev/null
@@ -1,524 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfoGroup;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException.ResultCodes;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.BucketInfo;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.VolumeInfo;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.VolumeList;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.BatchOperation;
-import org.apache.hadoop.utils.MetadataKeyFilters;
-import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter;
-import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConsts.DELETING_KEY_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.KSM_DB_NAME;
-import static org.apache.hadoop.ozone.OzoneConsts.OPEN_KEY_ID_DELIMINATOR;
-import static org.apache.hadoop.ozone.OzoneConsts.OPEN_KEY_PREFIX;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
-    .OZONE_KSM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
-    .OZONE_KSM_DB_CACHE_SIZE_MB;
-
-/**
- * KSM metadata manager interface.
- */
-public class KSMMetadataManagerImpl implements KSMMetadataManager {
-
-  private final MetadataStore store;
-  private final ReadWriteLock lock;
-  private final long openKeyExpireThresholdMS;
-
-  public KSMMetadataManagerImpl(OzoneConfiguration conf) throws IOException {
-    File metaDir = OzoneUtils.getOzoneMetaDirPath(conf);
-    final int cacheSize = conf.getInt(OZONE_KSM_DB_CACHE_SIZE_MB,
-        OZONE_KSM_DB_CACHE_SIZE_DEFAULT);
-    File ksmDBFile = new File(metaDir.getPath(), KSM_DB_NAME);
-    this.store = MetadataStoreBuilder.newBuilder()
-        .setConf(conf)
-        .setDbFile(ksmDBFile)
-        .setCacheSize(cacheSize * OzoneConsts.MB)
-        .build();
-    this.lock = new ReentrantReadWriteLock();
-    this.openKeyExpireThresholdMS = 1000 * conf.getInt(
-        OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS,
-        OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT);
-  }
-
-  /**
-   * Start metadata manager.
-   */
-  @Override
-  public void start() {
-
-  }
-
-  /**
-   * Stop metadata manager.
-   */
-  @Override
-  public void stop() throws IOException {
-    if (store != null) {
-      store.close();
-    }
-  }
-
-  /**
-   * Get metadata store.
-   * @return store - metadata store.
-   */
-  @VisibleForTesting
-  @Override
-  public MetadataStore getStore() {
-    return store;
-  }
-
-  /**
-   * Given a volume return the corresponding DB key.
-   * @param volume - Volume name
-   */
-  public byte[] getVolumeKey(String volume) {
-    String dbVolumeName = OzoneConsts.KSM_VOLUME_PREFIX + volume;
-    return DFSUtil.string2Bytes(dbVolumeName);
-  }
-
-  /**
-   * Given a user return the corresponding DB key.
-   * @param user - User name
-   */
-  public byte[] getUserKey(String user) {
-    String dbUserName = OzoneConsts.KSM_USER_PREFIX + user;
-    return DFSUtil.string2Bytes(dbUserName);
-  }
-
-  /**
-   * Given a volume and bucket, return the corresponding DB key.
-   * @param volume - User name
-   * @param bucket - Bucket name
-   */
-  public byte[] getBucketKey(String volume, String bucket) {
-    String bucketKeyString = OzoneConsts.KSM_VOLUME_PREFIX + volume
-        + OzoneConsts.KSM_BUCKET_PREFIX + bucket;
-    return DFSUtil.string2Bytes(bucketKeyString);
-  }
-
-  /**
-   * @param volume
-   * @param bucket
-   * @return
-   */
-  private String getBucketWithDBPrefix(String volume, String bucket) {
-    StringBuffer sb = new StringBuffer();
-    sb.append(OzoneConsts.KSM_VOLUME_PREFIX)
-        .append(volume)
-        .append(OzoneConsts.KSM_BUCKET_PREFIX);
-    if (!Strings.isNullOrEmpty(bucket)) {
-      sb.append(bucket);
-    }
-    return sb.toString();
-  }
-
-  @Override
-  public String getKeyWithDBPrefix(String volume, String bucket, String key) {
-    String keyVB = OzoneConsts.KSM_KEY_PREFIX + volume
-        + OzoneConsts.KSM_KEY_PREFIX + bucket
-        + OzoneConsts.KSM_KEY_PREFIX;
-    return Strings.isNullOrEmpty(key) ? keyVB : keyVB + key;
-  }
-
-  @Override
-  public byte[] getDBKeyBytes(String volume, String bucket, String key) {
-    return DFSUtil.string2Bytes(getKeyWithDBPrefix(volume, bucket, key));
-  }
-
-  @Override
-  public byte[] getDeletedKeyName(byte[] keyName) {
-    return DFSUtil.string2Bytes(
-        DELETING_KEY_PREFIX + DFSUtil.bytes2String(keyName));
-  }
-
-  @Override
-  public byte[] getOpenKeyNameBytes(String keyName, int id) {
-    return DFSUtil.string2Bytes(OPEN_KEY_PREFIX + id +
-        OPEN_KEY_ID_DELIMINATOR + keyName);
-  }
-
-  /**
-   * Returns the read lock used on Metadata DB.
-   * @return readLock
-   */
-  @Override
-  public Lock readLock() {
-    return lock.readLock();
-  }
-
-  /**
-   * Returns the write lock used on Metadata DB.
-   * @return writeLock
-   */
-  @Override
-  public Lock writeLock() {
-    return lock.writeLock();
-  }
-
-  /**
-   * Returns the value associated with this key.
-   * @param key - key
-   * @return value
-   */
-  @Override
-  public byte[] get(byte[] key) throws IOException {
-    return store.get(key);
-  }
-
-  /**
-   * Puts a Key into Metadata DB.
-   * @param key   - key
-   * @param value - value
-   */
-  @Override
-  public void put(byte[] key, byte[] value) throws IOException {
-    store.put(key, value);
-  }
-
-  /**
-   * Deletes a Key from Metadata DB.
-   * @param key   - key
-   */
-  public void delete(byte[] key) throws IOException {
-    store.delete(key);
-  }
-
-  @Override
-  public void writeBatch(BatchOperation batch) throws IOException {
-    this.store.writeBatch(batch);
-  }
-
-  /**
-   * Given a volume, check if it is empty, i.e there are no buckets inside it.
-   * @param volume - Volume name
-   * @return true if the volume is empty
-   */
-  public boolean isVolumeEmpty(String volume) throws IOException {
-    String dbVolumeRootName = OzoneConsts.KSM_VOLUME_PREFIX + volume
-        + OzoneConsts.KSM_BUCKET_PREFIX;
-    byte[] dbVolumeRootKey = DFSUtil.string2Bytes(dbVolumeRootName);
-    ImmutablePair<byte[], byte[]> volumeRoot =
-        store.peekAround(0, dbVolumeRootKey);
-    if (volumeRoot != null) {
-      return !DFSUtil.bytes2String(volumeRoot.getKey())
-          .startsWith(dbVolumeRootName);
-    }
-    return true;
-  }
-
-  /**
-   * Given a volume/bucket, check if it is empty,
-   * i.e there are no keys inside it.
-   * @param volume - Volume name
-   * @param bucket - Bucket name
-   * @return true if the bucket is empty
-   */
-  public boolean isBucketEmpty(String volume, String bucket)
-      throws IOException {
-    String keyRootName = getKeyWithDBPrefix(volume, bucket, null);
-    byte[] keyRoot = DFSUtil.string2Bytes(keyRootName);
-    ImmutablePair<byte[], byte[]> firstKey = store.peekAround(0, keyRoot);
-    if (firstKey != null) {
-      return !DFSUtil.bytes2String(firstKey.getKey())
-          .startsWith(keyRootName);
-    }
-    return true;
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public List<KsmBucketInfo> listBuckets(final String volumeName,
-      final String startBucket, final String bucketPrefix,
-      final int maxNumOfBuckets) throws IOException {
-    List<KsmBucketInfo> result = new ArrayList<>();
-    if (Strings.isNullOrEmpty(volumeName)) {
-      throw new KSMException("Volume name is required.",
-          ResultCodes.FAILED_VOLUME_NOT_FOUND);
-    }
-
-    byte[] volumeNameBytes = getVolumeKey(volumeName);
-    if (store.get(volumeNameBytes) == null) {
-      throw new KSMException("Volume " + volumeName + " not found.",
-          ResultCodes.FAILED_VOLUME_NOT_FOUND);
-    }
-
-
-    // A bucket starts with /#volume/#bucket_prefix
-    MetadataKeyFilter filter = (preKey, currentKey, nextKey) -> {
-      if (currentKey != null) {
-        String bucketNamePrefix =
-                getBucketWithDBPrefix(volumeName, bucketPrefix);
-        String bucket = DFSUtil.bytes2String(currentKey);
-        return bucket.startsWith(bucketNamePrefix);
-      }
-      return false;
-    };
-
-    List<Map.Entry<byte[], byte[]>> rangeResult;
-    if (!Strings.isNullOrEmpty(startBucket)) {
-      // Since we are excluding start key from the result,
-      // the maxNumOfBuckets is incremented.
-      rangeResult = store.getSequentialRangeKVs(
-          getBucketKey(volumeName, startBucket),
-          maxNumOfBuckets + 1, filter);
-      if (!rangeResult.isEmpty()) {
-        //Remove start key from result.
-        rangeResult.remove(0);
-      }
-    } else {
-      rangeResult = store.getSequentialRangeKVs(null, maxNumOfBuckets, filter);
-    }
-
-    for (Map.Entry<byte[], byte[]> entry : rangeResult) {
-      KsmBucketInfo info = KsmBucketInfo.getFromProtobuf(
-          BucketInfo.parseFrom(entry.getValue()));
-      result.add(info);
-    }
-    return result;
-  }
-
-  @Override
-  public List<KsmKeyInfo> listKeys(String volumeName, String bucketName,
-      String startKey, String keyPrefix, int maxKeys) throws IOException {
-    List<KsmKeyInfo> result = new ArrayList<>();
-    if (Strings.isNullOrEmpty(volumeName)) {
-      throw new KSMException("Volume name is required.",
-          ResultCodes.FAILED_VOLUME_NOT_FOUND);
-    }
-
-    if (Strings.isNullOrEmpty(bucketName)) {
-      throw new KSMException("Bucket name is required.",
-          ResultCodes.FAILED_BUCKET_NOT_FOUND);
-    }
-
-    byte[] bucketNameBytes = getBucketKey(volumeName, bucketName);
-    if (store.get(bucketNameBytes) == null) {
-      throw new KSMException("Bucket " + bucketName + " not found.",
-          ResultCodes.FAILED_BUCKET_NOT_FOUND);
-    }
-
-    MetadataKeyFilter filter = new KeyPrefixFilter(
-                getKeyWithDBPrefix(volumeName, bucketName, keyPrefix));
-
-    List<Map.Entry<byte[], byte[]>> rangeResult;
-    if (!Strings.isNullOrEmpty(startKey)) {
-      //Since we are excluding start key from the result,
-      // the maxNumOfBuckets is incremented.
-      rangeResult = store.getSequentialRangeKVs(
-          getDBKeyBytes(volumeName, bucketName, startKey),
-          maxKeys + 1, filter);
-      if (!rangeResult.isEmpty()) {
-        //Remove start key from result.
-        rangeResult.remove(0);
-      }
-    } else {
-      rangeResult = store.getSequentialRangeKVs(null, maxKeys, filter);
-    }
-
-    for (Map.Entry<byte[], byte[]> entry : rangeResult) {
-      KsmKeyInfo info = KsmKeyInfo.getFromProtobuf(
-          KeyInfo.parseFrom(entry.getValue()));
-      result.add(info);
-    }
-    return result;
-  }
-
-  @Override
-  public List<KsmVolumeArgs> listVolumes(String userName,
-      String prefix, String startKey, int maxKeys) throws IOException {
-    List<KsmVolumeArgs> result = Lists.newArrayList();
-    VolumeList volumes;
-    if (Strings.isNullOrEmpty(userName)) {
-      volumes = getAllVolumes();
-    } else {
-      volumes = getVolumesByUser(userName);
-    }
-
-    if (volumes == null || volumes.getVolumeNamesCount() == 0) {
-      return result;
-    }
-
-    boolean startKeyFound = Strings.isNullOrEmpty(startKey);
-    for (String volumeName : volumes.getVolumeNamesList()) {
-      if (!Strings.isNullOrEmpty(prefix)) {
-        if (!volumeName.startsWith(prefix)) {
-          continue;
-        }
-      }
-
-      if (!startKeyFound && volumeName.equals(startKey)) {
-        startKeyFound = true;
-        continue;
-      }
-      if (startKeyFound && result.size() < maxKeys) {
-        byte[] volumeInfo = store.get(this.getVolumeKey(volumeName));
-        if (volumeInfo == null) {
-          // Could not get volume info by given volume name,
-          // since the volume name is loaded from db,
-          // this probably means ksm db is corrupted or some entries are
-          // accidentally removed.
-          throw new KSMException("Volume info not found for " + volumeName,
-              ResultCodes.FAILED_VOLUME_NOT_FOUND);
-        }
-        VolumeInfo info = VolumeInfo.parseFrom(volumeInfo);
-        KsmVolumeArgs volumeArgs = KsmVolumeArgs.getFromProtobuf(info);
-        result.add(volumeArgs);
-      }
-    }
-
-    return result;
-  }
-
-  private VolumeList getVolumesByUser(String userName)
-      throws KSMException {
-    return getVolumesByUser(getUserKey(userName));
-  }
-
-  private VolumeList getVolumesByUser(byte[] userNameKey)
-      throws KSMException {
-    VolumeList volumes = null;
-    try {
-      byte[] volumesInBytes = store.get(userNameKey);
-      if (volumesInBytes == null) {
-        // No volume found for this user, return an empty list
-        return VolumeList.newBuilder().build();
-      }
-      volumes = VolumeList.parseFrom(volumesInBytes);
-    } catch (IOException e) {
-      throw new KSMException("Unable to get volumes info by the given user, "
-          + "metadata might be corrupted", e,
-          ResultCodes.FAILED_METADATA_ERROR);
-    }
-    return volumes;
-  }
-
-  private VolumeList getAllVolumes() throws IOException {
-    // Scan all users in database
-    KeyPrefixFilter filter = new KeyPrefixFilter(OzoneConsts.KSM_USER_PREFIX);
-    // We are not expecting a huge number of users per cluster,
-    // it should be fine to scan all users in db and return us a
-    // list of volume names in string per user.
-    List<Map.Entry<byte[], byte[]>> rangeKVs = store
-        .getSequentialRangeKVs(null, Integer.MAX_VALUE, filter);
-
-    VolumeList.Builder builder = VolumeList.newBuilder();
-    for (Map.Entry<byte[], byte[]> entry : rangeKVs) {
-      VolumeList volumes = this.getVolumesByUser(entry.getKey());
-      builder.addAllVolumeNames(volumes.getVolumeNamesList());
-    }
-
-    return builder.build();
-  }
-
-  @Override
-  public List<BlockGroup> getPendingDeletionKeys(final int count)
-      throws IOException {
-    List<BlockGroup> keyBlocksList = Lists.newArrayList();
-    List<Map.Entry<byte[], byte[]>> rangeResult =
-        store.getRangeKVs(null, count,
-            MetadataKeyFilters.getDeletingKeyFilter());
-    for (Map.Entry<byte[], byte[]> entry : rangeResult) {
-      KsmKeyInfo info =
-          KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(entry.getValue()));
-      // Get block keys as a list.
-      KsmKeyLocationInfoGroup latest = info.getLatestVersionLocations();
-      if (latest == null) {
-        return Collections.emptyList();
-      }
-      List<String> item = latest.getLocationList().stream()
-          .map(KsmKeyLocationInfo::getBlockID)
-          .collect(Collectors.toList());
-      BlockGroup keyBlocks = BlockGroup.newBuilder()
-          .setKeyName(DFSUtil.bytes2String(entry.getKey()))
-          .addAllBlockIDs(item)
-          .build();
-      keyBlocksList.add(keyBlocks);
-    }
-    return keyBlocksList;
-  }
-
-  @Override
-  public List<BlockGroup> getExpiredOpenKeys() throws IOException {
-    List<BlockGroup> keyBlocksList = Lists.newArrayList();
-    long now = Time.now();
-    final MetadataKeyFilter openKeyFilter =
-        new KeyPrefixFilter(OPEN_KEY_PREFIX);
-    List<Map.Entry<byte[], byte[]>> rangeResult =
-        store.getSequentialRangeKVs(null, Integer.MAX_VALUE,
-            openKeyFilter);
-    for (Map.Entry<byte[], byte[]> entry : rangeResult) {
-      KsmKeyInfo info =
-          KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(entry.getValue()));
-      long lastModify = info.getModificationTime();
-      if (now - lastModify < this.openKeyExpireThresholdMS) {
-        // consider as may still be active, not hanging.
-        continue;
-      }
-      // Get block keys as a list.
-      List<String> item = info.getLatestVersionLocations()
-          .getBlocksLatestVersionOnly().stream()
-          .map(KsmKeyLocationInfo::getBlockID)
-          .collect(Collectors.toList());
-      BlockGroup keyBlocks = BlockGroup.newBuilder()
-          .setKeyName(DFSUtil.bytes2String(entry.getKey()))
-          .addAllBlockIDs(item)
-          .build();
-      keyBlocksList.add(keyBlocks);
-    }
-    return keyBlocksList;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java
deleted file mode 100644
index bd29012..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java
+++ /dev/null
@@ -1,437 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.annotation.Metrics;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-
-/**
- * This class is for maintaining KeySpaceManager statistics.
- */
-@InterfaceAudience.Private
-@Metrics(about="Key Space Manager Metrics", context="dfs")
-public class KSMMetrics {
-  private static final String SOURCE_NAME =
-      KSMMetrics.class.getSimpleName();
-
-  // KSM request type op metrics
-  private @Metric MutableCounterLong numVolumeOps;
-  private @Metric MutableCounterLong numBucketOps;
-  private @Metric MutableCounterLong numKeyOps;
-
-  // KSM op metrics
-  private @Metric MutableCounterLong numVolumeCreates;
-  private @Metric MutableCounterLong numVolumeUpdates;
-  private @Metric MutableCounterLong numVolumeInfos;
-  private @Metric MutableCounterLong numVolumeCheckAccesses;
-  private @Metric MutableCounterLong numBucketCreates;
-  private @Metric MutableCounterLong numVolumeDeletes;
-  private @Metric MutableCounterLong numBucketInfos;
-  private @Metric MutableCounterLong numBucketUpdates;
-  private @Metric MutableCounterLong numBucketDeletes;
-  private @Metric MutableCounterLong numKeyAllocate;
-  private @Metric MutableCounterLong numKeyLookup;
-  private @Metric MutableCounterLong numKeyDeletes;
-  private @Metric MutableCounterLong numBucketLists;
-  private @Metric MutableCounterLong numKeyLists;
-  private @Metric MutableCounterLong numVolumeLists;
-  private @Metric MutableCounterLong numKeyCommits;
-  private @Metric MutableCounterLong numAllocateBlockCalls;
-  private @Metric MutableCounterLong numGetServiceLists;
-
-  // Failure Metrics
-  private @Metric MutableCounterLong numVolumeCreateFails;
-  private @Metric MutableCounterLong numVolumeUpdateFails;
-  private @Metric MutableCounterLong numVolumeInfoFails;
-  private @Metric MutableCounterLong numVolumeDeleteFails;
-  private @Metric MutableCounterLong numBucketCreateFails;
-  private @Metric MutableCounterLong numVolumeCheckAccessFails;
-  private @Metric MutableCounterLong numBucketInfoFails;
-  private @Metric MutableCounterLong numBucketUpdateFails;
-  private @Metric MutableCounterLong numBucketDeleteFails;
-  private @Metric MutableCounterLong numKeyAllocateFails;
-  private @Metric MutableCounterLong numKeyLookupFails;
-  private @Metric MutableCounterLong numKeyDeleteFails;
-  private @Metric MutableCounterLong numBucketListFails;
-  private @Metric MutableCounterLong numKeyListFails;
-  private @Metric MutableCounterLong numVolumeListFails;
-  private @Metric MutableCounterLong numKeyCommitFails;
-  private @Metric MutableCounterLong numBlockAllocateCallFails;
-  private @Metric MutableCounterLong numGetServiceListFails;
-
-  public KSMMetrics() {
-  }
-
-  public static KSMMetrics create() {
-    MetricsSystem ms = DefaultMetricsSystem.instance();
-    return ms.register(SOURCE_NAME,
-        "Key Space Manager Metrics",
-        new KSMMetrics());
-  }
-
-  public void incNumVolumeCreates() {
-    numVolumeOps.incr();
-    numVolumeCreates.incr();
-  }
-
-  public void incNumVolumeUpdates() {
-    numVolumeOps.incr();
-    numVolumeUpdates.incr();
-  }
-
-  public void incNumVolumeInfos() {
-    numVolumeOps.incr();
-    numVolumeInfos.incr();
-  }
-
-  public void incNumVolumeDeletes() {
-    numVolumeOps.incr();
-    numVolumeDeletes.incr();
-  }
-
-  public void incNumVolumeCheckAccesses() {
-    numVolumeOps.incr();
-    numVolumeCheckAccesses.incr();
-  }
-
-  public void incNumBucketCreates() {
-    numBucketOps.incr();
-    numBucketCreates.incr();
-  }
-
-  public void incNumBucketInfos() {
-    numBucketOps.incr();
-    numBucketInfos.incr();
-  }
-
-  public void incNumBucketUpdates() {
-    numBucketOps.incr();
-    numBucketUpdates.incr();
-  }
-
-  public void incNumBucketDeletes() {
-    numBucketOps.incr();
-    numBucketDeletes.incr();
-  }
-
-  public void incNumBucketLists() {
-    numBucketOps.incr();
-    numBucketLists.incr();
-  }
-
-  public void incNumKeyLists() {
-    numKeyOps.incr();
-    numKeyLists.incr();
-  }
-
-  public void incNumVolumeLists() {
-    numVolumeOps.incr();
-    numVolumeLists.incr();
-  }
-
-  public void incNumGetServiceLists() {
-    numGetServiceLists.incr();
-  }
-
-  public void incNumVolumeCreateFails() {
-    numVolumeCreateFails.incr();
-  }
-
-  public void incNumVolumeUpdateFails() {
-    numVolumeUpdateFails.incr();
-  }
-
-  public void incNumVolumeInfoFails() {
-    numVolumeInfoFails.incr();
-  }
-
-  public void incNumVolumeDeleteFails() {
-    numVolumeDeleteFails.incr();
-  }
-
-  public void incNumVolumeCheckAccessFails() {
-    numVolumeCheckAccessFails.incr();
-  }
-
-  public void incNumBucketCreateFails() {
-    numBucketCreateFails.incr();
-  }
-
-  public void incNumBucketInfoFails() {
-    numBucketInfoFails.incr();
-  }
-
-  public void incNumBucketUpdateFails() {
-    numBucketUpdateFails.incr();
-  }
-
-  public void incNumBucketDeleteFails() {
-    numBucketDeleteFails.incr();
-  }
-
-  public void incNumKeyAllocates() {
-    numKeyOps.incr();
-    numKeyAllocate.incr();
-  }
-
-  public void incNumKeyAllocateFails() {
-    numKeyAllocateFails.incr();
-  }
-
-  public void incNumKeyLookups() {
-    numKeyOps.incr();
-    numKeyLookup.incr();
-  }
-
-  public void incNumKeyLookupFails() {
-    numKeyLookupFails.incr();
-  }
-
-  public void incNumKeyDeleteFails() {
-    numKeyDeleteFails.incr();
-  }
-
-  public void incNumKeyDeletes() {
-    numKeyOps.incr();
-    numKeyDeletes.incr();
-  }
-
-  public void incNumKeyCommits() {
-    numKeyOps.incr();
-    numKeyCommits.incr();
-  }
-
-  public void incNumKeyCommitFails() {
-    numKeyCommitFails.incr();
-  }
-
-  public void incNumBlockAllocateCalls() {
-    numAllocateBlockCalls.incr();
-  }
-
-  public void incNumBlockAllocateCallFails() {
-    numBlockAllocateCallFails.incr();
-  }
-
-  public void incNumBucketListFails() {
-    numBucketListFails.incr();
-  }
-
-  public void incNumKeyListFails() {
-    numKeyListFails.incr();
-  }
-
-  public void incNumVolumeListFails() {
-    numVolumeListFails.incr();
-  }
-
-  public void incNumGetServiceListFails() {
-    numGetServiceListFails.incr();
-  }
-
-  @VisibleForTesting
-  public long getNumVolumeCreates() {
-    return numVolumeCreates.value();
-  }
-
-  @VisibleForTesting
-  public long getNumVolumeUpdates() {
-    return numVolumeUpdates.value();
-  }
-
-  @VisibleForTesting
-  public long getNumVolumeInfos() {
-    return numVolumeInfos.value();
-  }
-
-  @VisibleForTesting
-  public long getNumVolumeDeletes() {
-    return numVolumeDeletes.value();
-  }
-
-  @VisibleForTesting
-  public long getNumVolumeCheckAccesses() {
-    return numVolumeCheckAccesses.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBucketCreates() {
-    return numBucketCreates.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBucketInfos() {
-    return numBucketInfos.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBucketUpdates() {
-    return numBucketUpdates.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBucketDeletes() {
-    return numBucketDeletes.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBucketLists() {
-    return numBucketLists.value();
-  }
-
-  @VisibleForTesting
-  public long getNumVolumeLists() {
-    return numVolumeLists.value();
-  }
-
-  @VisibleForTesting
-  public long getNumKeyLists() {
-    return numKeyLists.value();
-  }
-
-  @VisibleForTesting
-  public long getNumGetServiceLists() {
-    return numGetServiceLists.value();
-  }
-
-  @VisibleForTesting
-  public long getNumVolumeCreateFails() {
-    return numVolumeCreateFails.value();
-  }
-
-  @VisibleForTesting
-  public long getNumVolumeUpdateFails() {
-    return numVolumeUpdateFails.value();
-  }
-
-  @VisibleForTesting
-  public long getNumVolumeInfoFails() {
-    return numVolumeInfoFails.value();
-  }
-
-  @VisibleForTesting
-  public long getNumVolumeDeleteFails() {
-    return numVolumeDeleteFails.value();
-  }
-
-  @VisibleForTesting
-  public long getNumVolumeCheckAccessFails() {
-    return numVolumeCheckAccessFails.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBucketCreateFails() {
-    return numBucketCreateFails.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBucketInfoFails() {
-    return numBucketInfoFails.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBucketUpdateFails() {
-    return numBucketUpdateFails.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBucketDeleteFails() {
-    return numBucketDeleteFails.value();
-  }
-
-  @VisibleForTesting
-  public long getNumKeyAllocates() {
-    return numKeyAllocate.value();
-  }
-
-  @VisibleForTesting
-  public long getNumKeyAllocateFails() {
-    return numKeyAllocateFails.value();
-  }
-
-  @VisibleForTesting
-  public long getNumKeyLookups() {
-    return numKeyLookup.value();
-  }
-
-  @VisibleForTesting
-  public long getNumKeyLookupFails() {
-    return numKeyLookupFails.value();
-  }
-
-  @VisibleForTesting
-  public long getNumKeyDeletes() {
-    return numKeyDeletes.value();
-  }
-
-  @VisibleForTesting
-  public long getNumKeyDeletesFails() {
-    return numKeyDeleteFails.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBucketListFails() {
-    return numBucketListFails.value();
-  }
-
-  @VisibleForTesting
-  public long getNumKeyListFails() {
-    return numKeyListFails.value();
-  }
-
-  @VisibleForTesting
-  public long getNumVolumeListFails() {
-    return numVolumeListFails.value();
-  }
-
-  @VisibleForTesting
-  public long getNumKeyCommits() {
-    return numKeyCommits.value();
-  }
-
-  @VisibleForTesting
-  public long getNumKeyCommitFails() {
-    return numKeyCommitFails.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBlockAllocates() {
-    return numAllocateBlockCalls.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBlockAllocateFails() {
-    return numBlockAllocateCallFails.value();
-  }
-
-  @VisibleForTesting
-  public long getNumGetServiceListFails() {
-    return numGetServiceListFails.value();
-  }
-
-  public void unRegister() {
-    MetricsSystem ms = DefaultMetricsSystem.instance();
-    ms.unregisterSource(SOURCE_NAME);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMStorage.java
deleted file mode 100644
index 8219212..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMStorage.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.common.Storage;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeType;
-import org.apache.hadoop.ozone.scm.SCMStorage;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-
-import java.io.IOException;
-import java.util.Properties;
-import java.util.UUID;
-
-/**
- * KSMStorage is responsible for management of the StorageDirectories used by
- * the KSM.
- */
-public class KSMStorage extends Storage {
-
-  public static final String STORAGE_DIR = "ksm";
-  public static final String KSM_ID = "ksmUuid";
-
-  /**
-   * Construct KSMStorage.
-   * @throws IOException if any directories are inaccessible.
-   */
-  public KSMStorage(OzoneConfiguration conf) throws IOException {
-    super(NodeType.KSM, OzoneUtils.getOzoneMetaDirPath(conf), STORAGE_DIR);
-  }
-
-  public void setScmId(String scmId) throws IOException {
-    if (getState() == StorageState.INITIALIZED) {
-      throw new IOException("KSM is already initialized.");
-    } else {
-      getStorageInfo().setProperty(SCMStorage.SCM_ID, scmId);
-    }
-  }
-
-  public void setKsmId(String ksmId) throws IOException {
-    if (getState() == StorageState.INITIALIZED) {
-      throw new IOException("KSM is already initialized.");
-    } else {
-      getStorageInfo().setProperty(KSM_ID, ksmId);
-    }
-  }
-
-  /**
-   * Retrieves the SCM ID from the version file.
-   * @return SCM_ID
-   */
-  public String getScmId() {
-    return getStorageInfo().getProperty(SCMStorage.SCM_ID);
-  }
-
-  /**
-   * Retrieves the KSM ID from the version file.
-   * @return KSM_ID
-   */
-  public String getKsmId() {
-    return getStorageInfo().getProperty(KSM_ID);
-  }
-
-  @Override
-  protected Properties getNodeProperties() {
-    String ksmId = getKsmId();
-    if (ksmId == null) {
-      ksmId = UUID.randomUUID().toString();
-    }
-    Properties ksmProperties = new Properties();
-    ksmProperties.setProperty(KSM_ID, ksmId);
-    return ksmProperties;
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyDeletingService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyDeletingService.java
deleted file mode 100644
index 3beaed4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyDeletingService.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.BackgroundService;
-import org.apache.hadoop.utils.BackgroundTask;
-import org.apache.hadoop.utils.BackgroundTaskQueue;
-import org.apache.hadoop.utils.BackgroundTaskResult;
-import org.apache.hadoop.utils.BackgroundTaskResult.EmptyTaskResult;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT;
-
-/**
- * This is the background service to delete keys.
- * Scan the metadata of ksm periodically to get
- * the keys with prefix "#deleting" and ask scm to
- * delete metadata accordingly, if scm returns
- * success for keys, then clean up those keys.
- */
-public class KeyDeletingService extends BackgroundService {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(KeyDeletingService.class);
-
-  // The thread pool size for key deleting service.
-  private final static int KEY_DELETING_CORE_POOL_SIZE = 2;
-
-  private final ScmBlockLocationProtocol scmClient;
-  private final KeyManager manager;
-  private final int keyLimitPerTask;
-
-  public KeyDeletingService(ScmBlockLocationProtocol scmClient,
-      KeyManager manager, long serviceInterval,
-      long serviceTimeout, Configuration conf) {
-    super("KeyDeletingService", serviceInterval, TimeUnit.MILLISECONDS,
-        KEY_DELETING_CORE_POOL_SIZE, serviceTimeout);
-    this.scmClient = scmClient;
-    this.manager = manager;
-    this.keyLimitPerTask = conf.getInt(OZONE_KEY_DELETING_LIMIT_PER_TASK,
-        OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT);
-  }
-
-  @Override
-  public BackgroundTaskQueue getTasks() {
-    BackgroundTaskQueue queue = new BackgroundTaskQueue();
-    queue.add(new KeyDeletingTask());
-    return queue;
-  }
-
-  /**
-   * A key deleting task scans KSM DB and looking for a certain number
-   * of pending-deletion keys, sends these keys along with their associated
-   * blocks to SCM for deletion. Once SCM confirms keys are deleted (once
-   * SCM persisted the blocks info in its deletedBlockLog), it removes
-   * these keys from the DB.
-   */
-  private class KeyDeletingTask implements
-      BackgroundTask<BackgroundTaskResult> {
-
-    @Override
-    public int getPriority() {
-      return 0;
-    }
-
-    @Override
-    public BackgroundTaskResult call() throws Exception {
-      try {
-        long startTime = Time.monotonicNow();
-        List<BlockGroup> keyBlocksList = manager
-            .getPendingDeletionKeys(keyLimitPerTask);
-        if (keyBlocksList.size() > 0) {
-          LOG.info("Found {} to-delete keys in KSM", keyBlocksList.size());
-          List<DeleteBlockGroupResult> results =
-              scmClient.deleteKeyBlocks(keyBlocksList);
-          for (DeleteBlockGroupResult result : results) {
-            if (result.isSuccess()) {
-              try {
-                // Purge key from KSM DB.
-                manager.deletePendingDeletionKey(result.getObjectKey());
-                LOG.debug("Key {} deleted from KSM DB", result.getObjectKey());
-              } catch (IOException e) {
-                // if a pending deletion key is failed to delete,
-                // print a warning here and retain it in this state,
-                // so that it can be attempt to delete next time.
-                LOG.warn("Failed to delete pending-deletion key {}",
-                    result.getObjectKey(), e);
-              }
-            } else {
-              // Key deletion failed, retry in next interval.
-              LOG.warn("Key {} deletion failed because some of the blocks"
-                  + " were failed to delete, failed blocks: {}",
-                  result.getObjectKey(),
-                  String.join(",", result.getFailedBlocks()));
-            }
-          }
-
-          if (!results.isEmpty()) {
-            LOG.info("Number of key deleted from KSM DB: {},"
-                + " task elapsed time: {}ms",
-                results.size(), Time.monotonicNow() - startTime);
-          }
-
-          return results::size;
-        } else {
-          LOG.debug("No pending deletion key found in KSM");
-        }
-      } catch (IOException e) {
-        LOG.error("Unable to get pending deletion keys, retry in"
-            + " next interval", e);
-      }
-      return EmptyTaskResult.newResult();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java
deleted file mode 100644
index e71ce5f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Handles key level commands.
- */
-public interface KeyManager {
-
-  /**
-   * Start key manager.
-   */
-  void start();
-
-  /**
-   * Stop key manager.
-   */
-  void stop() throws IOException;
-
-  /**
-   * After calling commit, the key will be made visible. There can be multiple
-   * open key writes in parallel (identified by client id). The most recently
-   * committed one will be the one visible.
-   *
-   * @param args the key to commit.
-   * @param clientID the client that is committing.
-   * @throws IOException
-   */
-  void commitKey(KsmKeyArgs args, int clientID) throws IOException;
-
-  /**
-   * A client calls this on an open key, to request to allocate a new block,
-   * and appended to the tail of current block list of the open client.
-   *
-   * @param args the key to append
-   * @param clientID the client requesting block.
-   * @return the reference to the new block.
-   * @throws IOException
-   */
-  KsmKeyLocationInfo allocateBlock(KsmKeyArgs args, int clientID)
-      throws IOException;
-  /**
-   * Given the args of a key to put, write an open key entry to meta data.
-   *
-   * In case that the container creation or key write failed on
-   * DistributedStorageHandler, this key's metadata will still stay in KSM.
-   * TODO garbage collect the open keys that never get closed
-   *
-   * @param args the args of the key provided by client.
-   * @return a OpenKeySession instance client uses to talk to container.
-   * @throws Exception
-   */
-  OpenKeySession openKey(KsmKeyArgs args) throws IOException;
-
-  /**
-   * Look up an existing key. Return the info of the key to client side, which
-   * DistributedStorageHandler will use to access the data on datanode.
-   *
-   * @param args the args of the key provided by client.
-   * @return a KsmKeyInfo instance client uses to talk to container.
-   * @throws IOException
-   */
-  KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException;
-
-  /**
-   * Deletes an object by an object key. The key will be immediately removed
-   * from KSM namespace and become invisible to clients. The object data
-   * will be removed in async manner that might retain for some time.
-   *
-   * @param args the args of the key provided by client.
-   * @throws IOException if specified key doesn't exist or
-   * some other I/O errors while deleting an object.
-   */
-  void deleteKey(KsmKeyArgs args) throws IOException;
-
-  /**
-   * Returns a list of keys represented by {@link KsmKeyInfo}
-   * in the given bucket.
-   *
-   * @param volumeName
-   *   the name of the volume.
-   * @param bucketName
-   *   the name of the bucket.
-   * @param startKey
-   *   the start key name, only the keys whose name is
-   *   after this value will be included in the result.
-   *   This key is excluded from the result.
-   * @param keyPrefix
-   *   key name prefix, only the keys whose name has
-   *   this prefix will be included in the result.
-   * @param maxKeys
-   *   the maximum number of keys to return. It ensures
-   *   the size of the result will not exceed this limit.
-   * @return a list of keys.
-   * @throws IOException
-   */
-  List<KsmKeyInfo> listKeys(String volumeName,
-      String bucketName, String startKey, String keyPrefix, int maxKeys)
-      throws IOException;
-
-  /**
-   * Returns a list of pending deletion key info that ups to the given count.
-   * Each entry is a {@link BlockGroup}, which contains the info about the
-   * key name and all its associated block IDs. A pending deletion key is
-   * stored with #deleting# prefix in KSM DB.
-   *
-   * @param count max number of keys to return.
-   * @return a list of {@link BlockGroup} representing keys and blocks.
-   * @throws IOException
-   */
-  List<BlockGroup> getPendingDeletionKeys(int count) throws IOException;
-
-  /**
-   * Deletes a pending deletion key by its name. This is often called when
-   * key can be safely deleted from this layer. Once called, all footprints
-   * of the key will be purged from KSM DB.
-   *
-   * @param objectKeyName object key name with #deleting# prefix.
-   * @throws IOException if specified key doesn't exist or other I/O errors.
-   */
-  void deletePendingDeletionKey(String objectKeyName) throws IOException;
-
-  /**
-   * Returns a list of all still open key info. Which contains the info about
-   * the key name and all its associated block IDs. A pending open key has
-   * prefix #open# in KSM DB.
-   *
-   * @return a list of {@link BlockGroup} representing keys and blocks.
-   * @throws IOException
-   */
-  List<BlockGroup> getExpiredOpenKeys() throws IOException;
-
-  /**
-   * Deletes a expired open key by its name. Called when a hanging key has been
-   * lingering for too long. Once called, the open key entries gets removed
-   * from KSM mdata data.
-   *
-   * @param objectKeyName object key name with #open# prefix.
-   * @throws IOException if specified key doesn't exist or other I/O errors.
-   */
-  void deleteExpiredOpenKey(String objectKeyName) throws IOException;
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[44/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
deleted file mode 100644
index 5069220..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
+++ /dev/null
@@ -1,308 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Proxy;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_CLIENT_PROTOCOL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_CLIENT_PROTOCOL_REST;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_CLIENT_PROTOCOL_RPC;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
-    .OZONE_KSM_HTTP_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
-    .OZONE_KSM_HTTP_BIND_PORT_DEFAULT;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_PORT_DEFAULT;
-
-/**
- * Factory class to create different types of OzoneClients.
- * Based on <code>ozone.client.protocol</code>, it decides which
- * protocol to use for the communication.
- * Default value is
- * <code>org.apache.hadoop.ozone.client.rpc.RpcClient</code>.<br>
- * OzoneClientFactory constructs a proxy using
- * {@link OzoneClientInvocationHandler}
- * and creates OzoneClient instance with it.
- * {@link OzoneClientInvocationHandler} dispatches the call to
- * underlying {@link ClientProtocol} implementation.
- */
-public final class OzoneClientFactory {
-
-  private static final Logger LOG = LoggerFactory.getLogger(
-      OzoneClientFactory.class);
-
-  /**
-   * Private constructor, class is not meant to be initialized.
-   */
-  private OzoneClientFactory(){}
-
-
-  /**
-   * Constructs and return an OzoneClient with default configuration.
-   *
-   * @return OzoneClient
-   *
-   * @throws IOException
-   */
-  public static OzoneClient getClient() throws IOException {
-    LOG.info("Creating OzoneClient with default configuration.");
-    return getClient(new OzoneConfiguration());
-  }
-
-  /**
-   * Constructs and return an OzoneClient based on the configuration object.
-   * Protocol type is decided by <code>ozone.client.protocol</code>.
-   *
-   * @param config
-   *        Configuration to be used for OzoneClient creation
-   *
-   * @return OzoneClient
-   *
-   * @throws IOException
-   */
-  public static OzoneClient getClient(Configuration config)
-      throws IOException {
-    Preconditions.checkNotNull(config);
-    Class<? extends ClientProtocol> clazz = (Class<? extends ClientProtocol>)
-        config.getClass(OZONE_CLIENT_PROTOCOL, OZONE_CLIENT_PROTOCOL_RPC);
-    return getClient(getClientProtocol(clazz, config), config);
-  }
-
-  /**
-   * Returns an OzoneClient which will use RPC protocol.
-   *
-   * @param ksmHost
-   *        hostname of KeySpaceManager to connect.
-   *
-   * @return OzoneClient
-   *
-   * @throws IOException
-   */
-  public static OzoneClient getRpcClient(String ksmHost)
-      throws IOException {
-    return getRpcClient(ksmHost, OZONE_KSM_PORT_DEFAULT,
-        new OzoneConfiguration());
-  }
-
-  /**
-   * Returns an OzoneClient which will use RPC protocol.
-   *
-   * @param ksmHost
-   *        hostname of KeySpaceManager to connect.
-   *
-   * @param ksmRpcPort
-   *        RPC port of KeySpaceManager.
-   *
-   * @return OzoneClient
-   *
-   * @throws IOException
-   */
-  public static OzoneClient getRpcClient(String ksmHost, Integer ksmRpcPort)
-      throws IOException {
-    return getRpcClient(ksmHost, ksmRpcPort, new OzoneConfiguration());
-  }
-
-  /**
-   * Returns an OzoneClient which will use RPC protocol.
-   *
-   * @param ksmHost
-   *        hostname of KeySpaceManager to connect.
-   *
-   * @param ksmRpcPort
-   *        RPC port of KeySpaceManager.
-   *
-   * @param config
-   *        Configuration to be used for OzoneClient creation
-   *
-   * @return OzoneClient
-   *
-   * @throws IOException
-   */
-  public static OzoneClient getRpcClient(String ksmHost, Integer ksmRpcPort,
-                                         Configuration config)
-      throws IOException {
-    Preconditions.checkNotNull(ksmHost);
-    Preconditions.checkNotNull(ksmRpcPort);
-    Preconditions.checkNotNull(config);
-    config.set(OZONE_KSM_ADDRESS_KEY, ksmHost + ":" + ksmRpcPort);
-    return getRpcClient(config);
-  }
-
-  /**
-   * Returns an OzoneClient which will use RPC protocol.
-   *
-   * @param config
-   *        used for OzoneClient creation
-   *
-   * @return OzoneClient
-   *
-   * @throws IOException
-   */
-  public static OzoneClient getRpcClient(Configuration config)
-      throws IOException {
-    Preconditions.checkNotNull(config);
-    return getClient(getClientProtocol(OZONE_CLIENT_PROTOCOL_RPC, config),
-        config);
-  }
-
-  /**
-   * Returns an OzoneClient which will use REST protocol.
-   *
-   * @param ksmHost
-   *        hostname of KeySpaceManager to connect.
-   *
-   * @return OzoneClient
-   *
-   * @throws IOException
-   */
-  public static OzoneClient getRestClient(String ksmHost)
-      throws IOException {
-    return getRestClient(ksmHost, OZONE_KSM_HTTP_BIND_PORT_DEFAULT);
-  }
-
-  /**
-   * Returns an OzoneClient which will use REST protocol.
-   *
-   * @param ksmHost
-   *        hostname of KeySpaceManager to connect.
-   *
-   * @param ksmHttpPort
-   *        HTTP port of KeySpaceManager.
-   *
-   * @return OzoneClient
-   *
-   * @throws IOException
-   */
-  public static OzoneClient getRestClient(String ksmHost, Integer ksmHttpPort)
-      throws IOException {
-    return getRestClient(ksmHost, ksmHttpPort, new OzoneConfiguration());
-  }
-
-  /**
-   * Returns an OzoneClient which will use REST protocol.
-   *
-   * @param ksmHost
-   *        hostname of KeySpaceManager to connect.
-   *
-   * @param ksmHttpPort
-   *        HTTP port of KeySpaceManager.
-   *
-   * @param config
-   *        Configuration to be used for OzoneClient creation
-   *
-   * @return OzoneClient
-   *
-   * @throws IOException
-   */
-  public static OzoneClient getRestClient(String ksmHost, Integer ksmHttpPort,
-                                          Configuration config)
-      throws IOException {
-    Preconditions.checkNotNull(ksmHost);
-    Preconditions.checkNotNull(ksmHttpPort);
-    Preconditions.checkNotNull(config);
-    config.set(OZONE_KSM_HTTP_ADDRESS_KEY, ksmHost + ":" +  ksmHttpPort);
-    return getRestClient(config);
-  }
-
-  /**
-   * Returns an OzoneClient which will use REST protocol.
-   *
-   * @param config
-   *        Configuration to be used for OzoneClient creation
-   *
-   * @return OzoneClient
-   *
-   * @throws IOException
-   */
-  public static OzoneClient getRestClient(Configuration config)
-      throws IOException {
-    Preconditions.checkNotNull(config);
-    return getClient(getClientProtocol(OZONE_CLIENT_PROTOCOL_REST, config),
-        config);
-  }
-
-  /**
-   * Creates OzoneClient with the given ClientProtocol and Configuration.
-   *
-   * @param clientProtocol
-   *        Protocol to be used by the OzoneClient
-   *
-   * @param config
-   *        Configuration to be used for OzoneClient creation
-   */
-  private static OzoneClient getClient(ClientProtocol clientProtocol,
-                                       Configuration config) {
-    OzoneClientInvocationHandler clientHandler =
-        new OzoneClientInvocationHandler(clientProtocol);
-    ClientProtocol proxy = (ClientProtocol) Proxy.newProxyInstance(
-        OzoneClientInvocationHandler.class.getClassLoader(),
-        new Class<?>[]{ClientProtocol.class}, clientHandler);
-    return new OzoneClient(config, proxy);
-  }
-
-  /**
-   * Returns an instance of Protocol class.
-   *
-   * @param protocolClass
-   *        Class object of the ClientProtocol.
-   *
-   * @param config
-   *        Configuration used to initialize ClientProtocol.
-   *
-   * @return ClientProtocol
-   *
-   * @throws IOException
-   */
-  private static ClientProtocol getClientProtocol(
-      Class<? extends ClientProtocol> protocolClass, Configuration config)
-      throws IOException {
-    try {
-      LOG.debug("Using {} as client protocol.",
-          protocolClass.getCanonicalName());
-      Constructor<? extends ClientProtocol> ctor =
-          protocolClass.getConstructor(Configuration.class);
-      return ctor.newInstance(config);
-    } catch (Exception e) {
-      final String message = "Couldn't create protocol " + protocolClass;
-      LOG.error(message + " exception:" + e);
-      if (e.getCause() instanceof IOException) {
-        throw (IOException) e.getCause();
-      } else if (e instanceof InvocationTargetException) {
-        throw new IOException(message,
-            ((InvocationTargetException) e).getTargetException());
-      } else {
-        throw new IOException(message, e);
-      }
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientInvocationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientInvocationHandler.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientInvocationHandler.java
deleted file mode 100644
index 3051e2d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientInvocationHandler.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-
-/**
- * Invocation Handler for ozone client which dispatches the call to underlying
- * ClientProtocol implementation.
- */
-public class OzoneClientInvocationHandler implements InvocationHandler {
-
-
-  private static final Logger LOG = LoggerFactory.getLogger(OzoneClient.class);
-  private final ClientProtocol target;
-
-  /**
-   * Constructs OzoneClientInvocationHandler with the proxy.
-   * @param target proxy to be used for method invocation.
-   */
-  public OzoneClientInvocationHandler(ClientProtocol target) {
-    this.target = target;
-  }
-
-  @Override
-  public Object invoke(Object proxy, Method method, Object[] args)
-      throws Throwable {
-    LOG.trace("Invoking method {} on proxy {}", method, proxy);
-    try {
-      long startTime = Time.monotonicNow();
-      Object result = method.invoke(target, args);
-      LOG.debug("Call: {} took {} ms", method,
-          Time.monotonicNow() - startTime);
-      return result;
-    } catch(InvocationTargetException iEx) {
-      throw iEx.getCause();
-    }
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java
deleted file mode 100644
index 1830bdf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java
+++ /dev/null
@@ -1,874 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import com.google.common.base.Optional;
-
-import com.google.common.base.Preconditions;
-import com.google.common.net.HostAndPort;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.InetSocketAddress;
-import java.text.ParseException;
-import java.time.Instant;
-import java.time.ZoneId;
-import java.time.ZonedDateTime;
-import java.time.format.DateTimeFormatter;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_SERVICERPC_PORT_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_JSCSI_PORT_DEFAULT;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
-    .OZONE_KSM_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
-    .OZONE_KSM_BIND_HOST_DEFAULT;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
-    .OZONE_KSM_PORT_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_DEADNODE_INTERVAL_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_DEADNODE_INTERVAL;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_HEARTBEAT_INTERVAL;
-
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
-
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_HEARTBEAT_RPC_TIMEOUT;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_STALENODE_INTERVAL_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_STALENODE_INTERVAL;
-
-/**
- * Utility methods for Ozone and Container Clients.
- *
- * The methods to retrieve SCM service endpoints assume there is a single
- * SCM service instance. This will change when we switch to replicated service
- * instances for redundancy.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public final class OzoneClientUtils {
-  private static final Logger LOG = LoggerFactory.getLogger(
-      OzoneClientUtils.class);
-  private static final int NO_PORT = -1;
-
-  /**
-   * Date format that used in ozone. Here the format is thread safe to use.
-   */
-  private static final ThreadLocal<DateTimeFormatter> DATE_FORMAT =
-      ThreadLocal.withInitial(() -> {
-        DateTimeFormatter format =
-            DateTimeFormatter.ofPattern(OzoneConsts.OZONE_DATE_FORMAT);
-        return format.withZone(ZoneId.of(OzoneConsts.OZONE_TIME_ZONE));
-      });
-
-  /**
-   * The service ID of the solitary Ozone SCM service.
-   */
-  public static final String OZONE_SCM_SERVICE_ID = "OzoneScmService";
-  public static final String OZONE_SCM_SERVICE_INSTANCE_ID =
-      "OzoneScmServiceInstance";
-
-  private OzoneClientUtils() {
-    // Never constructed
-  }
-
-  /**
-   * Retrieve the socket addresses of all storage container managers.
-   *
-   * @param conf
-   * @return A collection of SCM addresses
-   * @throws IllegalArgumentException If the configuration is invalid
-   */
-  public static Collection<InetSocketAddress> getSCMAddresses(
-      Configuration conf) throws IllegalArgumentException {
-    Collection<InetSocketAddress> addresses =
-        new HashSet<InetSocketAddress>();
-    Collection<String> names =
-        conf.getTrimmedStringCollection(ScmConfigKeys.OZONE_SCM_NAMES);
-    if (names == null || names.isEmpty()) {
-      throw new IllegalArgumentException(ScmConfigKeys.OZONE_SCM_NAMES
-          + " need to be a set of valid DNS names or IP addresses."
-          + " Null or empty address list found.");
-    }
-
-    final com.google.common.base.Optional<Integer>
-        defaultPort =  com.google.common.base.Optional.of(ScmConfigKeys
-        .OZONE_SCM_DEFAULT_PORT);
-    for (String address : names) {
-      com.google.common.base.Optional<String> hostname =
-          OzoneClientUtils.getHostName(address);
-      if (!hostname.isPresent()) {
-        throw new IllegalArgumentException("Invalid hostname for SCM: "
-            + hostname);
-      }
-      com.google.common.base.Optional<Integer> port =
-          OzoneClientUtils.getHostPort(address);
-      InetSocketAddress addr = NetUtils.createSocketAddr(hostname.get(),
-          port.or(defaultPort.get()));
-      addresses.add(addr);
-    }
-    return addresses;
-  }
-
-  /**
-   * Retrieve the socket address that should be used by clients to connect
-   * to the SCM.
-   *
-   * @param conf
-   * @return Target InetSocketAddress for the SCM client endpoint.
-   */
-  public static InetSocketAddress getScmAddressForClients(Configuration conf) {
-    final Optional<String> host = getHostNameFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
-
-    if (!host.isPresent()) {
-      throw new IllegalArgumentException(
-          ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY +
-          " must be defined. See" +
-          " https://wiki.apache.org/hadoop/Ozone#Configuration for details" +
-          " on configuring Ozone.");
-    }
-
-    final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
-
-    return NetUtils.createSocketAddr(host.get() + ":" +
-        port.or(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT));
-  }
-
-  /**
-   * Retrieve the socket address that should be used by clients to connect
-   * to the SCM for block service. If
-   * {@link ScmConfigKeys#OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY} is not defined
-   * then {@link ScmConfigKeys#OZONE_SCM_CLIENT_ADDRESS_KEY} is used.
-   *
-   * @param conf
-   * @return Target InetSocketAddress for the SCM block client endpoint.
-   * @throws IllegalArgumentException if configuration is not defined.
-   */
-  public static InetSocketAddress getScmAddressForBlockClients(
-      Configuration conf) {
-    Optional<String> host = getHostNameFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY);
-
-    if (!host.isPresent()) {
-      host = getHostNameFromConfigKeys(conf,
-              ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
-      if (!host.isPresent()) {
-        throw new IllegalArgumentException(
-                ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY +
-                        " must be defined. See" +
-                        " https://wiki.apache.org/hadoop/Ozone#Configuration" +
-                        " for details on configuring Ozone.");
-      }
-    }
-
-    final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY);
-
-    return NetUtils.createSocketAddr(host.get() + ":" +
-        port.or(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT));
-  }
-
-  /**
-   * Retrieve the socket address that should be used by DataNodes to connect
-   * to the SCM.
-   *
-   * @param conf
-   * @return Target InetSocketAddress for the SCM service endpoint.
-   */
-  public static InetSocketAddress getScmAddressForDataNodes(
-      Configuration conf) {
-    // We try the following settings in decreasing priority to retrieve the
-    // target host.
-    // - OZONE_SCM_DATANODE_ADDRESS_KEY
-    // - OZONE_SCM_CLIENT_ADDRESS_KEY
-    //
-    final Optional<String> host = getHostNameFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY,
-        ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
-
-    if (!host.isPresent()) {
-      throw new IllegalArgumentException(
-          ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY +
-          " must be defined. See" +
-          " https://wiki.apache.org/hadoop/Ozone#Configuration for details" +
-          " on configuring Ozone.");
-    }
-
-    // If no port number is specified then we'll just try the defaultBindPort.
-    final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY);
-
-    InetSocketAddress addr = NetUtils.createSocketAddr(host.get() + ":" +
-        port.or(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
-
-    return addr;
-  }
-
-  /**
-   * Retrieve the socket address that should be used by clients to connect
-   * to the SCM.
-   *
-   * @param conf
-   * @return Target InetSocketAddress for the SCM client endpoint.
-   */
-  public static InetSocketAddress getScmClientBindAddress(
-      Configuration conf) {
-    final Optional<String> host = getHostNameFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY);
-
-    final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
-
-    return NetUtils.createSocketAddr(
-        host.or(ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT) + ":" +
-            port.or(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT));
-  }
-
-  /**
-   * Retrieve the socket address that should be used by clients to connect
-   * to the SCM Block service.
-   *
-   * @param conf
-   * @return Target InetSocketAddress for the SCM block client endpoint.
-   */
-  public static InetSocketAddress getScmBlockClientBindAddress(
-      Configuration conf) {
-    final Optional<String> host = getHostNameFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_KEY);
-
-    final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY);
-
-    return NetUtils.createSocketAddr(
-        host.or(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT) +
-            ":" + port.or(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT));
-  }
-
-  /**
-   * Retrieve the socket address that should be used by DataNodes to connect
-   * to the SCM.
-   *
-   * @param conf
-   * @return Target InetSocketAddress for the SCM service endpoint.
-   */
-  public static InetSocketAddress getScmDataNodeBindAddress(
-      Configuration conf) {
-    final Optional<String> host = getHostNameFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_KEY);
-
-    // If no port number is specified then we'll just try the defaultBindPort.
-    final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY);
-
-    return NetUtils.createSocketAddr(
-        host.or(ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_DEFAULT) + ":" +
-            port.or(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
-  }
-
-
-  /**
-   * Retrieve the socket address that is used by KSM.
-   * @param conf
-   * @return Target InetSocketAddress for the SCM service endpoint.
-   */
-  public static InetSocketAddress getKsmAddress(
-      Configuration conf) {
-    final Optional<String> host = getHostNameFromConfigKeys(conf,
-        OZONE_KSM_ADDRESS_KEY);
-
-    // If no port number is specified then we'll just try the defaultBindPort.
-    final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
-        OZONE_KSM_ADDRESS_KEY);
-
-    return NetUtils.createSocketAddr(
-        host.or(OZONE_KSM_BIND_HOST_DEFAULT) + ":" +
-            port.or(OZONE_KSM_PORT_DEFAULT));
-  }
-
-  /**
-   * Retrieve the socket address that should be used by clients to connect
-   * to KSM.
-   * @param conf
-   * @return Target InetSocketAddress for the KSM service endpoint.
-   */
-  public static InetSocketAddress getKsmAddressForClients(
-      Configuration conf) {
-    final Optional<String> host = getHostNameFromConfigKeys(conf,
-        OZONE_KSM_ADDRESS_KEY);
-
-    if (!host.isPresent()) {
-      throw new IllegalArgumentException(
-          OZONE_KSM_ADDRESS_KEY + " must be defined. See" +
-              " https://wiki.apache.org/hadoop/Ozone#Configuration for" +
-              " details on configuring Ozone.");
-    }
-
-    // If no port number is specified then we'll just try the defaultBindPort.
-    final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
-        OZONE_KSM_ADDRESS_KEY);
-
-    return NetUtils.createSocketAddr(
-        host.get() + ":" + port.or(OZONE_KSM_PORT_DEFAULT));
-  }
-
-  /**
-   * Retrieve the socket address that is used by CBlock Service.
-   * @param conf
-   * @return Target InetSocketAddress for the CBlock Service endpoint.
-   */
-  public static InetSocketAddress getCblockServiceRpcAddr(
-      Configuration conf) {
-    final Optional<String> host = getHostNameFromConfigKeys(conf,
-        DFS_CBLOCK_SERVICERPC_ADDRESS_KEY);
-
-    // If no port number is specified then we'll just try the defaultBindPort.
-    final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
-        DFS_CBLOCK_SERVICERPC_ADDRESS_KEY);
-
-    return NetUtils.createSocketAddr(
-        host.or(DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT) + ":" +
-            port.or(DFS_CBLOCK_SERVICERPC_PORT_DEFAULT));
-  }
-
-  /**
-   * Retrieve the socket address that is used by CBlock Server.
-   * @param conf
-   * @return Target InetSocketAddress for the CBlock Server endpoint.
-   */
-  public static InetSocketAddress getCblockServerRpcAddr(
-      Configuration conf) {
-    final Optional<String> host = getHostNameFromConfigKeys(conf,
-        DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY);
-
-    // If no port number is specified then we'll just try the defaultBindPort.
-    final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
-        DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY);
-
-    return NetUtils.createSocketAddr(
-        host.or(DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT) + ":" +
-            port.or(DFS_CBLOCK_JSCSI_PORT_DEFAULT));
-  }
-
-  /**
-   * Retrieve the hostname, trying the supplied config keys in order.
-   * Each config value may be absent, or if present in the format
-   * host:port (the :port part is optional).
-   *
-   * @param conf  - Conf
-   * @param keys a list of configuration key names.
-   *
-   * @return first hostname component found from the given keys, or absent.
-   * @throws IllegalArgumentException if any values are not in the 'host'
-   *             or host:port format.
-   */
-  public static Optional<String> getHostNameFromConfigKeys(Configuration conf,
-      String... keys) {
-    for (final String key : keys) {
-      final String value = conf.getTrimmed(key);
-      final Optional<String> hostName = getHostName(value);
-      if (hostName.isPresent()) {
-        return hostName;
-      }
-    }
-    return Optional.absent();
-  }
-
-  /**
-   * Gets the hostname or Indicates that it is absent.
-   * @param value host or host:port
-   * @return hostname
-   */
-  public static Optional<String> getHostName(String value) {
-    if ((value == null) || value.isEmpty()) {
-      return Optional.absent();
-    }
-    return Optional.of(HostAndPort.fromString(value).getHostText());
-  }
-
-  /**
-   * Gets the port if there is one, throws otherwise.
-   * @param value  String in host:port format.
-   * @return Port
-   */
-  public static Optional<Integer> getHostPort(String value) {
-    if((value == null) || value.isEmpty()) {
-      return Optional.absent();
-    }
-    int port = HostAndPort.fromString(value).getPortOrDefault(NO_PORT);
-    if (port == NO_PORT) {
-      return Optional.absent();
-    } else {
-      return Optional.of(port);
-    }
-  }
-
-  /**
-   * Returns the cache value to be used for list calls.
-   * @param conf Configuration object
-   * @return list cache size
-   */
-  public static int getListCacheSize(Configuration conf) {
-    return conf.getInt(OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE,
-        OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE_DEFAULT);
-  }
-
-  /**
-   * Retrieve the port number, trying the supplied config keys in order.
-   * Each config value may be absent, or if present in the format
-   * host:port (the :port part is optional).
-   *
-   * @param conf Conf
-   * @param keys a list of configuration key names.
-   *
-   * @return first port number component found from the given keys, or absent.
-   * @throws IllegalArgumentException if any values are not in the 'host'
-   *             or host:port format.
-   */
-  public static Optional<Integer> getPortNumberFromConfigKeys(
-      Configuration conf, String... keys) {
-    for (final String key : keys) {
-      final String value = conf.getTrimmed(key);
-      final Optional<Integer> hostPort = getHostPort(value);
-      if (hostPort.isPresent()) {
-        return hostPort;
-      }
-    }
-    return Optional.absent();
-  }
-
-  /**
-   * Return the list of service addresses for the Ozone SCM. This method is used
-   * by the DataNodes to determine the service instances to connect to.
-   *
-   * @param conf
-   * @return list of SCM service addresses.
-   */
-  public static Map<String, ? extends Map<String, InetSocketAddress>>
-      getScmServiceRpcAddresses(Configuration conf) {
-    final Map<String, InetSocketAddress> serviceInstances = new HashMap<>();
-    serviceInstances.put(OZONE_SCM_SERVICE_INSTANCE_ID,
-        getScmAddressForDataNodes(conf));
-
-    final Map<String, Map<String, InetSocketAddress>> services =
-        new HashMap<>();
-    services.put(OZONE_SCM_SERVICE_ID, serviceInstances);
-    return services;
-  }
-
-  /**
-   * Checks that a given value is with a range.
-   *
-   * For example, sanitizeUserArgs(17, 3, 5, 10)
-   * ensures that 17 is greater/equal than 3 * 5 and less/equal to 3 * 10.
-   *
-   * @param valueTocheck  - value to check
-   * @param baseValue     - the base value that is being used.
-   * @param minFactor     - range min - a 2 here makes us ensure that value
-   *                        valueTocheck is at least twice the baseValue.
-   * @param maxFactor     - range max
-   * @return long
-   */
-  private static long sanitizeUserArgs(long valueTocheck, long baseValue,
-      long minFactor, long maxFactor)
-      throws IllegalArgumentException {
-    if ((valueTocheck >= (baseValue * minFactor)) &&
-        (valueTocheck <= (baseValue * maxFactor))) {
-      return valueTocheck;
-    }
-    String errMsg = String.format("%d is not within min = %d or max = " +
-        "%d", valueTocheck, baseValue * minFactor, baseValue * maxFactor);
-    throw new IllegalArgumentException(errMsg);
-  }
-
-  /**
-   * Returns the interval in which the heartbeat processor thread runs.
-   *
-   * @param conf - Configuration
-   * @return long in Milliseconds.
-   */
-  public static long getScmheartbeatCheckerInterval(Configuration conf) {
-    return conf.getTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
-        ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_DEFAULT,
-        TimeUnit.MILLISECONDS);
-  }
-
-  /**
-   * Heartbeat Interval - Defines the heartbeat frequency from a datanode to
-   * SCM.
-   *
-   * @param conf - Ozone Config
-   * @return - HB interval in seconds.
-   */
-  public static long getScmHeartbeatInterval(Configuration conf) {
-    return conf.getTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL,
-        ScmConfigKeys.OZONE_SCM_HEARBEAT_INTERVAL_DEFAULT,
-        TimeUnit.SECONDS);
-  }
-
-  /**
-   * Get the Stale Node interval, which is used by SCM to flag a datanode as
-   * stale, if the heartbeat from that node has been missing for this duration.
-   *
-   * @param conf - Configuration.
-   * @return - Long, Milliseconds to wait before flagging a node as stale.
-   */
-  public static long getStaleNodeInterval(Configuration conf) {
-
-    long staleNodeIntervalMs =
-        conf.getTimeDuration(OZONE_SCM_STALENODE_INTERVAL,
-        OZONE_SCM_STALENODE_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS);
-
-    long heartbeatThreadFrequencyMs = getScmheartbeatCheckerInterval(conf);
-
-    long heartbeatIntervalMs = getScmHeartbeatInterval(conf) * 1000;
-
-
-    // Make sure that StaleNodeInterval is configured way above the frequency
-    // at which we run the heartbeat thread.
-    //
-    // Here we check that staleNodeInterval is at least five times more than the
-    // frequency at which the accounting thread is going to run.
-    try {
-      sanitizeUserArgs(staleNodeIntervalMs, heartbeatThreadFrequencyMs,
-          5, 1000);
-    } catch (IllegalArgumentException ex) {
-      LOG.error("Stale Node Interval is cannot be honored due to " +
-              "mis-configured {}. ex:  {}",
-          OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, ex);
-      throw ex;
-    }
-
-    // Make sure that stale node value is greater than configured value that
-    // datanodes are going to send HBs.
-    try {
-      sanitizeUserArgs(staleNodeIntervalMs, heartbeatIntervalMs, 3, 1000);
-    } catch (IllegalArgumentException ex) {
-      LOG.error("Stale Node Interval MS is cannot be honored due to " +
-              "mis-configured {}. ex:  {}", OZONE_SCM_HEARTBEAT_INTERVAL, ex);
-      throw ex;
-    }
-    return staleNodeIntervalMs;
-  }
-
-  /**
-   * Gets the interval for dead node flagging. This has to be a value that is
-   * greater than stale node value,  and by transitive relation we also know
-   * that this value is greater than heartbeat interval and heartbeatProcess
-   * Interval.
-   *
-   * @param conf - Configuration.
-   * @return - the interval for dead node flagging.
-   */
-  public static long getDeadNodeInterval(Configuration conf) {
-    long staleNodeIntervalMs = getStaleNodeInterval(conf);
-    long deadNodeIntervalMs = conf.getTimeDuration(OZONE_SCM_DEADNODE_INTERVAL,
-        OZONE_SCM_DEADNODE_INTERVAL_DEFAULT,
-        TimeUnit.MILLISECONDS);
-
-    try {
-      // Make sure that dead nodes Ms is at least twice the time for staleNodes
-      // with a max of 1000 times the staleNodes.
-      sanitizeUserArgs(deadNodeIntervalMs, staleNodeIntervalMs, 2, 1000);
-    } catch (IllegalArgumentException ex) {
-      LOG.error("Dead Node Interval MS is cannot be honored due to " +
-              "mis-configured {}. ex:  {}", OZONE_SCM_STALENODE_INTERVAL, ex);
-      throw ex;
-    }
-    return deadNodeIntervalMs;
-  }
-
-  /**
-   * Returns the maximum number of heartbeat to process per loop of the process
-   * thread.
-   * @param conf Configuration
-   * @return - int -- Number of HBs to process
-   */
-  public static int getMaxHBToProcessPerLoop(Configuration conf) {
-    return conf.getInt(ScmConfigKeys.OZONE_SCM_MAX_HB_COUNT_TO_PROCESS,
-        ScmConfigKeys.OZONE_SCM_MAX_HB_COUNT_TO_PROCESS_DEFAULT);
-  }
-
-  /**
-   * Timeout value for the RPC from Datanode to SCM, primarily used for
-   * Heartbeats and container reports.
-   *
-   * @param conf - Ozone Config
-   * @return - Rpc timeout in Milliseconds.
-   */
-  public static long getScmRpcTimeOutInMilliseconds(Configuration conf) {
-    return conf.getTimeDuration(OZONE_SCM_HEARTBEAT_RPC_TIMEOUT,
-        OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
-  }
-
-  /**
-   * Log Warn interval.
-   *
-   * @param conf - Ozone Config
-   * @return - Log warn interval.
-   */
-  public static int getLogWarnInterval(Configuration conf) {
-    return conf.getInt(OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT,
-        OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT);
-  }
-
-  /**
-   * returns the Container port.
-   * @param conf - Conf
-   * @return port number.
-   */
-  public static int getContainerPort(Configuration conf) {
-    return conf.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
-        OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
-  }
-
-  /**
-   * After starting an RPC server, updates configuration with the actual
-   * listening address of that server. The listening address may be different
-   * from the configured address if, for example, the configured address uses
-   * port 0 to request use of an ephemeral port.
-   *
-   * @param conf configuration to update
-   * @param rpcAddressKey configuration key for RPC server address
-   * @param addr configured address
-   * @param rpcServer started RPC server.
-   */
-  public static InetSocketAddress updateRPCListenAddress(
-      OzoneConfiguration conf, String rpcAddressKey,
-      InetSocketAddress addr, RPC.Server rpcServer) {
-    return updateListenAddress(conf, rpcAddressKey, addr,
-        rpcServer.getListenerAddress());
-  }
-
-  /**
-   * After starting an server, updates configuration with the actual
-   * listening address of that server. The listening address may be different
-   * from the configured address if, for example, the configured address uses
-   * port 0 to request use of an ephemeral port.
-   *
-   * @param conf       configuration to update
-   * @param addressKey configuration key for RPC server address
-   * @param addr       configured address
-   * @param listenAddr the real listening address.
-   */
-  public static InetSocketAddress updateListenAddress(OzoneConfiguration conf,
-      String addressKey, InetSocketAddress addr, InetSocketAddress listenAddr) {
-    InetSocketAddress updatedAddr = new InetSocketAddress(addr.getHostString(),
-        listenAddr.getPort());
-    conf.set(addressKey,
-        addr.getHostString() + ":" + listenAddr.getPort());
-    return updatedAddr;
-  }
-
-  /**
-   * Releases a http connection if the request is not null.
-   * @param request
-   */
-  public static void releaseConnection(HttpRequestBase request) {
-    if (request != null) {
-      request.releaseConnection();
-    }
-  }
-
-  /**
-   * @return a default instance of {@link CloseableHttpClient}.
-   */
-  public static CloseableHttpClient newHttpClient() {
-    return OzoneClientUtils.newHttpClient(new OzoneConfiguration());
-  }
-
-  /**
-   * Returns a {@link CloseableHttpClient} configured by given configuration.
-   * If conf is null, returns a default instance.
-   *
-   * @param conf configuration
-   * @return a {@link CloseableHttpClient} instance.
-   */
-  public static CloseableHttpClient newHttpClient(Configuration conf) {
-    long socketTimeout = OzoneConfigKeys
-        .OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT;
-    long connectionTimeout = OzoneConfigKeys
-        .OZONE_CLIENT_CONNECTION_TIMEOUT_DEFAULT;
-    if (conf != null) {
-      socketTimeout = conf.getTimeDuration(
-          OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT,
-          OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT,
-          TimeUnit.MILLISECONDS);
-      connectionTimeout = conf.getTimeDuration(
-          OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT,
-          OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT_DEFAULT,
-          TimeUnit.MILLISECONDS);
-    }
-
-    CloseableHttpClient client = HttpClients.custom()
-        .setDefaultRequestConfig(
-            RequestConfig.custom()
-                .setSocketTimeout(Math.toIntExact(socketTimeout))
-                .setConnectTimeout(Math.toIntExact(connectionTimeout))
-                .build())
-        .build();
-    return client;
-  }
-
-  /**
-   * verifies that bucket name / volume name is a valid DNS name.
-   *
-   * @param resName Bucket or volume Name to be validated
-   *
-   * @throws IllegalArgumentException
-   */
-  public static void verifyResourceName(String resName)
-      throws IllegalArgumentException {
-
-    if (resName == null) {
-      throw new IllegalArgumentException("Bucket or Volume name is null");
-    }
-
-    if ((resName.length() < OzoneConsts.OZONE_MIN_BUCKET_NAME_LENGTH) ||
-        (resName.length() > OzoneConsts.OZONE_MAX_BUCKET_NAME_LENGTH)) {
-      throw new IllegalArgumentException(
-          "Bucket or Volume length is illegal, " +
-              "valid length is 3-63 characters");
-    }
-
-    if ((resName.charAt(0) == '.') || (resName.charAt(0) == '-')) {
-      throw new IllegalArgumentException(
-          "Bucket or Volume name cannot start with a period or dash");
-    }
-
-    if ((resName.charAt(resName.length() - 1) == '.') ||
-        (resName.charAt(resName.length() - 1) == '-')) {
-      throw new IllegalArgumentException(
-          "Bucket or Volume name cannot end with a period or dash");
-    }
-
-    boolean isIPv4 = true;
-    char prev = (char) 0;
-
-    for (int index = 0; index < resName.length(); index++) {
-      char currChar = resName.charAt(index);
-
-      if (currChar != '.') {
-        isIPv4 = ((currChar >= '0') && (currChar <= '9')) && isIPv4;
-      }
-
-      if (currChar > 'A' && currChar < 'Z') {
-        throw new IllegalArgumentException(
-            "Bucket or Volume name does not support uppercase characters");
-      }
-
-      if ((currChar != '.') && (currChar != '-')) {
-        if ((currChar < '0') || (currChar > '9' && currChar < 'a') ||
-            (currChar > 'z')) {
-          throw new IllegalArgumentException("Bucket or Volume name has an " +
-              "unsupported character : " +
-              currChar);
-        }
-      }
-
-      if ((prev == '.') && (currChar == '.')) {
-        throw new IllegalArgumentException("Bucket or Volume name should not " +
-            "have two contiguous periods");
-      }
-
-      if ((prev == '-') && (currChar == '.')) {
-        throw new IllegalArgumentException(
-            "Bucket or Volume name should not have period after dash");
-      }
-
-      if ((prev == '.') && (currChar == '-')) {
-        throw new IllegalArgumentException(
-            "Bucket or Volume name should not have dash after period");
-      }
-      prev = currChar;
-    }
-
-    if (isIPv4) {
-      throw new IllegalArgumentException(
-          "Bucket or Volume name cannot be an IPv4 address or all numeric");
-    }
-  }
-
-  /**
-   * Convert time in millisecond to a human readable format required in ozone.
-   * @return a human readable string for the input time
-   */
-  public static String formatDateTime(long millis) {
-    ZonedDateTime dateTime = ZonedDateTime.ofInstant(
-        Instant.ofEpochSecond(millis), DATE_FORMAT.get().getZone());
-    return  DATE_FORMAT.get().format(dateTime);
-  }
-
-  /**
-   * Convert time in ozone date format to millisecond.
-   * @return time in milliseconds
-   */
-  public static long formatDateTime(String date) throws ParseException {
-    Preconditions.checkNotNull(date, "Date string should not be null.");
-    return ZonedDateTime.parse(date, DATE_FORMAT.get())
-        .toInstant().getEpochSecond();
-  }
-
-  /**
-   * Returns the maximum no of outstanding async requests to be handled by
-   * Standalone and Ratis client.
-   */
-  public static int getMaxOutstandingRequests(Configuration config) {
-    return config
-        .getInt(ScmConfigKeys.SCM_CONTAINER_CLIENT_MAX_OUTSTANDING_REQUESTS,
-            ScmConfigKeys.SCM_CONTAINER_CLIENT_MAX_OUTSTANDING_REQUESTS_DEFAULT);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
deleted file mode 100644
index 0c723dd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-/**
- * A class that encapsulates OzoneKey.
- */
-public class OzoneKey {
-
-  /**
-   * Name of the Volume the Key belongs to.
-   */
-  private final String volumeName;
-  /**
-   * Name of the Bucket the Key belongs to.
-   */
-  private final String bucketName;
-  /**
-   * Name of the Key.
-   */
-  private final String name;
-  /**
-   * Size of the data.
-   */
-  private final long dataSize;
-  /**
-   * Creation time of the key.
-   */
-  private long creationTime;
-  /**
-   * Modification time of the key.
-   */
-  private long modificationTime;
-
-  /**
-   * Constructs OzoneKey from KsmKeyInfo.
-   *
-   */
-  public OzoneKey(String volumeName, String bucketName,
-                  String keyName, long size, long creationTime,
-                  long modificationTime) {
-    this.volumeName = volumeName;
-    this.bucketName = bucketName;
-    this.name = keyName;
-    this.dataSize = size;
-    this.creationTime = creationTime;
-    this.modificationTime = modificationTime;
-  }
-
-  /**
-   * Returns Volume Name associated with the Key.
-   *
-   * @return volumeName
-   */
-  public String getVolumeName() {
-    return volumeName;
-  }
-
-  /**
-   * Returns Bucket Name associated with the Key.
-   *
-   * @return bucketName
-   */
-  public String getBucketName(){
-    return bucketName;
-  }
-
-  /**
-   * Returns the Key Name.
-   *
-   * @return keyName
-   */
-  public String getName() {
-    return name;
-  }
-
-  /**
-   * Returns the size of the data.
-   *
-   * @return dataSize
-   */
-  public long getDataSize() {
-    return dataSize;
-  }
-
-  /**
-   * Returns the creation time of the key.
-   *
-   * @return creation time
-   */
-  public long getCreationTime() {
-    return creationTime;
-  }
-
-  /**
-   * Returns the modification time of the key.
-   *
-   * @return modification time
-   */
-  public long getModificationTime() {
-    return modificationTime;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneQuota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneQuota.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneQuota.java
deleted file mode 100644
index 032dd60..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneQuota.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import org.apache.hadoop.ozone.OzoneConsts;
-
-
-/**
- * represents an OzoneQuota Object that can be applied to
- * a storage volume.
- */
-public class OzoneQuota {
-
-  public static final String OZONE_QUOTA_BYTES = "BYTES";
-  public static final String OZONE_QUOTA_MB = "MB";
-  public static final String OZONE_QUOTA_GB = "GB";
-  public static final String OZONE_QUOTA_TB = "TB";
-
-  private Units unit;
-  private long size;
-
-  /** Quota Units.*/
-  public enum Units {UNDEFINED, BYTES, KB, MB, GB, TB}
-
-  /**
-   * Returns size.
-   *
-   * @return long
-   */
-  public long getSize() {
-    return size;
-  }
-
-  /**
-   * Returns Units.
-   *
-   * @return Unit in MB, GB or TB
-   */
-  public Units getUnit() {
-    return unit;
-  }
-
-  /**
-   * Constructs a default Quota object.
-   */
-  public OzoneQuota() {
-    this.size = 0;
-    this.unit = Units.UNDEFINED;
-  }
-
-  /**
-   * Constructor for Ozone Quota.
-   *
-   * @param size Long Size
-   * @param unit MB, GB  or TB
-   */
-  public OzoneQuota(long size, Units unit) {
-    this.size = size;
-    this.unit = unit;
-  }
-
-  /**
-   * Formats a quota as a string.
-   *
-   * @param quota the quota to format
-   * @return string representation of quota
-   */
-  public static String formatQuota(OzoneQuota quota) {
-    return String.valueOf(quota.size) + quota.unit;
-  }
-
-  /**
-   * Parses a user provided string and returns the
-   * Quota Object.
-   *
-   * @param quotaString Quota String
-   *
-   * @return OzoneQuota object
-   *
-   * @throws IllegalArgumentException
-   */
-  public static OzoneQuota parseQuota(String quotaString)
-      throws IllegalArgumentException {
-
-    if ((quotaString == null) || (quotaString.isEmpty())) {
-      throw new IllegalArgumentException(
-          "Quota string cannot be null or empty.");
-    }
-
-    String uppercase = quotaString.toUpperCase().replaceAll("\\s+", "");
-    String size = "";
-    int nSize;
-    Units currUnit = Units.MB;
-    Boolean found = false;
-    if (uppercase.endsWith(OZONE_QUOTA_MB)) {
-      size = uppercase
-          .substring(0, uppercase.length() - OZONE_QUOTA_MB.length());
-      currUnit = Units.MB;
-      found = true;
-    }
-
-    if (uppercase.endsWith(OZONE_QUOTA_GB)) {
-      size = uppercase
-          .substring(0, uppercase.length() - OZONE_QUOTA_GB.length());
-      currUnit = Units.GB;
-      found = true;
-    }
-
-    if (uppercase.endsWith(OZONE_QUOTA_TB)) {
-      size = uppercase
-          .substring(0, uppercase.length() - OZONE_QUOTA_TB.length());
-      currUnit = Units.TB;
-      found = true;
-    }
-
-    if (uppercase.endsWith(OZONE_QUOTA_BYTES)) {
-      size = uppercase
-          .substring(0, uppercase.length() - OZONE_QUOTA_BYTES.length());
-      currUnit = Units.BYTES;
-      found = true;
-    }
-
-    if (!found) {
-      throw new IllegalArgumentException(
-          "Quota unit not recognized. Supported values are BYTES, MB, GB and " +
-              "TB.");
-    }
-
-    nSize = Integer.parseInt(size);
-    if (nSize < 0) {
-      throw new IllegalArgumentException("Quota cannot be negative.");
-    }
-
-    return new OzoneQuota(nSize, currUnit);
-  }
-
-
-  /**
-   * Returns size in Bytes or -1 if there is no Quota.
-   */
-  public long sizeInBytes() {
-    switch (this.unit) {
-    case BYTES:
-      return this.getSize();
-    case MB:
-      return this.getSize() * OzoneConsts.MB;
-    case GB:
-      return this.getSize() * OzoneConsts.GB;
-    case TB:
-      return this.getSize() * OzoneConsts.TB;
-    case UNDEFINED:
-    default:
-      return -1;
-    }
-  }
-
-  /**
-   * Returns OzoneQuota corresponding to size in bytes.
-   *
-   * @param sizeInBytes size in bytes to be converted
-   *
-   * @return OzoneQuota object
-   */
-  public static OzoneQuota getOzoneQuota(long sizeInBytes) {
-    long size;
-    Units unit;
-    if (sizeInBytes % OzoneConsts.TB == 0) {
-      size = sizeInBytes / OzoneConsts.TB;
-      unit = Units.TB;
-    } else if (sizeInBytes % OzoneConsts.GB == 0) {
-      size = sizeInBytes / OzoneConsts.GB;
-      unit = Units.GB;
-    } else if (sizeInBytes % OzoneConsts.MB == 0) {
-      size = sizeInBytes / OzoneConsts.MB;
-      unit = Units.MB;
-    } else {
-      size = sizeInBytes;
-      unit = Units.BYTES;
-    }
-    return new OzoneQuota((int)size, unit);
-  }
-
-  @Override
-  public String toString() {
-    return size + " " + unit;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
deleted file mode 100644
index 31cbb40..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-/**
- * A class that encapsulates OzoneVolume.
- */
-public class OzoneVolume {
-
-  /**
-   * The proxy used for connecting to the cluster and perform
-   * client operations.
-   */
-  private final ClientProtocol proxy;
-
-  /**
-   * Name of the Volume.
-   */
-  private final String name;
-
-  /**
-   * Admin Name of the Volume.
-   */
-  private String admin;
-  /**
-   * Owner of the Volume.
-   */
-  private String owner;
-  /**
-   * Quota allocated for the Volume.
-   */
-  private long quotaInBytes;
-  /**
-   * Creation time of the volume.
-   */
-  private long creationTime;
-  /**
-   * Volume ACLs.
-   */
-  private List<OzoneAcl> acls;
-
-  private int listCacheSize;
-
-  /**
-   * Constructs OzoneVolume instance.
-   * @param conf Configuration object.
-   * @param proxy ClientProtocol proxy.
-   * @param name Name of the volume.
-   * @param admin Volume admin.
-   * @param owner Volume owner.
-   * @param quotaInBytes Volume quota in bytes.
-   * @param creationTime creation time of the volume
-   * @param acls ACLs associated with the volume.
-   */
-  public OzoneVolume(Configuration conf, ClientProtocol proxy, String name,
-                     String admin, String owner, long quotaInBytes,
-                     long creationTime, List<OzoneAcl> acls) {
-    this.proxy = proxy;
-    this.name = name;
-    this.admin = admin;
-    this.owner = owner;
-    this.quotaInBytes = quotaInBytes;
-    this.creationTime = creationTime;
-    this.acls = acls;
-    this.listCacheSize = OzoneClientUtils.getListCacheSize(conf);
-  }
-
-  /**
-   * Returns Volume name.
-   *
-   * @return volumeName
-   */
-  public String getName() {
-    return name;
-  }
-
-  /**
-   * Returns Volume's admin name.
-   *
-   * @return adminName
-   */
-  public String getAdmin() {
-    return admin;
-  }
-
-  /**
-   * Returns Volume's owner name.
-   *
-   * @return ownerName
-   */
-  public String getOwner() {
-    return owner;
-  }
-
-  /**
-   * Returns Quota allocated for the Volume in bytes.
-   *
-   * @return quotaInBytes
-   */
-  public long getQuota() {
-    return quotaInBytes;
-  }
-
-  /**
-   * Returns creation time of the volume.
-   *
-   * @return creation time.
-   */
-  public long getCreationTime() {
-    return creationTime;
-  }
-
-  /**
-   * Returns OzoneAcl list associated with the Volume.
-   *
-   * @return aclMap
-   */
-  public List<OzoneAcl> getAcls() {
-    return acls;
-  }
-
-  /**
-   * Sets/Changes the owner of this Volume.
-   * @param owner new owner
-   * @throws IOException
-   */
-  public void setOwner(String owner) throws IOException {
-    Preconditions.checkNotNull(proxy, "Client proxy is not set.");
-    Preconditions.checkNotNull(owner);
-    proxy.setVolumeOwner(name, owner);
-    this.owner = owner;
-  }
-
-  /**
-   * Sets/Changes the quota of this Volume.
-   * @param quota new quota
-   * @throws IOException
-   */
-  public void setQuota(OzoneQuota quota) throws IOException {
-    Preconditions.checkNotNull(proxy, "Client proxy is not set.");
-    Preconditions.checkNotNull(quota);
-    proxy.setVolumeQuota(name, quota);
-    this.quotaInBytes = quota.sizeInBytes();
-  }
-
-  /**
-   * Creates a new Bucket in this Volume, with default values.
-   * @param bucketName Name of the Bucket
-   * @throws IOException
-   */
-  public void createBucket(String bucketName)
-      throws IOException {
-    Preconditions.checkNotNull(proxy, "Client proxy is not set.");
-    Preconditions.checkNotNull(bucketName);
-    OzoneClientUtils.verifyResourceName(bucketName);
-    proxy.createBucket(name, bucketName);
-  }
-
-  /**
-   * Creates a new Bucket in this Volume, with properties set in bucketArgs.
-   * @param bucketName Name of the Bucket
-   * @param bucketArgs Properties to be set
-   * @throws IOException
-   */
-  public void createBucket(String bucketName, BucketArgs bucketArgs)
-      throws IOException {
-    Preconditions.checkNotNull(proxy, "Client proxy is not set.");
-    Preconditions.checkNotNull(bucketName);
-    Preconditions.checkNotNull(bucketArgs);
-    OzoneClientUtils.verifyResourceName(bucketName);
-    proxy.createBucket(name, bucketName, bucketArgs);
-  }
-
-  /**
-   * Get the Bucket from this Volume.
-   * @param bucketName Name of the Bucket
-   * @return OzoneBucket
-   * @throws IOException
-   */
-  public OzoneBucket getBucket(String bucketName) throws IOException {
-    Preconditions.checkNotNull(proxy, "Client proxy is not set.");
-    Preconditions.checkNotNull(bucketName);
-    OzoneClientUtils.verifyResourceName(bucketName);
-    OzoneBucket bucket = proxy.getBucketDetails(name, bucketName);
-    return bucket;
-  }
-
-  /**
-   * Returns Iterator to iterate over all buckets in the volume.
-   * The result can be restricted using bucket prefix, will return all
-   * buckets if bucket prefix is null.
-   *
-   * @param bucketPrefix Bucket prefix to match
-   * @return {@code Iterator<OzoneBucket>}
-   */
-  public Iterator<OzoneBucket> listBuckets(String bucketPrefix) {
-    return new BucketIterator(bucketPrefix);
-  }
-
-  /**
-   * Deletes the Bucket from this Volume.
-   * @param bucketName Name of the Bucket
-   * @throws IOException
-   */
-  public void deleteBucket(String bucketName) throws IOException {
-    Preconditions.checkNotNull(proxy, "Client proxy is not set.");
-    Preconditions.checkNotNull(bucketName);
-    OzoneClientUtils.verifyResourceName(bucketName);
-    proxy.deleteBucket(name, bucketName);
-  }
-
-
-  /**
-   * An Iterator to iterate over {@link OzoneBucket} list.
-   */
-  private class BucketIterator implements Iterator<OzoneBucket> {
-
-    private String bucketPrefix = null;
-
-    private Iterator<OzoneBucket> currentIterator;
-    private OzoneBucket currentValue;
-
-
-    /**
-     * Creates an Iterator to iterate over all buckets in the volume,
-     * which matches volume prefix.
-     * @param bucketPrefix
-     */
-    BucketIterator(String bucketPrefix) {
-      this.bucketPrefix = bucketPrefix;
-      this.currentValue = null;
-      this.currentIterator = getNextListOfBuckets(null).iterator();
-    }
-
-    @Override
-    public boolean hasNext() {
-      if(!currentIterator.hasNext()) {
-        currentIterator = getNextListOfBuckets(
-            currentValue != null ? currentValue.getName() : null)
-            .iterator();
-      }
-      return currentIterator.hasNext();
-    }
-
-    @Override
-    public OzoneBucket next() {
-      if(hasNext()) {
-        currentValue = currentIterator.next();
-        return currentValue;
-      }
-      throw new NoSuchElementException();
-    }
-
-    /**
-     * Gets the next set of bucket list using proxy.
-     * @param prevBucket
-     * @return {@code List<OzoneVolume>}
-     */
-    private List<OzoneBucket> getNextListOfBuckets(String prevBucket) {
-      try {
-        return proxy.listBuckets(name, bucketPrefix, prevBucket, listCacheSize);
-      } catch (IOException e) {
-        throw new RuntimeException(e);
-      }
-    }
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ReplicationFactor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ReplicationFactor.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ReplicationFactor.java
deleted file mode 100644
index 971cfec..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ReplicationFactor.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-/**
- * The replication factor to be used while writing key into ozone.
- */
-public enum ReplicationFactor {
-  ONE(1),
-  THREE(3);
-
-  /**
-   * Integer representation of replication.
-   */
-  private int value;
-
-  /**
-   * Initializes ReplicationFactor with value.
-   * @param value replication value
-   */
-  ReplicationFactor(int value) {
-    this.value = value;
-  }
-
-  /**
-   * Returns enum value corresponding to the int value.
-   * @param value replication value
-   * @return ReplicationFactor
-   */
-  public static ReplicationFactor valueOf(int value) {
-    if(value == 1) {
-      return ONE;
-    }
-    if (value == 3) {
-      return THREE;
-    }
-    throw new IllegalArgumentException("Unsupported value: " + value);
-  }
-
-  /**
-   * Returns integer representation of ReplicationFactor.
-   * @return replication value
-   */
-  public int getValue() {
-    return value;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ReplicationType.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ReplicationType.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ReplicationType.java
deleted file mode 100644
index 537c336..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ReplicationType.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-/**
- * The replication type to be used while writing key into ozone.
- */
-public enum ReplicationType {
-    RATIS,
-    STAND_ALONE,
-    CHAINED
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/VolumeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/VolumeArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/VolumeArgs.java
deleted file mode 100644
index f1aa031..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/VolumeArgs.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * This class encapsulates the arguments that are
- * required for creating a volume.
- */
-public final class VolumeArgs {
-
-  private final String admin;
-  private final String owner;
-  private final String quota;
-  private final List<OzoneAcl> acls;
-
-  /**
-   * Private constructor, constructed via builder.
-   * @param admin Administrator's name.
-   * @param owner Volume owner's name
-   * @param quota Volume Quota.
-   * @param acls User to access rights map.
-   */
-  private VolumeArgs(String admin, String owner,
-                        String quota, List<OzoneAcl> acls) {
-    this.admin = admin;
-    this.owner = owner;
-    this.quota = quota;
-    this.acls = acls;
-  }
-
-  /**
-   * Returns the Admin Name.
-   * @return String.
-   */
-  public String getAdmin() {
-    return admin;
-  }
-
-  /**
-   * Returns the owner Name.
-   * @return String
-   */
-  public String getOwner() {
-    return owner;
-  }
-
-  /**
-   * Returns Volume Quota.
-   * @return Quota.
-   */
-  public String getQuota() {
-    return quota;
-  }
-
-  public List<OzoneAcl> getAcls() {
-    return acls;
-  }
-  /**
-   * Returns new builder class that builds a KsmVolumeArgs.
-   *
-   * @return Builder
-   */
-  public static VolumeArgs.Builder newBuilder() {
-    return new VolumeArgs.Builder();
-  }
-
-  /**
-   * Builder for KsmVolumeArgs.
-   */
-  public static class Builder {
-    private String adminName;
-    private String ownerName;
-    private String volumeQuota;
-    private List<OzoneAcl> listOfAcls;
-
-
-    public VolumeArgs.Builder setAdmin(String admin) {
-      this.adminName = admin;
-      return this;
-    }
-
-    public VolumeArgs.Builder setOwner(String owner) {
-      this.ownerName = owner;
-      return this;
-    }
-
-    public VolumeArgs.Builder setQuota(String quota) {
-      this.volumeQuota = quota;
-      return this;
-    }
-
-    public VolumeArgs.Builder setAcls(List<OzoneAcl> acls)
-        throws IOException {
-      this.listOfAcls = acls;
-      return this;
-    }
-
-    /**
-     * Constructs a CreateVolumeArgument.
-     * @return CreateVolumeArgs.
-     */
-    public VolumeArgs build() {
-      return new VolumeArgs(adminName, ownerName, volumeQuota, listOfAcls);
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
deleted file mode 100644
index afe5e45..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
+++ /dev/null
@@ -1,319 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.client.io;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.fs.Seekable;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.storage.ChunkInputStream;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.apache.ratis.util.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Maintaining a list of ChunkInputStream. Read based on offset.
- */
-public class ChunkGroupInputStream extends InputStream implements Seekable {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ChunkGroupInputStream.class);
-
-  private static final int EOF = -1;
-
-  private final ArrayList<ChunkInputStreamEntry> streamEntries;
-  // streamOffset[i] stores the offset at which chunkInputStream i stores
-  // data in the key
-  private long[] streamOffset = null;
-  private int currentStreamIndex;
-  private long length = 0;
-  private boolean closed = false;
-  private String key;
-
-  public ChunkGroupInputStream() {
-    streamEntries = new ArrayList<>();
-    currentStreamIndex = 0;
-  }
-
-  @VisibleForTesting
-  public synchronized int getCurrentStreamIndex() {
-    return currentStreamIndex;
-  }
-
-  @VisibleForTesting
-  public long getRemainingOfIndex(int index) {
-    return streamEntries.get(index).getRemaining();
-  }
-
-  /**
-   * Append another stream to the end of the list.
-   *
-   * @param stream       the stream instance.
-   * @param streamLength the max number of bytes that should be written to this
-   *                     stream.
-   */
-  public synchronized void addStream(ChunkInputStream stream,
-      long streamLength) {
-    streamEntries.add(new ChunkInputStreamEntry(stream, streamLength));
-  }
-
-
-  @Override
-  public synchronized int read() throws IOException {
-    byte[] buf = new byte[1];
-    if (read(buf, 0, 1) == EOF) {
-      return EOF;
-    }
-    return Byte.toUnsignedInt(buf[0]);
-  }
-
-  @Override
-  public synchronized int read(byte[] b, int off, int len) throws IOException {
-    checkNotClosed();
-    if (b == null) {
-      throw new NullPointerException();
-    }
-    if (off < 0 || len < 0 || len > b.length - off) {
-      throw new IndexOutOfBoundsException();
-    }
-    if (len == 0) {
-      return 0;
-    }
-    int totalReadLen = 0;
-    while (len > 0) {
-      if (streamEntries.size() <= currentStreamIndex) {
-        return totalReadLen == 0 ? EOF : totalReadLen;
-      }
-      ChunkInputStreamEntry current = streamEntries.get(currentStreamIndex);
-      int readLen = Math.min(len, (int)current.getRemaining());
-      int actualLen = current.read(b, off, readLen);
-      // this means the underlying stream has nothing at all, return
-      if (actualLen == EOF) {
-        return totalReadLen > 0 ? totalReadLen : EOF;
-      }
-      totalReadLen += actualLen;
-      // this means there is no more data to read beyond this point, return
-      if (actualLen != readLen) {
-        return totalReadLen;
-      }
-      off += readLen;
-      len -= readLen;
-      if (current.getRemaining() <= 0) {
-        currentStreamIndex += 1;
-      }
-    }
-    return totalReadLen;
-  }
-
-  @Override
-  public void seek(long pos) throws IOException {
-    checkNotClosed();
-    if (pos < 0 || pos >= length) {
-      if (pos == 0) {
-        // It is possible for length and pos to be zero in which case
-        // seek should return instead of throwing exception
-        return;
-      }
-      throw new EOFException(
-          "EOF encountered at pos: " + pos + " for key: " + key);
-    }
-    Preconditions.assertTrue(currentStreamIndex >= 0);
-    if (currentStreamIndex >= streamEntries.size()) {
-      currentStreamIndex = Arrays.binarySearch(streamOffset, pos);
-    } else if (pos < streamOffset[currentStreamIndex]) {
-      currentStreamIndex =
-          Arrays.binarySearch(streamOffset, 0, currentStreamIndex, pos);
-    } else if (pos >= streamOffset[currentStreamIndex] + streamEntries
-        .get(currentStreamIndex).length) {
-      currentStreamIndex = Arrays
-          .binarySearch(streamOffset, currentStreamIndex + 1,
-              streamEntries.size(), pos);
-    }
-    if (currentStreamIndex < 0) {
-      // Binary search returns -insertionPoint - 1  if element is not present
-      // in the array. insertionPoint is the point at which element would be
-      // inserted in the sorted array. We need to adjust the currentStreamIndex
-      // accordingly so that currentStreamIndex = insertionPoint - 1
-      currentStreamIndex = -currentStreamIndex - 2;
-    }
-    // seek to the proper offset in the ChunkInputStream
-    streamEntries.get(currentStreamIndex)
-        .seek(pos - streamOffset[currentStreamIndex]);
-  }
-
-  @Override
-  public long getPos() throws IOException {
-    return length == 0 ? 0 :
-        streamOffset[currentStreamIndex] + streamEntries.get(currentStreamIndex)
-            .getPos();
-  }
-
-  @Override
-  public boolean seekToNewSource(long targetPos) throws IOException {
-    return false;
-  }
-
-  @Override
-  public int available() throws IOException {
-    checkNotClosed();
-    long remaining = length - getPos();
-    return remaining <= Integer.MAX_VALUE ? (int) remaining : Integer.MAX_VALUE;
-  }
-
-  @Override
-  public void close() throws IOException {
-    closed = true;
-    for (int i = 0; i < streamEntries.size(); i++) {
-      streamEntries.get(i).close();
-    }
-  }
-
-  /**
-   * Encapsulates ChunkInputStream.
-   */
-  public static class ChunkInputStreamEntry extends InputStream
-      implements Seekable {
-
-    private final ChunkInputStream chunkInputStream;
-    private final long length;
-    private long currentPosition;
-
-    public ChunkInputStreamEntry(ChunkInputStream chunkInputStream,
-        long length) {
-      this.chunkInputStream = chunkInputStream;
-      this.length = length;
-      this.currentPosition = 0;
-    }
-
-    synchronized long getRemaining() {
-      return length - currentPosition;
-    }
-
-    @Override
-    public synchronized int read(byte[] b, int off, int len)
-        throws IOException {
-      int readLen = chunkInputStream.read(b, off, len);
-      currentPosition += readLen;
-      return readLen;
-    }
-
-    @Override
-    public synchronized int read() throws IOException {
-      int data = chunkInputStream.read();
-      currentPosition += 1;
-      return data;
-    }
-
-    @Override
-    public synchronized void close() throws IOException {
-      chunkInputStream.close();
-    }
-
-    @Override
-    public void seek(long pos) throws IOException {
-      chunkInputStream.seek(pos);
-    }
-
-    @Override
-    public long getPos() throws IOException {
-      return chunkInputStream.getPos();
-    }
-
-    @Override
-    public boolean seekToNewSource(long targetPos) throws IOException {
-      return false;
-    }
-  }
-
-  public static LengthInputStream getFromKsmKeyInfo(KsmKeyInfo keyInfo,
-      XceiverClientManager xceiverClientManager,
-      StorageContainerLocationProtocolClientSideTranslatorPB
-          storageContainerLocationClient, String requestId)
-      throws IOException {
-    long length = 0;
-    String containerKey;
-    ChunkGroupInputStream groupInputStream = new ChunkGroupInputStream();
-    groupInputStream.key = keyInfo.getKeyName();
-    List<KsmKeyLocationInfo> keyLocationInfos =
-        keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly();
-    groupInputStream.streamOffset = new long[keyLocationInfos.size()];
-    for (int i = 0; i < keyLocationInfos.size(); i++) {
-      KsmKeyLocationInfo ksmKeyLocationInfo = keyLocationInfos.get(i);
-      String containerName = ksmKeyLocationInfo.getContainerName();
-      Pipeline pipeline =
-          storageContainerLocationClient.getContainer(containerName);
-      XceiverClientSpi xceiverClient =
-          xceiverClientManager.acquireClient(pipeline);
-      boolean success = false;
-      containerKey = ksmKeyLocationInfo.getBlockID();
-      try {
-        LOG.debug("get key accessing {} {}",
-            xceiverClient.getPipeline().getContainerName(), containerKey);
-        groupInputStream.streamOffset[i] = length;
-        ContainerProtos.KeyData containerKeyData = OzoneContainerTranslation
-            .containerKeyDataForRead(
-                xceiverClient.getPipeline().getContainerName(), containerKey);
-        ContainerProtos.GetKeyResponseProto response = ContainerProtocolCalls
-            .getKey(xceiverClient, containerKeyData, requestId);
-        List<ContainerProtos.ChunkInfo> chunks =
-            response.getKeyData().getChunksList();
-        for (ContainerProtos.ChunkInfo chunk : chunks) {
-          length += chunk.getLen();
-        }
-        success = true;
-        ChunkInputStream inputStream = new ChunkInputStream(
-            containerKey, xceiverClientManager, xceiverClient,
-            chunks, requestId);
-        groupInputStream.addStream(inputStream,
-            ksmKeyLocationInfo.getLength());
-      } finally {
-        if (!success) {
-          xceiverClientManager.releaseClient(xceiverClient);
-        }
-      }
-    }
-    groupInputStream.length = length;
-    return new LengthInputStream(groupInputStream, length);
-  }
-
-  /**
-   * Verify that the input stream is open. Non blocking; this gives
-   * the last state of the volatile {@link #closed} field.
-   * @throws IOException if the connection is closed.
-   */
-  private void checkNotClosed() throws IOException {
-    if (closed) {
-      throw new IOException(
-          ": " + FSExceptionMessages.STREAM_IS_CLOSED + " Key: " + key);
-    }
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[48/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java
new file mode 100644
index 0000000..26c174f
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
+
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import java.nio.ByteBuffer;
+
+/**
+ * Impl class for LogicalBlock.
+ */
+public class DiskBlock implements LogicalBlock {
+  private ByteBuffer data;
+  private long blockID;
+  private boolean persisted;
+
+  /**
+   * Constructs a DiskBlock Class from the following params.
+   * @param blockID - 64-bit block ID
+   * @param data - Byte Array
+   * @param persisted - Flag which tells us if this is persisted to remote
+   */
+  public DiskBlock(long blockID, byte[] data, boolean persisted) {
+    if (data !=null) {
+      this.data = ByteBuffer.wrap(data);
+    }
+    this.blockID = blockID;
+    this.persisted = persisted;
+  }
+
+  @Override
+  public ByteBuffer getData() {
+    return data;
+  }
+
+  /**
+   * Frees the byte buffer since we don't need it any more.
+   */
+  @Override
+  public void clearData() {
+    data.clear();
+  }
+
+  @Override
+  public long getBlockID() {
+    return blockID;
+  }
+
+  @Override
+  public boolean isPersisted() {
+    return persisted;
+  }
+
+  /**
+   * Sets the value of persisted.
+   * @param value - True if this has been persisted to container, false
+   * otherwise.
+   */
+  public void setPersisted(boolean value) {
+    persisted = value;
+  }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java
new file mode 100644
index 0000000..7d6e172
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
+
+import com.google.common.primitives.Longs;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdsl.protocol.proto.ContainerProtos;
+import org.apache.hadoop.scm.XceiverClientSpi;
+import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.util.concurrent.HadoopThreadPoolExecutor;
+import org.apache.hadoop.utils.LevelDBStore;
+import org.iq80.leveldb.DBException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Reads blocks from the container via the local cache.
+ */
+public class SyncBlockReader {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(SyncBlockReader.class);
+
+  /**
+   * Update Queue - The reason why we have the queue is that we want to
+   * return the block as soon as we read it from the containers. This queue
+   * is work queue which will take the read block and update the cache.
+   * During testing we found levelDB is slow during writes, hence we wanted
+   * to return as block as soon as possible and update levelDB asynchronously.
+   */
+  private final static int QUEUE_SIZE = 1024;
+  /**
+   * Config.
+   */
+  private final Configuration conf;
+  /**
+   * The parent cache this reader is operating against.
+   */
+  private final CBlockLocalCache parentCache;
+  private final BlockingQueue<Runnable> updateQueue;
+
+  /**
+   * executor is used for running LevelDB updates. In future, we might do
+   * read-aheads and this pool is useful for that too. The reason why we
+   * don't share an executor for reads and writes is because the write task
+   * is couple of magnitude slower than read task. So we don't want the
+   * update DB to queue up behind the writes.
+   */
+  private final ThreadPoolExecutor executor;
+
+  /**
+   * Number of threads that pool starts with.
+   */
+  private final int corePoolSize = 1;
+  /**
+   * Maximum number of threads our pool will ever create.
+   */
+  private final int maxPoolSize = 10;
+  /**
+   * The idle time a thread hangs around waiting for work. if we don't find
+   * new work in 60 seconds the worker thread is killed.
+   */
+  private final long keepAlive = 60L;
+
+  /**
+   * Constructs a SyncBlock reader.
+   *
+   * @param conf - Configuration
+   * @param cache - Cache
+   */
+  public SyncBlockReader(Configuration conf, CBlockLocalCache cache) {
+    this.conf = conf;
+    this.parentCache = cache;
+    updateQueue = new ArrayBlockingQueue<>(QUEUE_SIZE, true);
+    ThreadFactory workerThreadFactory = new ThreadFactoryBuilder()
+        .setNameFormat("SyncBlockReader Thread #%d").setDaemon(true).build();
+    executor = new HadoopThreadPoolExecutor(
+        corePoolSize, maxPoolSize, keepAlive, TimeUnit.SECONDS,
+        updateQueue, workerThreadFactory,
+        new ThreadPoolExecutor.CallerRunsPolicy());
+  }
+
+  /**
+   * Returns the cache DB.
+   *
+   * @return LevelDB
+   */
+  private LevelDBStore getCacheDB() {
+    return parentCache.getCacheDB();
+  }
+
+  /**
+   * Returns data from the local cache if found, else reads from the remote
+   * container.
+   *
+   * @param blockID - blockID
+   * @return LogicalBlock
+   */
+  LogicalBlock readBlock(long blockID) throws IOException {
+    XceiverClientSpi client = null;
+    byte[] data = getblockFromDB(blockID);
+    if (data != null) {
+      parentCache.getTargetMetrics().incNumReadCacheHits();
+      return new DiskBlock(blockID, data, false);
+    }
+
+    parentCache.getTargetMetrics().incNumReadCacheMiss();
+    try {
+      client = parentCache.getClientManager()
+          .acquireClient(parentCache.getPipeline(blockID));
+      LogicalBlock block = getBlockFromContainer(blockID, client);
+      return block;
+    } catch (Exception ex) {
+      parentCache.getTargetMetrics().incNumFailedReadBlocks();
+      LOG.error("read failed for BlockId: {}", blockID, ex);
+      throw ex;
+    } finally {
+      if (client != null) {
+        parentCache.getClientManager().releaseClient(client);
+      }
+    }
+  }
+
+  /**
+   * Gets data from the DB if it exists.
+   *
+   * @param blockID - block id
+   * @return data
+   */
+  private byte[] getblockFromDB(long blockID) {
+    try {
+      if(parentCache.isShortCircuitIOEnabled()) {
+        long startTime = Time.monotonicNow();
+        byte[] data = getCacheDB().get(Longs.toByteArray(blockID));
+        long endTime = Time.monotonicNow();
+
+        if (parentCache.isTraceEnabled()) {
+          parentCache.getTracer().info(
+              "Task=ReadTaskDBRead,BlockID={},SHA={},Time={}",
+              blockID, (data != null && data.length > 0)
+                  ? DigestUtils.sha256Hex(data) : null,
+              endTime - startTime);
+        }
+        parentCache.getTargetMetrics().updateDBReadLatency(
+            endTime - startTime);
+        return data;
+      }
+
+
+    } catch (DBException dbe) {
+      LOG.error("Error while reading from cacheDB.", dbe);
+      throw dbe;
+    }
+    return null;
+  }
+
+
+  /**
+   * Returns a block from a Remote Container. if the key is not found on a
+   * remote container we just return a block initialzied with zeros.
+   *
+   * @param blockID - blockID
+   * @param client - client
+   * @return LogicalBlock
+   * @throws IOException
+   */
+  private LogicalBlock getBlockFromContainer(long blockID,
+      XceiverClientSpi client) throws IOException {
+    String containerName = parentCache.getPipeline(blockID).getContainerName();
+    try {
+      long startTime = Time.monotonicNow();
+      ContainerProtos.GetSmallFileResponseProto response =
+          ContainerProtocolCalls.readSmallFile(client, containerName,
+              Long.toString(blockID), parentCache.getTraceID(blockID));
+      long endTime = Time.monotonicNow();
+      if (parentCache.isTraceEnabled()) {
+        parentCache.getTracer().info(
+            "Task=ReadTaskContainerRead,BlockID={},SHA={},Time={}",
+            blockID, response.getData().getData().toByteArray().length > 0 ?
+                DigestUtils.sha256Hex(response.getData()
+                    .getData().toByteArray()) : null, endTime - startTime);
+      }
+
+      parentCache.getTargetMetrics().updateContainerReadLatency(
+          endTime - startTime);
+      DiskBlock block = new DiskBlock(blockID,
+          response.getData().getData().toByteArray(), false);
+
+      if(parentCache.isShortCircuitIOEnabled()) {
+        queueUpdateTask(block);
+      }
+
+      return block;
+    } catch (IOException ex) {
+      if (ex instanceof StorageContainerException) {
+        parentCache.getTargetMetrics().incNumReadLostBlocks();
+        StorageContainerException sce = (StorageContainerException) ex;
+        if (sce.getResult() == ContainerProtos.Result.NO_SUCH_KEY ||
+            sce.getResult() == ContainerProtos.Result.IO_EXCEPTION) {
+          return new DiskBlock(blockID, new byte[parentCache.getBlockSize()],
+              false);
+        }
+      }
+      throw ex;
+    }
+  }
+
+  /**
+   * Updates the cache with the block that we just read.
+   *
+   * @param block
+   */
+  private void queueUpdateTask(final DiskBlock block) {
+    Runnable updateTask = () -> {
+      if(block.getData().array().length > 0) {
+        getCacheDB().put(Longs.toByteArray(block.getBlockID()),
+            block.getData().array());
+        block.setPersisted(true);
+      } else {
+        LOG.error("Refusing to update the a null block in the local cache.");
+      }
+    };
+    if (this.executor.isShutdown() || this.executor.isTerminated()) {
+      LOG.error("Thread executor is not running.");
+    } else {
+      this.executor.submit(updateTask);
+    }
+  }
+
+  /**
+   * This is a read operation, we don't care if we updated the cache with the
+   * last block e read.
+   */
+  void shutdown() {
+    this.executor.shutdownNow();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java
new file mode 100644
index 0000000..dfac110
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper.cache.impl;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java
new file mode 100644
index 0000000..47f76b8
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper.cache;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java
new file mode 100644
index 0000000..85f8d6f
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java
new file mode 100644
index 0000000..72b215b
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.cblock.kubernetes;
+
+import com.google.gson.reflect.TypeToken;
+import com.squareup.okhttp.RequestBody;
+import io.kubernetes.client.ApiClient;
+import io.kubernetes.client.ApiException;
+import io.kubernetes.client.Configuration;
+import io.kubernetes.client.apis.CoreV1Api;
+import io.kubernetes.client.models.V1ISCSIVolumeSource;
+import io.kubernetes.client.models.V1ObjectMeta;
+import io.kubernetes.client.models.V1ObjectReference;
+import io.kubernetes.client.models.V1PersistentVolume;
+import io.kubernetes.client.models.V1PersistentVolumeClaim;
+import io.kubernetes.client.models.V1PersistentVolumeSpec;
+import io.kubernetes.client.util.Config;
+import io.kubernetes.client.util.Watch;
+import okio.Buffer;
+
+import org.apache.hadoop.cblock.CblockUtils;
+import org.apache.hadoop.cblock.exception.CBlockException;
+import org.apache.hadoop.cblock.proto.MountVolumeResponse;
+import org.apache.hadoop.cblock.storage.StorageManager;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.ratis.shaded.com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.SocketTimeoutException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_ISCSI_ADVERTISED_IP;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_ISCSI_ADVERTISED_PORT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_KUBERNETES_CBLOCK_USER;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY;
+
+/**
+ * Kubernetes Dynamic Persistent Volume provisioner.
+ *
+ * Listens on the kubernetes feed and creates the appropriate cblock AND
+ * kubernetes PersistentVolume according to the created PersistentVolumeClaims.
+ */
+public class DynamicProvisioner implements Runnable{
+
+  protected static final Logger LOGGER =
+      LoggerFactory.getLogger(DynamicProvisioner.class);
+
+  private static final String STORAGE_CLASS = "cblock";
+
+  private static final String PROVISIONER_ID = "hadoop.apache.org/cblock";
+  private static final String KUBERNETES_PROVISIONER_KEY =
+      "volume.beta.kubernetes.io/storage-provisioner";
+  private static final String KUBERNETES_BIND_COMPLETED_KEY =
+      "pv.kubernetes.io/bind-completed";
+
+  private boolean running = true;
+
+  private final StorageManager storageManager;
+
+  private String kubernetesConfigFile;
+
+  private String externalIp;
+
+  private int externalPort;
+
+  private String cblockUser;
+
+  private CoreV1Api api;
+
+  private ApiClient client;
+
+  private Thread watcherThread;
+
+  public  DynamicProvisioner(OzoneConfiguration ozoneConf,
+      StorageManager storageManager) throws IOException {
+    this.storageManager = storageManager;
+
+    kubernetesConfigFile = ozoneConf
+        .getTrimmed(DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY);
+
+    String jscsiServerAddress = ozoneConf
+        .get(DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY,
+            DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT);
+
+    externalIp = ozoneConf.
+        getTrimmed(DFS_CBLOCK_ISCSI_ADVERTISED_IP, jscsiServerAddress);
+
+    externalPort = ozoneConf.
+        getInt(DFS_CBLOCK_ISCSI_ADVERTISED_PORT,
+            DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT);
+
+    cblockUser = ozoneConf.getTrimmed(DFS_CBLOCK_KUBERNETES_CBLOCK_USER,
+        DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT);
+
+
+  }
+
+  public void init() throws IOException {
+    if (kubernetesConfigFile != null) {
+      client = Config.fromConfig(kubernetesConfigFile);
+    } else {
+      client = Config.fromCluster();
+    }
+    client.getHttpClient().setReadTimeout(60, TimeUnit.SECONDS);
+    Configuration.setDefaultApiClient(client);
+    api = new CoreV1Api();
+
+    watcherThread = new Thread(this);
+    watcherThread.setName("DynamicProvisioner");
+    watcherThread.setDaemon(true);
+  }
+
+  @Override
+  public void run() {
+    LOGGER.info("Starting kubernetes dynamic provisioner.");
+    while (running) {
+      String resourceVersion = null;
+      try {
+
+        Watch<V1PersistentVolumeClaim> watch = Watch.createWatch(client,
+            api.listPersistentVolumeClaimForAllNamespacesCall(null,
+                null,
+                false,
+                null,
+                null,
+                null,
+                resourceVersion,
+                null,
+                true,
+                null,
+                null),
+            new TypeToken<Watch.Response<V1PersistentVolumeClaim>>() {
+            }.getType());
+
+
+        //check the new pvc resources, and create cblock + pv if needed
+        for (Watch.Response<V1PersistentVolumeClaim> item : watch) {
+          V1PersistentVolumeClaim claim = item.object;
+
+          if (isPvMissingForPvc(claim)) {
+
+            LOGGER.info("Provisioning volumes for PVC {}/{}",
+                claim.getMetadata().getNamespace(),
+                claim.getMetadata().getName());
+
+            if (LOGGER.isDebugEnabled()) {
+              RequestBody request =
+                  api.getApiClient().serialize(claim, "application/json");
+
+              final Buffer buffer = new Buffer();
+              request.writeTo(buffer);
+              LOGGER.debug("New PVC is detected: " + buffer.readUtf8());
+            }
+
+            String volumeName = createVolumeName(claim);
+
+            long size = CblockUtils.parseSize(
+                claim.getSpec().getResources().getRequests().get("storage"));
+
+            createCBlock(volumeName, size);
+            createPersistentVolumeFromPVC(item.object, volumeName);
+          }
+        }
+      } catch (Exception ex) {
+        if (ex.getCause() != null && ex
+            .getCause() instanceof SocketTimeoutException) {
+          //This is normal. We are connection to the kubernetes server and the
+          //connection should be reopened time to time...
+          LOGGER.debug("Time exception occured", ex);
+        } else {
+          LOGGER.error("Error on provisioning persistent volumes.", ex);
+          try {
+            //we can try again in the main loop
+            Thread.sleep(1000);
+          } catch (InterruptedException e) {
+            LOGGER.error("Error on sleeping after an error.", e);
+          }
+        }
+      }
+    }
+  }
+
+  private boolean isPvMissingForPvc(V1PersistentVolumeClaim claim) {
+
+    Map<String, String> annotations = claim.getMetadata().getAnnotations();
+
+    return claim.getStatus().getPhase().equals("Pending") && STORAGE_CLASS
+        .equals(claim.getSpec().getStorageClassName()) && PROVISIONER_ID
+        .equals(annotations.get(KUBERNETES_PROVISIONER_KEY)) && !"yes"
+        .equals(annotations.get(KUBERNETES_BIND_COMPLETED_KEY));
+  }
+
+  @VisibleForTesting
+  protected String createVolumeName(V1PersistentVolumeClaim claim) {
+    return claim.getMetadata().getName() + "-" + claim.getMetadata()
+        .getUid();
+  }
+
+  public void stop() {
+    running = false;
+    try {
+      watcherThread.join(60000);
+    } catch (InterruptedException e) {
+      LOGGER.error("Kubernetes watcher thread can't stopped gracefully.", e);
+    }
+  }
+
+  private void createCBlock(String volumeName, long size)
+      throws CBlockException {
+
+    MountVolumeResponse mountVolumeResponse =
+        storageManager.isVolumeValid(cblockUser, volumeName);
+    if (!mountVolumeResponse.getIsValid()) {
+      storageManager
+          .createVolume(cblockUser, volumeName, size, 4 * 1024);
+    }
+  }
+
+  private void createPersistentVolumeFromPVC(V1PersistentVolumeClaim claim,
+      String volumeName) throws ApiException, IOException {
+
+    V1PersistentVolume v1PersistentVolume =
+        persitenceVolumeBuilder(claim, volumeName);
+
+    if (LOGGER.isDebugEnabled()) {
+      RequestBody request =
+          api.getApiClient().serialize(v1PersistentVolume, "application/json");
+
+      final Buffer buffer = new Buffer();
+      request.writeTo(buffer);
+      LOGGER.debug("Creating new PV: " + buffer.readUtf8());
+    }
+    api.createPersistentVolume(v1PersistentVolume, null);
+  }
+
+  protected V1PersistentVolume persitenceVolumeBuilder(
+      V1PersistentVolumeClaim claim,
+      String volumeName) {
+
+    V1PersistentVolume v1PersistentVolume = new V1PersistentVolume();
+    v1PersistentVolume.setKind("PersistentVolume");
+    v1PersistentVolume.setApiVersion("v1");
+
+    V1ObjectMeta metadata = new V1ObjectMeta();
+    metadata.setName(volumeName);
+    metadata.setNamespace(claim.getMetadata().getNamespace());
+    metadata.setAnnotations(new HashMap<>());
+
+    metadata.getAnnotations()
+        .put("pv.kubernetes.io/provisioned-by", PROVISIONER_ID);
+
+    metadata.getAnnotations()
+        .put("volume.beta.kubernetes.io/storage-class", STORAGE_CLASS);
+
+    v1PersistentVolume.setMetadata(metadata);
+
+    V1PersistentVolumeSpec spec = new V1PersistentVolumeSpec();
+
+    spec.setCapacity(new HashMap<>());
+    spec.getCapacity().put("storage",
+        claim.getSpec().getResources().getRequests().get("storage"));
+
+    spec.setAccessModes(new ArrayList<>());
+    spec.getAccessModes().add("ReadWriteOnce");
+
+    V1ObjectReference claimRef = new V1ObjectReference();
+    claimRef.setName(claim.getMetadata().getName());
+    claimRef.setNamespace(claim.getMetadata().getNamespace());
+    claimRef.setKind(claim.getKind());
+    claimRef.setApiVersion(claim.getApiVersion());
+    claimRef.setUid(claim.getMetadata().getUid());
+    spec.setClaimRef(claimRef);
+
+    spec.persistentVolumeReclaimPolicy("Delete");
+
+    V1ISCSIVolumeSource iscsi = new V1ISCSIVolumeSource();
+    iscsi.setIqn(cblockUser + ":" + volumeName);
+    iscsi.setLun(0);
+    iscsi.setFsType("ext4");
+    String portal = externalIp + ":" + externalPort;
+    iscsi.setTargetPortal(portal);
+    iscsi.setPortals(new ArrayList<>());
+    iscsi.getPortals().add(portal);
+
+    spec.iscsi(iscsi);
+    v1PersistentVolume.setSpec(spec);
+    return v1PersistentVolume;
+  }
+
+
+  @VisibleForTesting
+  protected CoreV1Api getApi() {
+    return api;
+  }
+
+  public void start() {
+    watcherThread.start();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java
new file mode 100644
index 0000000..3ec5aab
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains helper classes to run hadoop cluster in kubernetes
+ * environment.
+ */
+package org.apache.hadoop.cblock.kubernetes;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java
new file mode 100644
index 0000000..00064a6
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.meta;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+
+/**
+ *
+ * The internal representation of a container maintained by CBlock server.
+ * Include enough information to exactly identify a container for read/write
+ * operation.
+ *
+ * NOTE that this class is work-in-progress. Depends on HDFS-7240 container
+ * implementation. Currently only to allow testing.
+ */
+public class ContainerDescriptor {
+  private final String containerID;
+  // the index of this container with in a volume
+  // on creation, there may be no way to know the index of the container
+  // as it is a volume specific information
+  private int containerIndex;
+  private Pipeline pipeline;
+
+  public ContainerDescriptor(String containerID) {
+    this.containerID = containerID;
+  }
+
+  public ContainerDescriptor(String containerID, int containerIndex) {
+    this.containerID = containerID;
+    this.containerIndex = containerIndex;
+  }
+
+  public void setContainerIndex(int idx) {
+    this.containerIndex = idx;
+  }
+
+  public String getContainerID() {
+    return containerID;
+  }
+
+  public void setPipeline(Pipeline pipeline) {
+    this.pipeline = pipeline;
+  }
+
+  public Pipeline getPipeline() {
+    return pipeline;
+  }
+
+  public int getContainerIndex() {
+    return containerIndex;
+  }
+
+  public long getUtilization() {
+    return 0;
+  }
+
+  public CBlockClientServerProtocolProtos.ContainerIDProto toProtobuf() {
+    CBlockClientServerProtocolProtos.ContainerIDProto.Builder builder =
+        CBlockClientServerProtocolProtos.ContainerIDProto.newBuilder();
+    builder.setContainerID(containerID);
+    builder.setIndex(containerIndex);
+    if (pipeline != null) {
+      builder.setPipeline(pipeline.getProtobufMessage());
+    }
+    return builder.build();
+  }
+
+  public static ContainerDescriptor fromProtobuf(byte[] data)
+      throws InvalidProtocolBufferException {
+    CBlockClientServerProtocolProtos.ContainerIDProto id =
+        CBlockClientServerProtocolProtos.ContainerIDProto.parseFrom(data);
+    return new ContainerDescriptor(id.getContainerID(),
+        (int)id.getIndex());
+  }
+
+  @Override
+  public int hashCode() {
+    return containerID.hashCode()*37 + containerIndex;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (o != null && o instanceof ContainerDescriptor) {
+      ContainerDescriptor other = (ContainerDescriptor)o;
+      return containerID.equals(other.containerID) &&
+          containerIndex == other.containerIndex;
+    }
+    return false;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java
new file mode 100644
index 0000000..4f5930d
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.meta;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * The internal representation maintained by CBlock server as the info for
+ * a volume. Contains the list of containers belonging to this volume.
+ *
+ * Many methods of this class is made such that the volume information (
+ * including container list) can be easily transformed into a Json string
+ * that can be stored/parsed from a persistent store for cblock server
+ * persistence.
+ *
+ * This class is still work-in-progress.
+ */
+public class VolumeDescriptor {
+  // The main data structure is the container location map
+  // other thing are mainly just information
+
+  // since only one operation at a time is allowed, no
+  // need to consider concurrency control here
+
+  // key is container id
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(VolumeDescriptor.class);
+
+  private ConcurrentHashMap<String, ContainerDescriptor> containerMap;
+  private String userName;
+  private int blockSize;
+  private long volumeSize;
+  private String volumeName;
+  // this is essentially the ordered keys of containerMap
+  // which is kind of redundant information. But since we
+  // are likely to access it frequently based on ordering.
+  // keeping this copy to avoid having to sort the key every
+  // time
+  private List<String> containerIdOrdered;
+
+  /**
+   * This is not being called explicitly, but this is necessary as
+   * it will be called by the parse method implicitly when
+   * reconstructing the object from json string. The get*() methods
+   * and set*() methods are for the same purpose also.
+   */
+  public VolumeDescriptor() {
+    this(null, null, 0, 0);
+  }
+
+  public VolumeDescriptor(String userName, String volumeName, long volumeSize,
+      int blockSize) {
+    this.containerMap = new ConcurrentHashMap<>();
+    this.userName = userName;
+    this.volumeName = volumeName;
+    this.blockSize = blockSize;
+    this.volumeSize = volumeSize;
+    this.containerIdOrdered = new LinkedList<>();
+  }
+
+  public String getUserName() {
+    return userName;
+  }
+
+  public void setUserName(String userName) {
+    this.userName = userName;
+  }
+
+  public String getVolumeName() {
+    return volumeName;
+  }
+
+  public void setVolumeName(String volumeName) {
+    this.volumeName = volumeName;
+  }
+
+  public long getVolumeSize() {
+    return volumeSize;
+  }
+
+  public void setVolumeSize(long volumeSize) {
+    this.volumeSize = volumeSize;
+  }
+
+  public int getBlockSize() {
+    return blockSize;
+  }
+
+  public void setBlockSize(int blockSize) {
+    this.blockSize = blockSize;
+  }
+
+  public void setContainerIDs(ArrayList<String> containerIDs) {
+    containerIdOrdered.addAll(containerIDs);
+  }
+
+  public void addContainer(ContainerDescriptor containerDescriptor) {
+    containerMap.put(containerDescriptor.getContainerID(),
+        containerDescriptor);
+  }
+
+
+  public HashMap<String, Pipeline> getPipelines() {
+    HashMap<String, Pipeline> pipelines = new HashMap<>();
+    for (Map.Entry<String, ContainerDescriptor> entry :
+        containerMap.entrySet()) {
+      pipelines.put(entry.getKey(), entry.getValue().getPipeline());
+    }
+    return pipelines;
+  }
+
+  public boolean isEmpty() {
+    VolumeInfo info = getInfo();
+    return info.getUsage() == 0;
+  }
+
+  public VolumeInfo getInfo() {
+    // TODO : need to actually go through all containers of this volume and
+    // ask for their utilization.
+    long utilization = 0;
+    for (Map.Entry<String, ContainerDescriptor> entry :
+        containerMap.entrySet()) {
+      utilization += entry.getValue().getUtilization();
+    }
+    return new VolumeInfo(this.userName, this.volumeName,
+        this.volumeSize, this.blockSize,
+        utilization * blockSize);
+  }
+
+  public String[] getContainerIDs() {
+    //ArrayList<Long> ids = new ArrayList(containerMap.keySet());
+    //return ids.toArray(new Long[ids.size()]);
+    return containerIdOrdered.toArray(new String[containerIdOrdered.size()]);
+  }
+
+  public List<String> getContainerIDsList() {
+    return new ArrayList<>(containerIdOrdered);
+  }
+
+  public List<Pipeline> getContainerPipelines() {
+    Map<String, Pipeline> tmp = getPipelines();
+    List<Pipeline> pipelineList = new LinkedList<>();
+    for (String containerIDString : containerIdOrdered) {
+      pipelineList.add(tmp.get(containerIDString));
+    }
+    return pipelineList;
+  }
+
+  @Override
+  public String toString() {
+    String string = "";
+    string += "Username:" + userName + "\n";
+    string += "VolumeName:" + volumeName + "\n";
+    string += "VolumeSize:" + volumeSize + "\n";
+    string += "blockSize:" + blockSize + "\n";
+    string += "containerIds:" + containerIdOrdered + "\n";
+    string += "containerIdsWithObject:" + containerMap.keySet();
+    return string;
+  }
+
+  public CBlockClientServerProtocolProtos.MountVolumeResponseProto
+      toProtobuf() {
+    CBlockClientServerProtocolProtos.MountVolumeResponseProto.Builder volume =
+        CBlockClientServerProtocolProtos.MountVolumeResponseProto.newBuilder();
+    volume.setIsValid(true);
+    volume.setVolumeName(volumeName);
+    volume.setUserName(userName);
+    volume.setVolumeSize(volumeSize);
+    volume.setBlockSize(blockSize);
+    for (String containerIDString : containerIdOrdered) {
+      ContainerDescriptor containerDescriptor = containerMap.get(
+          containerIDString);
+      volume.addAllContainerIDs(containerDescriptor.toProtobuf());
+    }
+    return volume.build();
+  }
+
+  public static VolumeDescriptor fromProtobuf(byte[] data)
+      throws InvalidProtocolBufferException {
+    CBlockClientServerProtocolProtos.MountVolumeResponseProto volume =
+        CBlockClientServerProtocolProtos.MountVolumeResponseProto
+            .parseFrom(data);
+    String userName = volume.getUserName();
+    String volumeName = volume.getVolumeName();
+    long volumeSize = volume.getVolumeSize();
+    int blockSize = volume.getBlockSize();
+    VolumeDescriptor volumeDescriptor = new VolumeDescriptor(userName,
+        volumeName, volumeSize, blockSize);
+    List<CBlockClientServerProtocolProtos.ContainerIDProto> containers
+        = volume.getAllContainerIDsList();
+
+    String[] containerOrdering = new String[containers.size()];
+
+    for (CBlockClientServerProtocolProtos.ContainerIDProto containerProto :
+        containers) {
+      ContainerDescriptor containerDescriptor = new ContainerDescriptor(
+          containerProto.getContainerID(),
+          (int)containerProto.getIndex());
+      if(containerProto.hasPipeline()) {
+        containerDescriptor.setPipeline(
+            Pipeline.getFromProtoBuf(containerProto.getPipeline()));
+      }
+      volumeDescriptor.addContainer(containerDescriptor);
+      containerOrdering[containerDescriptor.getContainerIndex()] =
+          containerDescriptor.getContainerID();
+    }
+    volumeDescriptor.setContainerIDs(
+        new ArrayList<>(Arrays.asList(containerOrdering)));
+    return volumeDescriptor;
+  }
+
+  @Override
+  public int hashCode() {
+    return userName.hashCode()*37 + volumeName.hashCode();
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (o != null && o instanceof VolumeDescriptor) {
+      VolumeDescriptor other = (VolumeDescriptor)o;
+      if (!userName.equals(other.getUserName()) ||
+          !volumeName.equals(other.getVolumeName()) ||
+          volumeSize != other.getVolumeSize() ||
+          blockSize != other.getBlockSize()) {
+        return false;
+      }
+      if (containerIdOrdered.size() != other.containerIdOrdered.size() ||
+          containerMap.size() != other.containerMap.size()) {
+        return false;
+      }
+      for (int i = 0; i<containerIdOrdered.size(); i++) {
+        if (!containerIdOrdered.get(i).equals(
+            other.containerIdOrdered.get(i))) {
+          return false;
+        }
+      }
+      return containerMap.equals(other.containerMap);
+    }
+    return false;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java
new file mode 100644
index 0000000..7f50c41
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.meta;
+
+/**
+ * A wrapper class that represents the information about a volume. Used in
+ * communication between CBlock client and CBlock server only.
+ */
+public class VolumeInfo {
+  private final String userName;
+  private final String volumeName;
+  private final long volumeSize;
+  private final long blockSize;
+  private final long usage;
+
+  public VolumeInfo(String userName, String volumeName, long volumeSize,
+      long blockSize, long usage) {
+    this.userName = userName;
+    this.volumeName = volumeName;
+    this.volumeSize = volumeSize;
+    this.blockSize = blockSize;
+    this.usage = usage;
+  }
+
+  // When listing volume, the usage will not be set.
+  public VolumeInfo(String userName, String volumeName, long volumeSize,
+      long blockSize) {
+    this.userName = userName;
+    this.volumeName = volumeName;
+    this.volumeSize = volumeSize;
+    this.blockSize = blockSize;
+    this.usage = -1;
+  }
+
+  public long getVolumeSize() {
+    return volumeSize;
+  }
+
+  public long getBlockSize() {
+    return blockSize;
+  }
+
+  public long getUsage() {
+    return usage;
+  }
+
+  public String getUserName() {
+    return userName;
+  }
+
+  public String getVolumeName() {
+    return volumeName;
+  }
+
+  @Override
+  public String toString() {
+    return " userName:" + userName +
+        " volumeName:" + volumeName +
+        " volumeSize:" + volumeSize +
+        " blockSize:" + blockSize +
+        " (sizeInBlocks:" + volumeSize/blockSize + ")" +
+        " usageInBlocks:" + usage;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/package-info.java
new file mode 100644
index 0000000..a331d7a
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.meta;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/package-info.java
new file mode 100644
index 0000000..a7d5d8b
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java
new file mode 100644
index 0000000..fc40cef
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.proto;
+
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * The protocol that CBlock client side uses to talk to server side. CBlock
+ * client is the point where a volume is mounted. All the actual volume IO
+ * operations will go through CBlock client after the volume is mounted.
+ *
+ * When users mount a volume on CBlock client, CBlock client side uses this
+ * protocol to send mount request to CBlock server.
+ */
+public interface CBlockClientProtocol {
+  MountVolumeResponse mountVolume(String userName, String volumeName)
+      throws IOException;
+
+  List<VolumeInfo> listVolumes() throws IOException;
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java
new file mode 100644
index 0000000..bf00bc0
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.proto;
+
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * CBlock uses a separate command line tool to send volume management
+ * operations to CBlock server, including create/delete/info/list volumes. This
+ * is the protocol used by the command line tool to send these requests and get
+ * responses from CBlock server.
+ */
+@InterfaceAudience.Private
+public interface CBlockServiceProtocol {
+
+  void createVolume(String userName, String volumeName,
+      long volumeSize, int blockSize) throws IOException;
+
+  void deleteVolume(String userName, String volumeName,
+      boolean force) throws IOException;
+
+  VolumeInfo infoVolume(String userName,
+      String volumeName) throws IOException;
+
+  List<VolumeInfo> listVolume(String userName) throws IOException;
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java
new file mode 100644
index 0000000..70ccd90
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.proto;
+
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ * The response message of mounting a volume. Including enough information
+ * for the client to communicate (perform IO) with the volume containers
+ * directly.
+ */
+public class MountVolumeResponse {
+  private final boolean isValid;
+  private final String userName;
+  private final String volumeName;
+  private final long volumeSize;
+  private final int blockSize;
+  private List<Pipeline> containerList;
+  private HashMap<String, Pipeline> pipelineMap;
+
+  public MountVolumeResponse(boolean isValid, String userName,
+      String volumeName, long volumeSize, int blockSize,
+      List<Pipeline> containerList,
+      HashMap<String, Pipeline> pipelineMap) {
+    this.isValid = isValid;
+    this.userName = userName;
+    this.volumeName = volumeName;
+    this.volumeSize = volumeSize;
+    this.blockSize = blockSize;
+    this.containerList = containerList;
+    this.pipelineMap = pipelineMap;
+  }
+
+  public boolean getIsValid() {
+    return isValid;
+  }
+
+  public String getUserName() {
+    return userName;
+  }
+
+  public String getVolumeName() {
+    return volumeName;
+  }
+
+  public long getVolumeSize() {
+    return volumeSize;
+  }
+
+  public int getBlockSize() {
+    return blockSize;
+  }
+
+  public List<Pipeline> getContainerList() {
+    return containerList;
+  }
+
+  public HashMap<String, Pipeline> getPipelineMap() {
+    return pipelineMap;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/package-info.java
new file mode 100644
index 0000000..33438ec
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.proto;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java
new file mode 100644
index 0000000..99f3110
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.protocolPB;
+
+import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.ipc.ProtocolInfo;
+
+/**
+ * This is the protocol CBlock client uses to talk to CBlock server.
+ * CBlock client is the mounting point of a volume. When a user mounts a
+ * volume, the cBlock client running on the local node will use this protocol
+ * to talk to CBlock server to mount the volume.
+ */
+@ProtocolInfo(protocolName =
+    "org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocol",
+    protocolVersion = 1)
+@InterfaceAudience.Private
+public interface CBlockClientServerProtocolPB extends
+    CBlockClientServerProtocolProtos
+        .CBlockClientServerProtocolService.BlockingInterface {
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java
new file mode 100644
index 0000000..bfe2130
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.protocolPB;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.proto.CBlockClientProtocol;
+import org.apache.hadoop.cblock.proto.MountVolumeResponse;
+import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
+import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * The server side implementation of cblock client to server protocol.
+ */
+@InterfaceAudience.Private
+public class CBlockClientServerProtocolServerSideTranslatorPB implements
+    CBlockClientServerProtocolPB {
+
+  private final CBlockClientProtocol impl;
+
+  public CBlockClientServerProtocolServerSideTranslatorPB(
+      CBlockClientProtocol impl) {
+    this.impl = impl;
+  }
+
+  @Override
+  public CBlockClientServerProtocolProtos.MountVolumeResponseProto mountVolume(
+      RpcController controller,
+      CBlockClientServerProtocolProtos.MountVolumeRequestProto request)
+      throws ServiceException {
+    String userName = request.getUserName();
+    String volumeName = request.getVolumeName();
+    CBlockClientServerProtocolProtos.MountVolumeResponseProto.Builder
+        resp =
+        CBlockClientServerProtocolProtos
+            .MountVolumeResponseProto.newBuilder();
+    try {
+      MountVolumeResponse result = impl.mountVolume(userName, volumeName);
+      boolean isValid = result.getIsValid();
+      resp.setIsValid(isValid);
+      if (isValid) {
+        resp.setUserName(result.getUserName());
+        resp.setVolumeName(result.getVolumeName());
+        resp.setVolumeSize(result.getVolumeSize());
+        resp.setBlockSize(result.getBlockSize());
+        List<Pipeline> containers = result.getContainerList();
+        HashMap<String, Pipeline> pipelineMap = result.getPipelineMap();
+
+        for (int i=0; i<containers.size(); i++) {
+          CBlockClientServerProtocolProtos.ContainerIDProto.Builder id =
+              CBlockClientServerProtocolProtos.ContainerIDProto.newBuilder();
+          String containerName = containers.get(i).getContainerName();
+          id.setContainerID(containerName);
+          id.setIndex(i);
+          if (pipelineMap.containsKey(containerName)) {
+            id.setPipeline(pipelineMap.get(containerName).getProtobufMessage());
+          }
+          resp.addAllContainerIDs(id.build());
+        }
+      }
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+    return resp.build();
+  }
+
+  @Override
+  public CBlockClientServerProtocolProtos.ListVolumesResponseProto listVolumes(
+      RpcController controller,
+      CBlockClientServerProtocolProtos.ListVolumesRequestProto request)
+      throws ServiceException {
+    try {
+      CBlockClientServerProtocolProtos.ListVolumesResponseProto.Builder resp =
+          CBlockClientServerProtocolProtos.ListVolumesResponseProto
+              .newBuilder();
+      List<VolumeInfo> volumeInfos = impl.listVolumes();
+      List<CBlockServiceProtocolProtos.VolumeInfoProto> convertedInfos =
+          volumeInfos.stream().map(
+              volumeInfo -> CBlockServiceProtocolProtos.VolumeInfoProto
+                  .newBuilder().setUserName(volumeInfo.getUserName())
+                  .setBlockSize(volumeInfo.getBlockSize())
+                  .setVolumeName(volumeInfo.getVolumeName())
+                  .setVolumeSize(volumeInfo.getVolumeSize())
+                  .setUsage(volumeInfo.getUsage()).build())
+              .collect(Collectors.toList());
+      resp.addAllVolumeEntry(convertedInfos);
+      return resp.build();
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java
new file mode 100644
index 0000000..282d6cd
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.protocolPB;
+
+import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.ipc.ProtocolInfo;
+
+/**
+ * Users use a independent command line tool to talk to CBlock server for
+ * volume operations (create/delete/info/list). This is the protocol used by
+ * the the command line tool to send these requests to CBlock server.
+ */
+@ProtocolInfo(protocolName =
+    "org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocol",
+    protocolVersion = 1)
+@InterfaceAudience.Private
+public interface CBlockServiceProtocolPB extends
+    CBlockServiceProtocolProtos.CBlockServiceProtocolService.BlockingInterface {
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java
new file mode 100644
index 0000000..8924a0c
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.cblock.protocolPB;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.proto.CBlockServiceProtocol;
+import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT;
+
+/**
+ * Server side implementation of the protobuf service.
+ */
+@InterfaceAudience.Private
+public class CBlockServiceProtocolServerSideTranslatorPB
+    implements CBlockServiceProtocolPB {
+
+  private final CBlockServiceProtocol impl;
+  private static final Logger LOG =
+      LoggerFactory.getLogger(
+          CBlockServiceProtocolServerSideTranslatorPB.class);
+
+  @Override
+  public CBlockServiceProtocolProtos.CreateVolumeResponseProto createVolume(
+      RpcController controller,
+      CBlockServiceProtocolProtos.CreateVolumeRequestProto request)
+      throws ServiceException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("createVolume called! volume size: " + request.getVolumeSize()
+          + " block size: " + request.getBlockSize());
+    }
+    try {
+      if (request.hasBlockSize()) {
+        impl.createVolume(request.getUserName(), request.getVolumeName(),
+            request.getVolumeSize(), request.getBlockSize());
+      } else{
+        impl.createVolume(request.getUserName(), request.getVolumeName(),
+            request.getVolumeSize(), DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT);
+      }
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+    return CBlockServiceProtocolProtos.CreateVolumeResponseProto
+        .newBuilder().build();
+  }
+
+  @Override
+  public CBlockServiceProtocolProtos.DeleteVolumeResponseProto deleteVolume(
+      RpcController controller,
+      CBlockServiceProtocolProtos.DeleteVolumeRequestProto request)
+      throws ServiceException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("deleteVolume called! volume name: " + request.getVolumeName()
+          + " force:" + request.getForce());
+    }
+    try {
+      if (request.hasForce()) {
+        impl.deleteVolume(request.getUserName(), request.getVolumeName(),
+            request.getForce());
+      } else {
+        impl.deleteVolume(request.getUserName(), request.getVolumeName(),
+            false);
+      }
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+    return CBlockServiceProtocolProtos.DeleteVolumeResponseProto
+        .newBuilder().build();
+  }
+
+  @Override
+  public CBlockServiceProtocolProtos.InfoVolumeResponseProto infoVolume(
+      RpcController controller,
+      CBlockServiceProtocolProtos.InfoVolumeRequestProto request
+  ) throws ServiceException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("infoVolume called! volume name: " + request.getVolumeName());
+    }
+    CBlockServiceProtocolProtos.InfoVolumeResponseProto.Builder resp =
+        CBlockServiceProtocolProtos.InfoVolumeResponseProto.newBuilder();
+    CBlockServiceProtocolProtos.VolumeInfoProto.Builder volumeInfoProto =
+        CBlockServiceProtocolProtos.VolumeInfoProto.newBuilder();
+    VolumeInfo volumeInfo;
+    try {
+      volumeInfo = impl.infoVolume(request.getUserName(),
+          request.getVolumeName());
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+
+    volumeInfoProto.setVolumeSize(volumeInfo.getVolumeSize());
+    volumeInfoProto.setBlockSize(volumeInfo.getBlockSize());
+    volumeInfoProto.setUsage(volumeInfo.getUsage());
+    volumeInfoProto.setUserName(volumeInfo.getUserName());
+    volumeInfoProto.setVolumeName(volumeInfo.getVolumeName());
+    resp.setVolumeInfo(volumeInfoProto);
+    return resp.build();
+  }
+
+  @Override
+  public CBlockServiceProtocolProtos.ListVolumeResponseProto listVolume(
+      RpcController controller,
+      CBlockServiceProtocolProtos.ListVolumeRequestProto request
+  ) throws ServiceException {
+    CBlockServiceProtocolProtos.ListVolumeResponseProto.Builder resp =
+        CBlockServiceProtocolProtos.ListVolumeResponseProto.newBuilder();
+    String userName = null;
+    if (request.hasUserName()) {
+      userName = request.getUserName();
+    }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("list volume received for :" + userName);
+    }
+    List<VolumeInfo> volumes;
+    try {
+      volumes = impl.listVolume(userName);
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+    for (VolumeInfo volume : volumes) {
+      CBlockServiceProtocolProtos.VolumeInfoProto.Builder volumeEntryProto
+          = CBlockServiceProtocolProtos.VolumeInfoProto.newBuilder();
+      volumeEntryProto.setUserName(volume.getUserName());
+      volumeEntryProto.setVolumeName(volume.getVolumeName());
+      volumeEntryProto.setVolumeSize(volume.getVolumeSize());
+      volumeEntryProto.setBlockSize(volume.getBlockSize());
+      resp.addVolumeEntry(volumeEntryProto.build());
+    }
+    return resp.build();
+  }
+
+  public CBlockServiceProtocolServerSideTranslatorPB(
+      CBlockServiceProtocol impl) {
+    this.impl = impl;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java
new file mode 100644
index 0000000..5e03a92
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.protocolPB;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[34/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java
deleted file mode 100644
index 85f8d6f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java
deleted file mode 100644
index 93ed005..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.cblock.kubernetes;
-
-import com.google.gson.reflect.TypeToken;
-import com.squareup.okhttp.RequestBody;
-import io.kubernetes.client.ApiClient;
-import io.kubernetes.client.ApiException;
-import io.kubernetes.client.Configuration;
-import io.kubernetes.client.apis.CoreV1Api;
-import io.kubernetes.client.models.V1ISCSIVolumeSource;
-import io.kubernetes.client.models.V1ObjectMeta;
-import io.kubernetes.client.models.V1ObjectReference;
-import io.kubernetes.client.models.V1PersistentVolume;
-import io.kubernetes.client.models.V1PersistentVolumeClaim;
-import io.kubernetes.client.models.V1PersistentVolumeSpec;
-import io.kubernetes.client.util.Config;
-import io.kubernetes.client.util.Watch;
-import okio.Buffer;
-import org.apache.hadoop.cblock.cli.CBlockCli;
-import org.apache.hadoop.cblock.exception.CBlockException;
-import org.apache.hadoop.cblock.proto.MountVolumeResponse;
-import org.apache.hadoop.cblock.storage.StorageManager;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.ratis.shaded.com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.SocketTimeoutException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_ISCSI_ADVERTISED_IP;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_ISCSI_ADVERTISED_PORT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_KUBERNETES_CBLOCK_USER;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY;
-
-/**
- * Kubernetes Dynamic Persistent Volume provisioner.
- *
- * Listens on the kubernetes feed and creates the appropriate cblock AND
- * kubernetes PersistentVolume according to the created PersistentVolumeClaims.
- */
-public class DynamicProvisioner implements Runnable{
-
-  protected static final Logger LOGGER =
-      LoggerFactory.getLogger(DynamicProvisioner.class);
-
-  private static final String STORAGE_CLASS = "cblock";
-
-  private static final String PROVISIONER_ID = "hadoop.apache.org/cblock";
-  private static final String KUBERNETES_PROVISIONER_KEY =
-      "volume.beta.kubernetes.io/storage-provisioner";
-  private static final String KUBERNETES_BIND_COMPLETED_KEY =
-      "pv.kubernetes.io/bind-completed";
-
-  private boolean running = true;
-
-  private final StorageManager storageManager;
-
-  private String kubernetesConfigFile;
-
-  private String externalIp;
-
-  private int externalPort;
-
-  private String cblockUser;
-
-  private CoreV1Api api;
-
-  private ApiClient client;
-
-  private Thread watcherThread;
-
-  public  DynamicProvisioner(OzoneConfiguration ozoneConf,
-      StorageManager storageManager) throws IOException {
-    this.storageManager = storageManager;
-
-    kubernetesConfigFile = ozoneConf
-        .getTrimmed(DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY);
-
-    String jscsiServerAddress = ozoneConf
-        .get(DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY,
-            DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT);
-
-    externalIp = ozoneConf.
-        getTrimmed(DFS_CBLOCK_ISCSI_ADVERTISED_IP, jscsiServerAddress);
-
-    externalPort = ozoneConf.
-        getInt(DFS_CBLOCK_ISCSI_ADVERTISED_PORT,
-            DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT);
-
-    cblockUser = ozoneConf.getTrimmed(DFS_CBLOCK_KUBERNETES_CBLOCK_USER,
-        DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT);
-
-
-  }
-
-  public void init() throws IOException {
-    if (kubernetesConfigFile != null) {
-      client = Config.fromConfig(kubernetesConfigFile);
-    } else {
-      client = Config.fromCluster();
-    }
-    client.getHttpClient().setReadTimeout(60, TimeUnit.SECONDS);
-    Configuration.setDefaultApiClient(client);
-    api = new CoreV1Api();
-
-    watcherThread = new Thread(this);
-    watcherThread.setName("DynamicProvisioner");
-    watcherThread.setDaemon(true);
-  }
-
-  @Override
-  public void run() {
-    LOGGER.info("Starting kubernetes dynamic provisioner.");
-    while (running) {
-      String resourceVersion = null;
-      try {
-
-        Watch<V1PersistentVolumeClaim> watch = Watch.createWatch(client,
-            api.listPersistentVolumeClaimForAllNamespacesCall(null,
-                null,
-                false,
-                null,
-                null,
-                null,
-                resourceVersion,
-                null,
-                true,
-                null,
-                null),
-            new TypeToken<Watch.Response<V1PersistentVolumeClaim>>() {
-            }.getType());
-
-
-        //check the new pvc resources, and create cblock + pv if needed
-        for (Watch.Response<V1PersistentVolumeClaim> item : watch) {
-          V1PersistentVolumeClaim claim = item.object;
-
-          if (isPvMissingForPvc(claim)) {
-
-            LOGGER.info("Provisioning volumes for PVC {}/{}",
-                claim.getMetadata().getNamespace(),
-                claim.getMetadata().getName());
-
-            if (LOGGER.isDebugEnabled()) {
-              RequestBody request =
-                  api.getApiClient().serialize(claim, "application/json");
-
-              final Buffer buffer = new Buffer();
-              request.writeTo(buffer);
-              LOGGER.debug("New PVC is detected: " + buffer.readUtf8());
-            }
-
-            String volumeName = createVolumeName(claim);
-
-            long size = CBlockCli.parseSize(
-                claim.getSpec().getResources().getRequests().get("storage"));
-
-            createCBlock(volumeName, size);
-            createPersistentVolumeFromPVC(item.object, volumeName);
-          }
-        }
-      } catch (Exception ex) {
-        if (ex.getCause() != null && ex
-            .getCause() instanceof SocketTimeoutException) {
-          //This is normal. We are connection to the kubernetes server and the
-          //connection should be reopened time to time...
-          LOGGER.debug("Time exception occured", ex);
-        } else {
-          LOGGER.error("Error on provisioning persistent volumes.", ex);
-          try {
-            //we can try again in the main loop
-            Thread.sleep(1000);
-          } catch (InterruptedException e) {
-            LOGGER.error("Error on sleeping after an error.", e);
-          }
-        }
-      }
-    }
-  }
-
-  private boolean isPvMissingForPvc(V1PersistentVolumeClaim claim) {
-
-    Map<String, String> annotations = claim.getMetadata().getAnnotations();
-
-    return claim.getStatus().getPhase().equals("Pending") && STORAGE_CLASS
-        .equals(claim.getSpec().getStorageClassName()) && PROVISIONER_ID
-        .equals(annotations.get(KUBERNETES_PROVISIONER_KEY)) && !"yes"
-        .equals(annotations.get(KUBERNETES_BIND_COMPLETED_KEY));
-  }
-
-  @VisibleForTesting
-  protected String createVolumeName(V1PersistentVolumeClaim claim) {
-    return claim.getMetadata().getName() + "-" + claim.getMetadata()
-        .getUid();
-  }
-
-  public void stop() {
-    running = false;
-    try {
-      watcherThread.join(60000);
-    } catch (InterruptedException e) {
-      LOGGER.error("Kubernetes watcher thread can't stopped gracefully.", e);
-    }
-  }
-
-  private void createCBlock(String volumeName, long size)
-      throws CBlockException {
-
-    MountVolumeResponse mountVolumeResponse =
-        storageManager.isVolumeValid(cblockUser, volumeName);
-    if (!mountVolumeResponse.getIsValid()) {
-      storageManager
-          .createVolume(cblockUser, volumeName, size, 4 * 1024);
-    }
-  }
-
-  private void createPersistentVolumeFromPVC(V1PersistentVolumeClaim claim,
-      String volumeName) throws ApiException, IOException {
-
-    V1PersistentVolume v1PersistentVolume =
-        persitenceVolumeBuilder(claim, volumeName);
-
-    if (LOGGER.isDebugEnabled()) {
-      RequestBody request =
-          api.getApiClient().serialize(v1PersistentVolume, "application/json");
-
-      final Buffer buffer = new Buffer();
-      request.writeTo(buffer);
-      LOGGER.debug("Creating new PV: " + buffer.readUtf8());
-    }
-    api.createPersistentVolume(v1PersistentVolume, null);
-  }
-
-  protected V1PersistentVolume persitenceVolumeBuilder(
-      V1PersistentVolumeClaim claim,
-      String volumeName) {
-
-    V1PersistentVolume v1PersistentVolume = new V1PersistentVolume();
-    v1PersistentVolume.setKind("PersistentVolume");
-    v1PersistentVolume.setApiVersion("v1");
-
-    V1ObjectMeta metadata = new V1ObjectMeta();
-    metadata.setName(volumeName);
-    metadata.setNamespace(claim.getMetadata().getNamespace());
-    metadata.setAnnotations(new HashMap<>());
-
-    metadata.getAnnotations()
-        .put("pv.kubernetes.io/provisioned-by", PROVISIONER_ID);
-
-    metadata.getAnnotations()
-        .put("volume.beta.kubernetes.io/storage-class", STORAGE_CLASS);
-
-    v1PersistentVolume.setMetadata(metadata);
-
-    V1PersistentVolumeSpec spec = new V1PersistentVolumeSpec();
-
-    spec.setCapacity(new HashMap<>());
-    spec.getCapacity().put("storage",
-        claim.getSpec().getResources().getRequests().get("storage"));
-
-    spec.setAccessModes(new ArrayList<>());
-    spec.getAccessModes().add("ReadWriteOnce");
-
-    V1ObjectReference claimRef = new V1ObjectReference();
-    claimRef.setName(claim.getMetadata().getName());
-    claimRef.setNamespace(claim.getMetadata().getNamespace());
-    claimRef.setKind(claim.getKind());
-    claimRef.setApiVersion(claim.getApiVersion());
-    claimRef.setUid(claim.getMetadata().getUid());
-    spec.setClaimRef(claimRef);
-
-    spec.persistentVolumeReclaimPolicy("Delete");
-
-    V1ISCSIVolumeSource iscsi = new V1ISCSIVolumeSource();
-    iscsi.setIqn(cblockUser + ":" + volumeName);
-    iscsi.setLun(0);
-    iscsi.setFsType("ext4");
-    String portal = externalIp + ":" + externalPort;
-    iscsi.setTargetPortal(portal);
-    iscsi.setPortals(new ArrayList<>());
-    iscsi.getPortals().add(portal);
-
-    spec.iscsi(iscsi);
-    v1PersistentVolume.setSpec(spec);
-    return v1PersistentVolume;
-  }
-
-
-  @VisibleForTesting
-  protected CoreV1Api getApi() {
-    return api;
-  }
-
-  public void start() {
-    watcherThread.start();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java
deleted file mode 100644
index 3ec5aab..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package contains helper classes to run hadoop cluster in kubernetes
- * environment.
- */
-package org.apache.hadoop.cblock.kubernetes;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java
deleted file mode 100644
index 00064a6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.meta;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-/**
- *
- * The internal representation of a container maintained by CBlock server.
- * Include enough information to exactly identify a container for read/write
- * operation.
- *
- * NOTE that this class is work-in-progress. Depends on HDFS-7240 container
- * implementation. Currently only to allow testing.
- */
-public class ContainerDescriptor {
-  private final String containerID;
-  // the index of this container with in a volume
-  // on creation, there may be no way to know the index of the container
-  // as it is a volume specific information
-  private int containerIndex;
-  private Pipeline pipeline;
-
-  public ContainerDescriptor(String containerID) {
-    this.containerID = containerID;
-  }
-
-  public ContainerDescriptor(String containerID, int containerIndex) {
-    this.containerID = containerID;
-    this.containerIndex = containerIndex;
-  }
-
-  public void setContainerIndex(int idx) {
-    this.containerIndex = idx;
-  }
-
-  public String getContainerID() {
-    return containerID;
-  }
-
-  public void setPipeline(Pipeline pipeline) {
-    this.pipeline = pipeline;
-  }
-
-  public Pipeline getPipeline() {
-    return pipeline;
-  }
-
-  public int getContainerIndex() {
-    return containerIndex;
-  }
-
-  public long getUtilization() {
-    return 0;
-  }
-
-  public CBlockClientServerProtocolProtos.ContainerIDProto toProtobuf() {
-    CBlockClientServerProtocolProtos.ContainerIDProto.Builder builder =
-        CBlockClientServerProtocolProtos.ContainerIDProto.newBuilder();
-    builder.setContainerID(containerID);
-    builder.setIndex(containerIndex);
-    if (pipeline != null) {
-      builder.setPipeline(pipeline.getProtobufMessage());
-    }
-    return builder.build();
-  }
-
-  public static ContainerDescriptor fromProtobuf(byte[] data)
-      throws InvalidProtocolBufferException {
-    CBlockClientServerProtocolProtos.ContainerIDProto id =
-        CBlockClientServerProtocolProtos.ContainerIDProto.parseFrom(data);
-    return new ContainerDescriptor(id.getContainerID(),
-        (int)id.getIndex());
-  }
-
-  @Override
-  public int hashCode() {
-    return containerID.hashCode()*37 + containerIndex;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (o != null && o instanceof ContainerDescriptor) {
-      ContainerDescriptor other = (ContainerDescriptor)o;
-      return containerID.equals(other.containerID) &&
-          containerIndex == other.containerIndex;
-    }
-    return false;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java
deleted file mode 100644
index 4f5930d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.meta;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * The internal representation maintained by CBlock server as the info for
- * a volume. Contains the list of containers belonging to this volume.
- *
- * Many methods of this class is made such that the volume information (
- * including container list) can be easily transformed into a Json string
- * that can be stored/parsed from a persistent store for cblock server
- * persistence.
- *
- * This class is still work-in-progress.
- */
-public class VolumeDescriptor {
-  // The main data structure is the container location map
-  // other thing are mainly just information
-
-  // since only one operation at a time is allowed, no
-  // need to consider concurrency control here
-
-  // key is container id
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(VolumeDescriptor.class);
-
-  private ConcurrentHashMap<String, ContainerDescriptor> containerMap;
-  private String userName;
-  private int blockSize;
-  private long volumeSize;
-  private String volumeName;
-  // this is essentially the ordered keys of containerMap
-  // which is kind of redundant information. But since we
-  // are likely to access it frequently based on ordering.
-  // keeping this copy to avoid having to sort the key every
-  // time
-  private List<String> containerIdOrdered;
-
-  /**
-   * This is not being called explicitly, but this is necessary as
-   * it will be called by the parse method implicitly when
-   * reconstructing the object from json string. The get*() methods
-   * and set*() methods are for the same purpose also.
-   */
-  public VolumeDescriptor() {
-    this(null, null, 0, 0);
-  }
-
-  public VolumeDescriptor(String userName, String volumeName, long volumeSize,
-      int blockSize) {
-    this.containerMap = new ConcurrentHashMap<>();
-    this.userName = userName;
-    this.volumeName = volumeName;
-    this.blockSize = blockSize;
-    this.volumeSize = volumeSize;
-    this.containerIdOrdered = new LinkedList<>();
-  }
-
-  public String getUserName() {
-    return userName;
-  }
-
-  public void setUserName(String userName) {
-    this.userName = userName;
-  }
-
-  public String getVolumeName() {
-    return volumeName;
-  }
-
-  public void setVolumeName(String volumeName) {
-    this.volumeName = volumeName;
-  }
-
-  public long getVolumeSize() {
-    return volumeSize;
-  }
-
-  public void setVolumeSize(long volumeSize) {
-    this.volumeSize = volumeSize;
-  }
-
-  public int getBlockSize() {
-    return blockSize;
-  }
-
-  public void setBlockSize(int blockSize) {
-    this.blockSize = blockSize;
-  }
-
-  public void setContainerIDs(ArrayList<String> containerIDs) {
-    containerIdOrdered.addAll(containerIDs);
-  }
-
-  public void addContainer(ContainerDescriptor containerDescriptor) {
-    containerMap.put(containerDescriptor.getContainerID(),
-        containerDescriptor);
-  }
-
-
-  public HashMap<String, Pipeline> getPipelines() {
-    HashMap<String, Pipeline> pipelines = new HashMap<>();
-    for (Map.Entry<String, ContainerDescriptor> entry :
-        containerMap.entrySet()) {
-      pipelines.put(entry.getKey(), entry.getValue().getPipeline());
-    }
-    return pipelines;
-  }
-
-  public boolean isEmpty() {
-    VolumeInfo info = getInfo();
-    return info.getUsage() == 0;
-  }
-
-  public VolumeInfo getInfo() {
-    // TODO : need to actually go through all containers of this volume and
-    // ask for their utilization.
-    long utilization = 0;
-    for (Map.Entry<String, ContainerDescriptor> entry :
-        containerMap.entrySet()) {
-      utilization += entry.getValue().getUtilization();
-    }
-    return new VolumeInfo(this.userName, this.volumeName,
-        this.volumeSize, this.blockSize,
-        utilization * blockSize);
-  }
-
-  public String[] getContainerIDs() {
-    //ArrayList<Long> ids = new ArrayList(containerMap.keySet());
-    //return ids.toArray(new Long[ids.size()]);
-    return containerIdOrdered.toArray(new String[containerIdOrdered.size()]);
-  }
-
-  public List<String> getContainerIDsList() {
-    return new ArrayList<>(containerIdOrdered);
-  }
-
-  public List<Pipeline> getContainerPipelines() {
-    Map<String, Pipeline> tmp = getPipelines();
-    List<Pipeline> pipelineList = new LinkedList<>();
-    for (String containerIDString : containerIdOrdered) {
-      pipelineList.add(tmp.get(containerIDString));
-    }
-    return pipelineList;
-  }
-
-  @Override
-  public String toString() {
-    String string = "";
-    string += "Username:" + userName + "\n";
-    string += "VolumeName:" + volumeName + "\n";
-    string += "VolumeSize:" + volumeSize + "\n";
-    string += "blockSize:" + blockSize + "\n";
-    string += "containerIds:" + containerIdOrdered + "\n";
-    string += "containerIdsWithObject:" + containerMap.keySet();
-    return string;
-  }
-
-  public CBlockClientServerProtocolProtos.MountVolumeResponseProto
-      toProtobuf() {
-    CBlockClientServerProtocolProtos.MountVolumeResponseProto.Builder volume =
-        CBlockClientServerProtocolProtos.MountVolumeResponseProto.newBuilder();
-    volume.setIsValid(true);
-    volume.setVolumeName(volumeName);
-    volume.setUserName(userName);
-    volume.setVolumeSize(volumeSize);
-    volume.setBlockSize(blockSize);
-    for (String containerIDString : containerIdOrdered) {
-      ContainerDescriptor containerDescriptor = containerMap.get(
-          containerIDString);
-      volume.addAllContainerIDs(containerDescriptor.toProtobuf());
-    }
-    return volume.build();
-  }
-
-  public static VolumeDescriptor fromProtobuf(byte[] data)
-      throws InvalidProtocolBufferException {
-    CBlockClientServerProtocolProtos.MountVolumeResponseProto volume =
-        CBlockClientServerProtocolProtos.MountVolumeResponseProto
-            .parseFrom(data);
-    String userName = volume.getUserName();
-    String volumeName = volume.getVolumeName();
-    long volumeSize = volume.getVolumeSize();
-    int blockSize = volume.getBlockSize();
-    VolumeDescriptor volumeDescriptor = new VolumeDescriptor(userName,
-        volumeName, volumeSize, blockSize);
-    List<CBlockClientServerProtocolProtos.ContainerIDProto> containers
-        = volume.getAllContainerIDsList();
-
-    String[] containerOrdering = new String[containers.size()];
-
-    for (CBlockClientServerProtocolProtos.ContainerIDProto containerProto :
-        containers) {
-      ContainerDescriptor containerDescriptor = new ContainerDescriptor(
-          containerProto.getContainerID(),
-          (int)containerProto.getIndex());
-      if(containerProto.hasPipeline()) {
-        containerDescriptor.setPipeline(
-            Pipeline.getFromProtoBuf(containerProto.getPipeline()));
-      }
-      volumeDescriptor.addContainer(containerDescriptor);
-      containerOrdering[containerDescriptor.getContainerIndex()] =
-          containerDescriptor.getContainerID();
-    }
-    volumeDescriptor.setContainerIDs(
-        new ArrayList<>(Arrays.asList(containerOrdering)));
-    return volumeDescriptor;
-  }
-
-  @Override
-  public int hashCode() {
-    return userName.hashCode()*37 + volumeName.hashCode();
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (o != null && o instanceof VolumeDescriptor) {
-      VolumeDescriptor other = (VolumeDescriptor)o;
-      if (!userName.equals(other.getUserName()) ||
-          !volumeName.equals(other.getVolumeName()) ||
-          volumeSize != other.getVolumeSize() ||
-          blockSize != other.getBlockSize()) {
-        return false;
-      }
-      if (containerIdOrdered.size() != other.containerIdOrdered.size() ||
-          containerMap.size() != other.containerMap.size()) {
-        return false;
-      }
-      for (int i = 0; i<containerIdOrdered.size(); i++) {
-        if (!containerIdOrdered.get(i).equals(
-            other.containerIdOrdered.get(i))) {
-          return false;
-        }
-      }
-      return containerMap.equals(other.containerMap);
-    }
-    return false;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java
deleted file mode 100644
index 7f50c41..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.meta;
-
-/**
- * A wrapper class that represents the information about a volume. Used in
- * communication between CBlock client and CBlock server only.
- */
-public class VolumeInfo {
-  private final String userName;
-  private final String volumeName;
-  private final long volumeSize;
-  private final long blockSize;
-  private final long usage;
-
-  public VolumeInfo(String userName, String volumeName, long volumeSize,
-      long blockSize, long usage) {
-    this.userName = userName;
-    this.volumeName = volumeName;
-    this.volumeSize = volumeSize;
-    this.blockSize = blockSize;
-    this.usage = usage;
-  }
-
-  // When listing volume, the usage will not be set.
-  public VolumeInfo(String userName, String volumeName, long volumeSize,
-      long blockSize) {
-    this.userName = userName;
-    this.volumeName = volumeName;
-    this.volumeSize = volumeSize;
-    this.blockSize = blockSize;
-    this.usage = -1;
-  }
-
-  public long getVolumeSize() {
-    return volumeSize;
-  }
-
-  public long getBlockSize() {
-    return blockSize;
-  }
-
-  public long getUsage() {
-    return usage;
-  }
-
-  public String getUserName() {
-    return userName;
-  }
-
-  public String getVolumeName() {
-    return volumeName;
-  }
-
-  @Override
-  public String toString() {
-    return " userName:" + userName +
-        " volumeName:" + volumeName +
-        " volumeSize:" + volumeSize +
-        " blockSize:" + blockSize +
-        " (sizeInBlocks:" + volumeSize/blockSize + ")" +
-        " usageInBlocks:" + usage;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/package-info.java
deleted file mode 100644
index a331d7a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.meta;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/package-info.java
deleted file mode 100644
index a7d5d8b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java
deleted file mode 100644
index fc40cef..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.proto;
-
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * The protocol that CBlock client side uses to talk to server side. CBlock
- * client is the point where a volume is mounted. All the actual volume IO
- * operations will go through CBlock client after the volume is mounted.
- *
- * When users mount a volume on CBlock client, CBlock client side uses this
- * protocol to send mount request to CBlock server.
- */
-public interface CBlockClientProtocol {
-  MountVolumeResponse mountVolume(String userName, String volumeName)
-      throws IOException;
-
-  List<VolumeInfo> listVolumes() throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java
deleted file mode 100644
index bf00bc0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.proto;
-
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * CBlock uses a separate command line tool to send volume management
- * operations to CBlock server, including create/delete/info/list volumes. This
- * is the protocol used by the command line tool to send these requests and get
- * responses from CBlock server.
- */
-@InterfaceAudience.Private
-public interface CBlockServiceProtocol {
-
-  void createVolume(String userName, String volumeName,
-      long volumeSize, int blockSize) throws IOException;
-
-  void deleteVolume(String userName, String volumeName,
-      boolean force) throws IOException;
-
-  VolumeInfo infoVolume(String userName,
-      String volumeName) throws IOException;
-
-  List<VolumeInfo> listVolume(String userName) throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java
deleted file mode 100644
index 70ccd90..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.proto;
-
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.util.HashMap;
-import java.util.List;
-
-/**
- * The response message of mounting a volume. Including enough information
- * for the client to communicate (perform IO) with the volume containers
- * directly.
- */
-public class MountVolumeResponse {
-  private final boolean isValid;
-  private final String userName;
-  private final String volumeName;
-  private final long volumeSize;
-  private final int blockSize;
-  private List<Pipeline> containerList;
-  private HashMap<String, Pipeline> pipelineMap;
-
-  public MountVolumeResponse(boolean isValid, String userName,
-      String volumeName, long volumeSize, int blockSize,
-      List<Pipeline> containerList,
-      HashMap<String, Pipeline> pipelineMap) {
-    this.isValid = isValid;
-    this.userName = userName;
-    this.volumeName = volumeName;
-    this.volumeSize = volumeSize;
-    this.blockSize = blockSize;
-    this.containerList = containerList;
-    this.pipelineMap = pipelineMap;
-  }
-
-  public boolean getIsValid() {
-    return isValid;
-  }
-
-  public String getUserName() {
-    return userName;
-  }
-
-  public String getVolumeName() {
-    return volumeName;
-  }
-
-  public long getVolumeSize() {
-    return volumeSize;
-  }
-
-  public int getBlockSize() {
-    return blockSize;
-  }
-
-  public List<Pipeline> getContainerList() {
-    return containerList;
-  }
-
-  public HashMap<String, Pipeline> getPipelineMap() {
-    return pipelineMap;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/package-info.java
deleted file mode 100644
index 33438ec..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.proto;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java
deleted file mode 100644
index 99f3110..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.protocolPB;
-
-import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtocolInfo;
-
-/**
- * This is the protocol CBlock client uses to talk to CBlock server.
- * CBlock client is the mounting point of a volume. When a user mounts a
- * volume, the cBlock client running on the local node will use this protocol
- * to talk to CBlock server to mount the volume.
- */
-@ProtocolInfo(protocolName =
-    "org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocol",
-    protocolVersion = 1)
-@InterfaceAudience.Private
-public interface CBlockClientServerProtocolPB extends
-    CBlockClientServerProtocolProtos
-        .CBlockClientServerProtocolService.BlockingInterface {
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java
deleted file mode 100644
index bfe2130..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.protocolPB;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.proto.CBlockClientProtocol;
-import org.apache.hadoop.cblock.proto.MountVolumeResponse;
-import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
-import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * The server side implementation of cblock client to server protocol.
- */
-@InterfaceAudience.Private
-public class CBlockClientServerProtocolServerSideTranslatorPB implements
-    CBlockClientServerProtocolPB {
-
-  private final CBlockClientProtocol impl;
-
-  public CBlockClientServerProtocolServerSideTranslatorPB(
-      CBlockClientProtocol impl) {
-    this.impl = impl;
-  }
-
-  @Override
-  public CBlockClientServerProtocolProtos.MountVolumeResponseProto mountVolume(
-      RpcController controller,
-      CBlockClientServerProtocolProtos.MountVolumeRequestProto request)
-      throws ServiceException {
-    String userName = request.getUserName();
-    String volumeName = request.getVolumeName();
-    CBlockClientServerProtocolProtos.MountVolumeResponseProto.Builder
-        resp =
-        CBlockClientServerProtocolProtos
-            .MountVolumeResponseProto.newBuilder();
-    try {
-      MountVolumeResponse result = impl.mountVolume(userName, volumeName);
-      boolean isValid = result.getIsValid();
-      resp.setIsValid(isValid);
-      if (isValid) {
-        resp.setUserName(result.getUserName());
-        resp.setVolumeName(result.getVolumeName());
-        resp.setVolumeSize(result.getVolumeSize());
-        resp.setBlockSize(result.getBlockSize());
-        List<Pipeline> containers = result.getContainerList();
-        HashMap<String, Pipeline> pipelineMap = result.getPipelineMap();
-
-        for (int i=0; i<containers.size(); i++) {
-          CBlockClientServerProtocolProtos.ContainerIDProto.Builder id =
-              CBlockClientServerProtocolProtos.ContainerIDProto.newBuilder();
-          String containerName = containers.get(i).getContainerName();
-          id.setContainerID(containerName);
-          id.setIndex(i);
-          if (pipelineMap.containsKey(containerName)) {
-            id.setPipeline(pipelineMap.get(containerName).getProtobufMessage());
-          }
-          resp.addAllContainerIDs(id.build());
-        }
-      }
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-    return resp.build();
-  }
-
-  @Override
-  public CBlockClientServerProtocolProtos.ListVolumesResponseProto listVolumes(
-      RpcController controller,
-      CBlockClientServerProtocolProtos.ListVolumesRequestProto request)
-      throws ServiceException {
-    try {
-      CBlockClientServerProtocolProtos.ListVolumesResponseProto.Builder resp =
-          CBlockClientServerProtocolProtos.ListVolumesResponseProto
-              .newBuilder();
-      List<VolumeInfo> volumeInfos = impl.listVolumes();
-      List<CBlockServiceProtocolProtos.VolumeInfoProto> convertedInfos =
-          volumeInfos.stream().map(
-              volumeInfo -> CBlockServiceProtocolProtos.VolumeInfoProto
-                  .newBuilder().setUserName(volumeInfo.getUserName())
-                  .setBlockSize(volumeInfo.getBlockSize())
-                  .setVolumeName(volumeInfo.getVolumeName())
-                  .setVolumeSize(volumeInfo.getVolumeSize())
-                  .setUsage(volumeInfo.getUsage()).build())
-              .collect(Collectors.toList());
-      resp.addAllVolumeEntry(convertedInfos);
-      return resp.build();
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java
deleted file mode 100644
index 282d6cd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.protocolPB;
-
-import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtocolInfo;
-
-/**
- * Users use a independent command line tool to talk to CBlock server for
- * volume operations (create/delete/info/list). This is the protocol used by
- * the the command line tool to send these requests to CBlock server.
- */
-@ProtocolInfo(protocolName =
-    "org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocol",
-    protocolVersion = 1)
-@InterfaceAudience.Private
-public interface CBlockServiceProtocolPB extends
-    CBlockServiceProtocolProtos.CBlockServiceProtocolService.BlockingInterface {
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java
deleted file mode 100644
index 8924a0c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.protocolPB;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.proto.CBlockServiceProtocol;
-import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT;
-
-/**
- * Server side implementation of the protobuf service.
- */
-@InterfaceAudience.Private
-public class CBlockServiceProtocolServerSideTranslatorPB
-    implements CBlockServiceProtocolPB {
-
-  private final CBlockServiceProtocol impl;
-  private static final Logger LOG =
-      LoggerFactory.getLogger(
-          CBlockServiceProtocolServerSideTranslatorPB.class);
-
-  @Override
-  public CBlockServiceProtocolProtos.CreateVolumeResponseProto createVolume(
-      RpcController controller,
-      CBlockServiceProtocolProtos.CreateVolumeRequestProto request)
-      throws ServiceException {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("createVolume called! volume size: " + request.getVolumeSize()
-          + " block size: " + request.getBlockSize());
-    }
-    try {
-      if (request.hasBlockSize()) {
-        impl.createVolume(request.getUserName(), request.getVolumeName(),
-            request.getVolumeSize(), request.getBlockSize());
-      } else{
-        impl.createVolume(request.getUserName(), request.getVolumeName(),
-            request.getVolumeSize(), DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT);
-      }
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-    return CBlockServiceProtocolProtos.CreateVolumeResponseProto
-        .newBuilder().build();
-  }
-
-  @Override
-  public CBlockServiceProtocolProtos.DeleteVolumeResponseProto deleteVolume(
-      RpcController controller,
-      CBlockServiceProtocolProtos.DeleteVolumeRequestProto request)
-      throws ServiceException {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("deleteVolume called! volume name: " + request.getVolumeName()
-          + " force:" + request.getForce());
-    }
-    try {
-      if (request.hasForce()) {
-        impl.deleteVolume(request.getUserName(), request.getVolumeName(),
-            request.getForce());
-      } else {
-        impl.deleteVolume(request.getUserName(), request.getVolumeName(),
-            false);
-      }
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-    return CBlockServiceProtocolProtos.DeleteVolumeResponseProto
-        .newBuilder().build();
-  }
-
-  @Override
-  public CBlockServiceProtocolProtos.InfoVolumeResponseProto infoVolume(
-      RpcController controller,
-      CBlockServiceProtocolProtos.InfoVolumeRequestProto request
-  ) throws ServiceException {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("infoVolume called! volume name: " + request.getVolumeName());
-    }
-    CBlockServiceProtocolProtos.InfoVolumeResponseProto.Builder resp =
-        CBlockServiceProtocolProtos.InfoVolumeResponseProto.newBuilder();
-    CBlockServiceProtocolProtos.VolumeInfoProto.Builder volumeInfoProto =
-        CBlockServiceProtocolProtos.VolumeInfoProto.newBuilder();
-    VolumeInfo volumeInfo;
-    try {
-      volumeInfo = impl.infoVolume(request.getUserName(),
-          request.getVolumeName());
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-
-    volumeInfoProto.setVolumeSize(volumeInfo.getVolumeSize());
-    volumeInfoProto.setBlockSize(volumeInfo.getBlockSize());
-    volumeInfoProto.setUsage(volumeInfo.getUsage());
-    volumeInfoProto.setUserName(volumeInfo.getUserName());
-    volumeInfoProto.setVolumeName(volumeInfo.getVolumeName());
-    resp.setVolumeInfo(volumeInfoProto);
-    return resp.build();
-  }
-
-  @Override
-  public CBlockServiceProtocolProtos.ListVolumeResponseProto listVolume(
-      RpcController controller,
-      CBlockServiceProtocolProtos.ListVolumeRequestProto request
-  ) throws ServiceException {
-    CBlockServiceProtocolProtos.ListVolumeResponseProto.Builder resp =
-        CBlockServiceProtocolProtos.ListVolumeResponseProto.newBuilder();
-    String userName = null;
-    if (request.hasUserName()) {
-      userName = request.getUserName();
-    }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("list volume received for :" + userName);
-    }
-    List<VolumeInfo> volumes;
-    try {
-      volumes = impl.listVolume(userName);
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-    for (VolumeInfo volume : volumes) {
-      CBlockServiceProtocolProtos.VolumeInfoProto.Builder volumeEntryProto
-          = CBlockServiceProtocolProtos.VolumeInfoProto.newBuilder();
-      volumeEntryProto.setUserName(volume.getUserName());
-      volumeEntryProto.setVolumeName(volume.getVolumeName());
-      volumeEntryProto.setVolumeSize(volume.getVolumeSize());
-      volumeEntryProto.setBlockSize(volume.getBlockSize());
-      resp.addVolumeEntry(volumeEntryProto.build());
-    }
-    return resp.build();
-  }
-
-  public CBlockServiceProtocolServerSideTranslatorPB(
-      CBlockServiceProtocol impl) {
-    this.impl = impl;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java
deleted file mode 100644
index 5e03a92..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.protocolPB;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java
deleted file mode 100644
index 865f3b2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java
+++ /dev/null
@@ -1,427 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.storage;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.cblock.CBlockConfigKeys;
-import org.apache.hadoop.cblock.exception.CBlockException;
-import org.apache.hadoop.cblock.meta.ContainerDescriptor;
-import org.apache.hadoop.cblock.meta.VolumeDescriptor;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.proto.MountVolumeResponse;
-import org.apache.hadoop.cblock.util.KeyUtil;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * This class maintains the key space of CBlock, more specifically, the
- * volume to container mapping. The core data structure
- * is a map from users to their volumes info, where volume info is a handler
- * to a volume, containing information for IO on that volume and a storage
- * client responsible for talking to the SCM.
- */
-public class StorageManager {
-  private static final Logger LOGGER =
-      LoggerFactory.getLogger(StorageManager.class);
-  private final ScmClient storageClient;
-  private final int numThreads;
-  private static final int MAX_THREADS =
-      Runtime.getRuntime().availableProcessors() * 2;
-  private static final int MAX_QUEUE_CAPACITY = 1024;
-  private final String cblockId;
-
-  /**
-   * We will NOT have the situation where same kv pair getting
-   * processed, but it is possible to have multiple kv pair being
-   * processed at same time.
-   *
-   * So using just ConcurrentHashMap should be sufficient
-   *
-   * Again since currently same user accessing from multiple places
-   * is not allowed, no need to consider concurrency of volume map
-   * within one user
-   */
-  private ConcurrentHashMap<String, HashMap<String, VolumeDescriptor>>
-      user2VolumeMap;
-  // size of an underlying container.
-  // TODO : assuming all containers are of the same size
-  private long containerSizeB;
-
-  public StorageManager(ScmClient storageClient,
-      OzoneConfiguration ozoneConfig, String cblockId) throws IOException {
-    this.storageClient = storageClient;
-    this.user2VolumeMap = new ConcurrentHashMap<>();
-    this.containerSizeB = storageClient.getContainerSize(null);
-    this.numThreads =
-        ozoneConfig.getInt(CBlockConfigKeys.DFS_CBLOCK_MANAGER_POOL_SIZE,
-            CBlockConfigKeys.DFS_CBLOCK_MANAGER_POOL_SIZE_DEFAULT);
-    this.cblockId = cblockId;
-  }
-
-  /**
-   * This call will put the volume into in-memory map.
-   *
-   * more specifically, make the volume discoverable on jSCSI server
-   * and keep it's reference in-memory for look up.
-   * @param userName the user name of the volume.
-   * @param volumeName the name of the volume,
-   * @param volume a {@link VolumeDescriptor} object encapsulating the
-   *               information about the volume.
-   */
-  private void makeVolumeReady(String userName, String volumeName,
-      VolumeDescriptor volume) {
-    HashMap<String, VolumeDescriptor> userVolumes;
-    if (user2VolumeMap.containsKey(userName)) {
-      userVolumes = user2VolumeMap.get(userName);
-    } else {
-      userVolumes = new HashMap<>();
-      user2VolumeMap.put(userName, userVolumes);
-    }
-    userVolumes.put(volumeName, volume);
-  }
-
-  /**
-   * Called by CBlockManager to add volumes read from persistent store into
-   * memory, need to contact SCM to setup the reference to the containers given
-   * their id.
-   *
-   * Only for failover process where container meta info is read from
-   * persistent store, and containers themselves are alive.
-   *
-   * TODO : Currently, this method is not being called as failover process
-   * is not implemented yet.
-   *
-   * @param volumeDescriptor a {@link VolumeDescriptor} object encapsulating
-   *                         the information about a volume.
-   * @throws IOException when adding the volume failed. e.g. volume already
-   * exist, or no more container available.
-   */
-  public synchronized void addVolume(VolumeDescriptor volumeDescriptor)
-      throws IOException{
-    String userName = volumeDescriptor.getUserName();
-    String volumeName = volumeDescriptor.getVolumeName();
-    LOGGER.info("addVolume:" + userName + ":" + volumeName);
-    if (user2VolumeMap.containsKey(userName)
-        && user2VolumeMap.get(userName).containsKey(volumeName)) {
-      throw new CBlockException("Volume already exist for "
-          + userName + ":" + volumeName);
-    }
-    // the container ids are read from levelDB, setting up the
-    // container handlers here.
-    String[] containerIds = volumeDescriptor.getContainerIDs();
-
-    for (String containerId : containerIds) {
-      try {
-        Pipeline pipeline = storageClient.getContainer(containerId);
-        ContainerDescriptor containerDescriptor =
-            new ContainerDescriptor(containerId);
-        containerDescriptor.setPipeline(pipeline);
-        volumeDescriptor.addContainer(containerDescriptor);
-      } catch (IOException e) {
-        LOGGER.error("Getting container failed! Container:{} error:{}",
-            containerId, e);
-        throw e;
-      }
-    }
-    // now ready to put into in-memory map.
-    makeVolumeReady(userName, volumeName, volumeDescriptor);
-  }
-
-  private class CreateContainerTask implements Runnable {
-    private final VolumeDescriptor volume;
-    private final int containerIdx;
-    private final ArrayList<String> containerIds;
-    private final AtomicInteger numFailed;
-
-    CreateContainerTask(VolumeDescriptor volume, int containerIdx,
-                        ArrayList<String> containerIds,
-                        AtomicInteger numFailed) {
-      this.volume = volume;
-      this.containerIdx = containerIdx;
-      this.containerIds = containerIds;
-      this.numFailed = numFailed;
-    }
-
-    /**
-     * When an object implementing interface <code>Runnable</code> is used
-     * to create a thread, starting the thread causes the object's
-     * <code>run</code> method to be called in that separately executing
-     * thread.
-     * <p>
-     * The general contract of the method <code>run</code> is that it may
-     * take any action whatsoever.
-     *
-     * @see Thread#run()
-     */
-    public void run() {
-      ContainerDescriptor container = null;
-      try {
-        Pipeline pipeline = storageClient.createContainer(
-            OzoneProtos.ReplicationType.STAND_ALONE,
-            OzoneProtos.ReplicationFactor.ONE,
-            KeyUtil.getContainerName(volume.getUserName(),
-                volume.getVolumeName(), containerIdx), cblockId);
-
-        container = new ContainerDescriptor(pipeline.getContainerName());
-
-        container.setPipeline(pipeline);
-        container.setContainerIndex(containerIdx);
-        volume.addContainer(container);
-        containerIds.set(containerIdx, container.getContainerID());
-      } catch (Exception e) {
-        numFailed.incrementAndGet();
-        if (container != null) {
-          LOGGER.error("Error creating container Container:{}:" +
-              " index:{} error:{}", container.getContainerID(),
-              containerIdx, e);
-        } else {
-          LOGGER.error("Error creating container.", e);
-        }
-      }
-    }
-  }
-
-  private boolean createVolumeContainers(VolumeDescriptor volume) {
-    ArrayList<String> containerIds = new ArrayList<>();
-    ThreadPoolExecutor executor = new ThreadPoolExecutor(
-        Math.min(numThreads, MAX_THREADS),
-        MAX_THREADS, 1, TimeUnit.SECONDS,
-        new ArrayBlockingQueue<>(MAX_QUEUE_CAPACITY),
-        new ThreadPoolExecutor.CallerRunsPolicy());
-
-    AtomicInteger numFailedCreates = new AtomicInteger(0);
-    long allocatedSize = 0;
-    int containerIdx = 0;
-    while (allocatedSize < volume.getVolumeSize()) {
-      // adding null to allocate space in ArrayList
-      containerIds.add(containerIdx, null);
-      Runnable task = new CreateContainerTask(volume, containerIdx,
-          containerIds, numFailedCreates);
-      executor.submit(task);
-      allocatedSize += containerSizeB;
-      containerIdx += 1;
-    }
-
-    // issue the command and then wait for it to finish
-    executor.shutdown();
-    try {
-      executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
-    } catch (InterruptedException e) {
-      LOGGER.error("Error creating volume:{} error:{}",
-          volume.getVolumeName(), e);
-      executor.shutdownNow();
-      Thread.currentThread().interrupt();
-    }
-
-    volume.setContainerIDs(containerIds);
-    return numFailedCreates.get() == 0;
-  }
-
-  private void deleteContainer(String containerID, boolean force) {
-    try {
-      Pipeline pipeline = storageClient.getContainer(containerID);
-      storageClient.deleteContainer(pipeline, force);
-    } catch (Exception e) {
-      LOGGER.error("Error deleting container Container:{} error:{}",
-          containerID, e);
-    }
-  }
-
-  private void deleteVolumeContainers(List<String> containers, boolean force)
-      throws CBlockException {
-    ThreadPoolExecutor executor = new ThreadPoolExecutor(
-        Math.min(numThreads, MAX_THREADS),
-        MAX_THREADS, 1, TimeUnit.SECONDS,
-        new ArrayBlockingQueue<>(MAX_QUEUE_CAPACITY),
-        new ThreadPoolExecutor.CallerRunsPolicy());
-
-    for (String deleteContainer : containers) {
-      if (deleteContainer != null) {
-        Runnable task = () -> deleteContainer(deleteContainer, force);
-        executor.submit(task);
-      }
-    }
-
-    // issue the command and then wait for it to finish
-    executor.shutdown();
-    try {
-      executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
-    } catch (InterruptedException e) {
-      LOGGER.error("Error deleting containers error:{}", e);
-      executor.shutdownNow();
-      Thread.currentThread().interrupt();
-    }
-  }
-
-  /**
-   * Called by CBlock server when creating a fresh volume. The core
-   * logic is adding needed information into in-memory meta data.
-   *
-   * @param userName the user name of the volume.
-   * @param volumeName the name of the volume.
-   * @param volumeSize the size of the volume.
-   * @param blockSize the block size of the volume.
-   * @throws CBlockException when the volume can not be created.
-   */
-  public synchronized void createVolume(String userName, String volumeName,
-      long volumeSize, int blockSize) throws CBlockException {
-    LOGGER.debug("createVolume:" + userName + ":" + volumeName);
-    if (user2VolumeMap.containsKey(userName)
-        && user2VolumeMap.get(userName).containsKey(volumeName)) {
-      throw new CBlockException("Volume already exist for "
-          + userName + ":" + volumeName);
-    }
-    if (volumeSize < blockSize) {
-      throw new CBlockException("Volume size smaller than block size? " +
-          "volume size:" + volumeSize + " block size:" + blockSize);
-    }
-    VolumeDescriptor volume
-        = new VolumeDescriptor(userName, volumeName, volumeSize, blockSize);
-    boolean success = createVolumeContainers(volume);
-    if (!success) {
-      // cleanup the containers and throw the exception
-      deleteVolumeContainers(volume.getContainerIDsList(), true);
-      throw new CBlockException("Error when creating volume:" + volumeName);
-    }
-    makeVolumeReady(userName, volumeName, volume);
-  }
-
-  /**
-   * Called by CBlock server to delete a specific volume. Mainly
-   * to check whether it can be deleted, and remove it from in-memory meta
-   * data.
-   *
-   * @param userName the user name of the volume.
-   * @param volumeName the name of the volume.
-   * @param force if set to false, only delete volume it is empty, otherwise
-   *              throw exception. if set to true, delete regardless.
-   * @throws CBlockException when the volume can not be deleted.
-   */
-  public synchronized void deleteVolume(String userName, String volumeName,
-      boolean force) throws CBlockException {
-    if (!user2VolumeMap.containsKey(userName)
-        || !user2VolumeMap.get(userName).containsKey(volumeName)) {
-      throw new CBlockException("Deleting non-exist volume "
-          + userName + ":" + volumeName);
-    }
-    if (!force && !user2VolumeMap.get(userName).get(volumeName).isEmpty()) {
-      throw new CBlockException("Deleting a non-empty volume without force!");
-    }
-    VolumeDescriptor volume = user2VolumeMap.get(userName).remove(volumeName);
-    deleteVolumeContainers(volume.getContainerIDsList(), force);
-    if (user2VolumeMap.get(userName).size() == 0) {
-      user2VolumeMap.remove(userName);
-    }
-  }
-
-  /**
-   * Called by CBlock server to get information of a specific volume.
-   *
-   * @param userName the user name of the volume.
-   * @param volumeName the name of the volume.
-   * @return a {@link VolumeInfo} object encapsulating the information of the
-   * volume.
-   * @throws CBlockException when the information can not be retrieved.
-   */
-  public synchronized VolumeInfo infoVolume(String userName, String volumeName)
-      throws CBlockException {
-    if (!user2VolumeMap.containsKey(userName)
-        || !user2VolumeMap.get(userName).containsKey(volumeName)) {
-      throw new CBlockException("Getting info for non-exist volume "
-          + userName + ":" + volumeName);
-    }
-    return user2VolumeMap.get(userName).get(volumeName).getInfo();
-  }
-
-  /**
-   * Called by CBlock server to check whether the given volume can be
-   * mounted, i.e. whether it can be found in the meta data.
-   *
-   * return a {@link MountVolumeResponse} with isValid flag to indicate
-   * whether the volume can be mounted or not.
-   *
-   * @param userName the user name of the volume.
-   * @param volumeName the name of the volume
-   * @return a {@link MountVolumeResponse} object encapsulating whether the
-   * volume is valid, and if yes, the requried information for client to
-   * read/write the volume.
-   */
-  public synchronized MountVolumeResponse isVolumeValid(
-      String userName, String volumeName) {
-    if (!user2VolumeMap.containsKey(userName)
-        || !user2VolumeMap.get(userName).containsKey(volumeName)) {
-      // in the case of invalid volume, no need to set any value other than
-      // isValid flag.
-      return new MountVolumeResponse(false, null, null, 0, 0, null, null);
-    }
-    VolumeDescriptor volume = user2VolumeMap.get(userName).get(volumeName);
-    return new MountVolumeResponse(true, userName,
-        volumeName, volume.getVolumeSize(), volume.getBlockSize(),
-        volume.getContainerPipelines(), volume.getPipelines());
-  }
-
-  /**
-   * Called by CBlock manager to list all volumes.
-   *
-   * @param userName the userName whose volume to be listed, if set to null,
-   *                 all volumes will be listed.
-   * @return a list of {@link VolumeDescriptor} representing all volumes
-   * requested.
-   */
-  public synchronized List<VolumeDescriptor> getAllVolume(String userName) {
-    ArrayList<VolumeDescriptor> allVolumes = new ArrayList<>();
-    if (userName == null) {
-      for (Map.Entry<String, HashMap<String, VolumeDescriptor>> entry
-          : user2VolumeMap.entrySet()) {
-        allVolumes.addAll(entry.getValue().values());
-      }
-    } else {
-      if (user2VolumeMap.containsKey(userName)) {
-        allVolumes.addAll(user2VolumeMap.get(userName).values());
-      }
-    }
-    return allVolumes;
-  }
-
-  /**
-   * Only for testing the behavior of create/delete volumes.
-   */
-  @VisibleForTesting
-  public VolumeDescriptor getVolume(String userName, String volumeName) {
-    if (!user2VolumeMap.containsKey(userName)
-        || !user2VolumeMap.get(userName).containsKey(volumeName)) {
-      return null;
-    }
-    return user2VolumeMap.get(userName).get(volumeName);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/package-info.java
deleted file mode 100644
index 4426e6d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.storage;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java
deleted file mode 100644
index beb9e32..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.util;
-
-/**
- * A simply class that generates key mappings. (e.g. from (userName, volumeName)
- * pair to a single string volumeKey.
- */
-public final class KeyUtil {
-  public static String getVolumeKey(String userName, String volumeName) {
-    return userName + ":" + volumeName;
-  }
-
-  public static String getContainerName(String userName, String volumeName,
-      int containerID) {
-    return getVolumeKey(userName, volumeName) + "#" + containerID;
-  }
-
-  public static String getUserNameFromVolumeKey(String key) {
-    return key.split(":")[0];
-  }
-
-  public static String getVolumeFromVolumeKey(String key) {
-    return key.split(":")[1];
-  }
-
-  public static boolean isValidVolumeKey(String key) {
-    return key.contains(":");
-  }
-
-  private KeyUtil() {
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/package-info.java
deleted file mode 100644
index 5b9aa0c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.util;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index 72f39fc..4c94e38 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -36,8 +36,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYPASSWORD_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED_DEFAULT;
 
 import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
@@ -73,7 +71,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -1497,23 +1494,6 @@ public class DFSUtil {
     return password;
   }
 
-  public static boolean isOzoneEnabled(Configuration conf) {
-    String securityEnabled = conf.get(CommonConfigurationKeysPublic
-            .HADOOP_SECURITY_AUTHENTICATION,
-        "simple");
-    boolean securityAuthorizationEnabled = conf.getBoolean(
-        CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
-        false);
-
-    if (securityEnabled.equals("kerberos") || securityAuthorizationEnabled) {
-      LOG.error("Ozone is not supported in a security enabled cluster. ");
-      return false;
-    } else {
-      return conf.getBoolean(OZONE_ENABLED,
-          OZONE_ENABLED_DEFAULT);
-    }
-  }
-
   /**
    * Converts a Date into an ISO-8601 formatted datetime string.
    */


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[30/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java
deleted file mode 100644
index eb18486..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.impl;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
-import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
-import org.apache.hadoop.ozone.container.common.interfaces.KeyManager;
-import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter;
-import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
-import org.apache.hadoop.utils.MetadataStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
-    .Result.NO_SUCH_KEY;
-
-/**
- * Key Manager impl.
- */
-public class KeyManagerImpl implements KeyManager {
-  static final Logger LOG =
-      LoggerFactory.getLogger(KeyManagerImpl.class);
-
-  private static final float LOAD_FACTOR = 0.75f;
-  private final ContainerManager containerManager;
-  private final Configuration conf;
-
-  /**
-   * Constructs a key Manager.
-   *
-   * @param containerManager - Container Manager.
-   */
-  public KeyManagerImpl(ContainerManager containerManager, Configuration conf) {
-    Preconditions.checkNotNull(containerManager, "Container manager cannot be" +
-        " null");
-    Preconditions.checkNotNull(conf, "Config cannot be null");
-    this.containerManager = containerManager;
-    this.conf = conf;
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public void putKey(Pipeline pipeline, KeyData data) throws IOException {
-    containerManager.readLock();
-    try {
-      // We are not locking the key manager since LevelDb serializes all actions
-      // against a single DB. We rely on DB level locking to avoid conflicts.
-      Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
-      String containerName = pipeline.getContainerName();
-      Preconditions.checkNotNull(containerName,
-          "Container name cannot be null");
-      ContainerData cData = containerManager.readContainer(containerName);
-      MetadataStore db = KeyUtils.getDB(cData, conf);
-
-      // This is a post condition that acts as a hint to the user.
-      // Should never fail.
-      Preconditions.checkNotNull(db, "DB cannot be null here");
-      db.put(data.getKeyName().getBytes(KeyUtils.ENCODING), data
-          .getProtoBufMessage().toByteArray());
-    } finally {
-      containerManager.readUnlock();
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public KeyData getKey(KeyData data) throws IOException {
-    containerManager.readLock();
-    try {
-      Preconditions.checkNotNull(data, "Key data cannot be null");
-      Preconditions.checkNotNull(data.getContainerName(),
-          "Container name cannot be null");
-      ContainerData cData = containerManager.readContainer(data
-          .getContainerName());
-      MetadataStore db = KeyUtils.getDB(cData, conf);
-
-      // This is a post condition that acts as a hint to the user.
-      // Should never fail.
-      Preconditions.checkNotNull(db, "DB cannot be null here");
-
-      byte[] kData = db.get(data.getKeyName().getBytes(KeyUtils.ENCODING));
-      if (kData == null) {
-        throw new StorageContainerException("Unable to find the key.",
-            NO_SUCH_KEY);
-      }
-      ContainerProtos.KeyData keyData =
-          ContainerProtos.KeyData.parseFrom(kData);
-      return KeyData.getFromProtoBuf(keyData);
-    } finally {
-      containerManager.readUnlock();
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public void deleteKey(Pipeline pipeline, String keyName)
-      throws IOException {
-    containerManager.readLock();
-    try {
-      Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
-      String containerName = pipeline.getContainerName();
-      Preconditions.checkNotNull(containerName,
-          "Container name cannot be null");
-      ContainerData cData = containerManager.readContainer(containerName);
-      MetadataStore db = KeyUtils.getDB(cData, conf);
-
-      // This is a post condition that acts as a hint to the user.
-      // Should never fail.
-      Preconditions.checkNotNull(db, "DB cannot be null here");
-      // Note : There is a race condition here, since get and delete
-      // are not atomic. Leaving it here since the impact is refusing
-      // to delete a key which might have just gotten inserted after
-      // the get check.
-
-      byte[] kData = db.get(keyName.getBytes(KeyUtils.ENCODING));
-      if (kData == null) {
-        throw new StorageContainerException("Unable to find the key.",
-            NO_SUCH_KEY);
-      }
-      db.delete(keyName.getBytes(KeyUtils.ENCODING));
-    } finally {
-      containerManager.readUnlock();
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public List<KeyData> listKey(
-      Pipeline pipeline, String prefix, String startKey, int count)
-      throws IOException {
-    Preconditions.checkNotNull(pipeline,
-        "Pipeline cannot be null.");
-    Preconditions.checkArgument(count > 0,
-        "Count must be a positive number.");
-    ContainerData cData = containerManager.readContainer(pipeline
-        .getContainerName());
-    MetadataStore db = KeyUtils.getDB(cData, conf);
-
-    List<KeyData> result = new ArrayList<KeyData>();
-    byte[] startKeyInBytes = startKey == null ? null :
-        DFSUtil.string2Bytes(startKey);
-    MetadataKeyFilter prefixFilter = new KeyPrefixFilter(prefix);
-    List<Map.Entry<byte[], byte[]>> range =
-        db.getSequentialRangeKVs(startKeyInBytes, count, prefixFilter);
-    for (Map.Entry<byte[], byte[]> entry : range) {
-      String keyName = KeyUtils.getKeyName(entry.getKey());
-      KeyData value = KeyUtils.getKeyData(entry.getValue());
-      KeyData data = new KeyData(value.getContainerName(), keyName);
-      result.add(data);
-    }
-    return result;
-  }
-
-  /**
-   * Shutdown keyManager.
-   */
-  @Override
-  public void shutdown() {
-    Preconditions.checkState(this.containerManager.hasWriteLock(), "asserts " +
-        "that we are holding the container manager lock when shutting down.");
-    KeyUtils.shutdownCache(ContainerCache.getInstance(conf));
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java
deleted file mode 100644
index 0123ab1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDeletionChoosingPolicy;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Randomly choosing containers for block deletion.
- */
-public class RandomContainerDeletionChoosingPolicy
-    implements ContainerDeletionChoosingPolicy {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(RandomContainerDeletionChoosingPolicy.class);
-
-  @Override
-  public List<ContainerData> chooseContainerForBlockDeletion(int count,
-      Map<String, ContainerStatus> candidateContainers)
-      throws StorageContainerException {
-    Preconditions.checkNotNull(candidateContainers,
-        "Internal assertion: candidate containers cannot be null");
-
-    int currentCount = 0;
-    List<ContainerData> result = new LinkedList<>();
-    ContainerStatus[] values = new ContainerStatus[candidateContainers.size()];
-    // to get a shuffle list
-    for (ContainerStatus entry : DFSUtil.shuffle(
-        candidateContainers.values().toArray(values))) {
-      if (currentCount < count) {
-        result.add(entry.getContainer());
-        currentCount++;
-
-        LOG.debug("Select container {} for block deletion, "
-            + "pending deletion blocks num: {}.",
-            entry.getContainer().getContainerName(),
-            entry.getNumPendingDeletionBlocks());
-      } else {
-        break;
-      }
-    }
-
-    return result;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
deleted file mode 100644
index 7ef91a9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.impl;
-
-/**
- * Storage location stats of datanodes that provide back store for containers.
- *
- */
-public class StorageLocationReport {
-  public static final StorageLocationReport[] EMPTY_ARRAY = {};
-
-  private final String id;
-  private final boolean failed;
-  private final long capacity;
-  private final long scmUsed;
-  private final long remaining;
-
-  public StorageLocationReport(String id, boolean failed,
-      long capacity, long scmUsed, long remaining) {
-    this.id = id;
-    this.failed = failed;
-    this.capacity = capacity;
-    this.scmUsed = scmUsed;
-    this.remaining = remaining;
-  }
-
-  public String getId() {
-    return id;
-  }
-
-  public boolean isFailed() {
-    return failed;
-  }
-
-  public long getCapacity() {
-    return capacity;
-  }
-
-  public long getScmUsed() {
-    return scmUsed;
-  }
-
-  public long getRemaining() {
-    return remaining;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java
deleted file mode 100644
index 3f4cdaa..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDeletionChoosingPolicy;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-
-/**
- * TopN Ordered choosing policy that choosing containers based on pending
- * deletion blocks' number.
- */
-public class TopNOrderedContainerDeletionChoosingPolicy
-    implements ContainerDeletionChoosingPolicy {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(TopNOrderedContainerDeletionChoosingPolicy.class);
-
-  /** customized comparator used to compare differentiate container status. **/
-  private static final Comparator<ContainerStatus> CONTAINER_STATUS_COMPARATOR
-      = new Comparator<ContainerStatus>() {
-        @Override
-        public int compare(ContainerStatus c1, ContainerStatus c2) {
-          return Integer.compare(c2.getNumPendingDeletionBlocks(),
-              c1.getNumPendingDeletionBlocks());
-        }
-      };
-
-  @Override
-  public List<ContainerData> chooseContainerForBlockDeletion(int count,
-      Map<String, ContainerStatus> candidateContainers)
-      throws StorageContainerException {
-    Preconditions.checkNotNull(candidateContainers,
-        "Internal assertion: candidate containers cannot be null");
-
-    List<ContainerData> result = new LinkedList<>();
-    List<ContainerStatus> orderedList = new LinkedList<>();
-    orderedList.addAll(candidateContainers.values());
-    Collections.sort(orderedList, CONTAINER_STATUS_COMPARATOR);
-
-    // get top N list ordered by pending deletion blocks' number
-    int currentCount = 0;
-    for (ContainerStatus entry : orderedList) {
-      if (currentCount < count) {
-        if (entry.getNumPendingDeletionBlocks() > 0) {
-          result.add(entry.getContainer());
-          currentCount++;
-
-          LOG.debug(
-              "Select container {} for block deletion, "
-                  + "pending deletion blocks num: {}.",
-              entry.getContainer().getContainerName(),
-              entry.getNumPendingDeletionBlocks());
-        } else {
-          LOG.debug("Stop looking for next container, there is no"
-              + " pending deletion block contained in remaining containers.");
-          break;
-        }
-      } else {
-        break;
-      }
-    }
-
-    return result;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/package-info.java
deleted file mode 100644
index 16da5d9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-/**
- This package is contains Ozone container implementation.
-**/
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java
deleted file mode 100644
index c933924..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-/**
- * Chunk Manager allows read, write, delete and listing of chunks in
- * a container.
- */
-public interface ChunkManager {
-
-  /**
-   * writes a given chunk.
-   * @param pipeline - Name and the set of machines that make this container.
-   * @param keyName - Name of the Key.
-   * @param info - ChunkInfo.
-   * @param stage - Chunk Stage write.
-   * @throws StorageContainerException
-   */
-  void writeChunk(Pipeline pipeline, String keyName,
-                  ChunkInfo info, byte[] data, ContainerProtos.Stage stage)
-      throws StorageContainerException;
-
-  /**
-   * reads the data defined by a chunk.
-   * @param pipeline - container pipeline.
-   * @param keyName - Name of the Key
-   * @param info - ChunkInfo.
-   * @return  byte array
-   * @throws StorageContainerException
-   *
-   * TODO: Right now we do not support partial reads and writes of chunks.
-   * TODO: Explore if we need to do that for ozone.
-   */
-  byte[] readChunk(Pipeline pipeline, String keyName, ChunkInfo info) throws
-      StorageContainerException;
-
-  /**
-   * Deletes a given chunk.
-   * @param pipeline  - Pipeline.
-   * @param keyName   - Key Name
-   * @param info  - Chunk Info
-   * @throws StorageContainerException
-   */
-  void deleteChunk(Pipeline pipeline, String keyName, ChunkInfo info) throws
-      StorageContainerException;
-
-  // TODO : Support list operations.
-
-  /**
-   * Shutdown the chunkManager.
-   */
-  void shutdown();
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java
deleted file mode 100644
index 3e0a283..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.impl.ContainerStatus;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-
-/**
- * This interface is used for choosing desired containers for
- * block deletion.
- */
-public interface ContainerDeletionChoosingPolicy {
-
-  /**
-   * Chooses desired containers for block deletion.
-   * @param count
-   *          how many to return
-   * @param candidateContainers
-   *          candidate containers collection
-   * @return container data list
-   * @throws StorageContainerException
-   */
-  List<ContainerData> chooseContainerForBlockDeletion(int count,
-      Map<String, ContainerStatus> candidateContainers)
-      throws StorageContainerException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
deleted file mode 100644
index 8aae004..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandRequestProto;
-
-/**
- * Dispatcher acts as the bridge between the transport layer and
- * the actual container layer. This layer is capable of transforming
- * protobuf objects into corresponding class and issue the function call
- * into the lower layers.
- *
- * The reply from the request is dispatched to the client.
- */
-public interface ContainerDispatcher {
-  /**
-   * Dispatches commands to container layer.
-   * @param msg - Command Request
-   * @return Command Response
-   */
-  ContainerCommandResponseProto dispatch(ContainerCommandRequestProto msg);
-
-  /**
-   * Initialize the Dispatcher.
-   */
-  void init();
-
-  /**
-   * Shutdown Dispatcher services.
-   */
-  void shutdown();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManager.java
deleted file mode 100644
index 9c5fcea..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManager.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport;
-
-import java.io.IOException;
-import java.nio.file.Path;
-
-/**
- * Returns physical path locations, where the containers will be created.
- */
-public interface ContainerLocationManager {
-  /**
-   * Returns the path where the container should be placed from a set of
-   * locations.
-   *
-   * @return A path where we should place this container and metadata.
-   * @throws IOException
-   */
-  Path getContainerPath() throws IOException;
-
-  /**
-   * Returns the path where the container Data file are stored.
-   *
-   * @return a path where we place the LevelDB and data files of a container.
-   * @throws IOException
-   */
-  Path getDataPath(String containerName) throws IOException;
-
-  /**
-   * Returns an array of storage location usage report.
-   * @return storage location usage report.
-   */
-  StorageLocationReport[] getLocationReport() throws IOException;
-
-  /**
-   * Supports clean shutdown of container.
-   *
-   * @throws IOException
-   */
-  void shutdown() throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManagerMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManagerMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManagerMXBean.java
deleted file mode 100644
index 88e6148..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManagerMXBean.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport;
-
-import java.io.IOException;
-
-/**
- * Returns physical path locations, where the containers will be created.
- */
-public interface ContainerLocationManagerMXBean {
-
-  /**
-   * Returns an array of storage location usage report.
-   *
-   * @return storage location usage report.
-   */
-  StorageLocationReport[] getLocationReport() throws IOException;
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
deleted file mode 100644
index 04afc3c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
-import org.apache.hadoop.hdfs.util.RwLock;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-import java.util.List;
-
-/**
- * Interface for container operations.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public interface ContainerManager extends RwLock {
-
-  /**
-   * Init call that sets up a container Manager.
-   *
-   * @param config        - Configuration.
-   * @param containerDirs - List of Metadata Container locations.
-   * @param datanodeID - Datanode ID
-   * @throws StorageContainerException
-   */
-  void init(Configuration config, List<StorageLocation> containerDirs,
-      DatanodeID datanodeID) throws IOException;
-
-  /**
-   * Creates a container with the given name.
-   *
-   * @param pipeline      -- Nodes which make up this container.
-   * @param containerData - Container Name and metadata.
-   * @throws StorageContainerException
-   */
-  void createContainer(Pipeline pipeline, ContainerData containerData)
-      throws StorageContainerException;
-
-  /**
-   * Deletes an existing container.
-   *
-   * @param pipeline      - nodes that make this container.
-   * @param containerName - name of the container.
-   * @param forceDelete   - whether this container should be deleted forcibly.
-   * @throws StorageContainerException
-   */
-  void deleteContainer(Pipeline pipeline, String containerName,
-      boolean forceDelete) throws StorageContainerException;
-
-  /**
-   * Update an existing container.
-   *
-   * @param pipeline container nodes
-   * @param containerName name of the container
-   * @param data container data
-   * @param forceUpdate if true, update container forcibly.
-   * @throws StorageContainerException
-   */
-  void updateContainer(Pipeline pipeline, String containerName,
-      ContainerData data, boolean forceUpdate) throws StorageContainerException;
-
-  /**
-   * As simple interface for container Iterations.
-   *
-   * @param prefix - Return only values matching this prefix
-   * @param count   - how many to return
-   * @param prevKey - Previous key - Server returns results from this point.
-   * @param data    - Actual containerData
-   * @throws StorageContainerException
-   */
-  void listContainer(String prefix, long count, String prevKey,
-                     List<ContainerData> data)
-      throws StorageContainerException;
-
-  /**
-   * Choose containers for block deletion.
-   *
-   * @param count   - how many to return
-   * @throws StorageContainerException
-   */
-  List<ContainerData> chooseContainerForBlockDeletion(int count)
-      throws StorageContainerException;
-
-  /**
-   * Get metadata about a specific container.
-   *
-   * @param containerName - Name of the container
-   * @return ContainerData - Container Data.
-   * @throws StorageContainerException
-   */
-  ContainerData readContainer(String containerName)
-      throws StorageContainerException;
-
-  /**
-   * Closes a open container, if it is already closed or does not exist a
-   * StorageContainerException is thrown.
-   * @param containerName - Name of the container.
-   * @throws StorageContainerException
-   */
-  void closeContainer(String containerName)
-      throws StorageContainerException, NoSuchAlgorithmException;
-
-  /**
-   * Checks if a container exists.
-   * @param containerName - Name of the container.
-   * @return true if the container is open false otherwise.
-   * @throws StorageContainerException  - Throws Exception if we are not
-   * able to find the container.
-   */
-  boolean isOpen(String containerName) throws StorageContainerException;
-
-  /**
-   * Supports clean shutdown of container.
-   *
-   * @throws StorageContainerException
-   */
-  void shutdown() throws IOException;
-
-  /**
-   * Sets the Chunk Manager.
-   *
-   * @param chunkManager - ChunkManager.
-   */
-  void setChunkManager(ChunkManager chunkManager);
-
-  /**
-   * Gets the Chunk Manager.
-   *
-   * @return ChunkManager.
-   */
-  ChunkManager getChunkManager();
-
-  /**
-   * Sets the Key Manager.
-   *
-   * @param keyManager - Key Manager.
-   */
-  void setKeyManager(KeyManager keyManager);
-
-  /**
-   * Gets the Key Manager.
-   *
-   * @return KeyManager.
-   */
-  KeyManager getKeyManager();
-
-  /**
-   * Get the Node Report of container storage usage.
-   * @return node report.
-   */
-  SCMNodeReport getNodeReport() throws IOException;
-
-  /**
-   * Gets container report.
-   * @return container report.
-   * @throws IOException
-   */
-  ContainerReportsRequestProto getContainerReport() throws IOException;
-
-  /**
-   * Gets container reports.
-   * @return List of all closed containers.
-   * @throws IOException
-   */
-  List<ContainerData> getContainerReports() throws IOException;
-
-  /**
-   * Increase pending deletion blocks count number of specified container.
-   *
-   * @param numBlocks
-   *          increment  count number
-   * @param containerId
-   *          container id
-   */
-  void incrPendingDeletionBlocks(int numBlocks, String containerId);
-
-  /**
-   * Decrease pending deletion blocks count number of specified container.
-   *
-   * @param numBlocks
-   *          decrement count number
-   * @param containerId
-   *          container id
-   */
-  void decrPendingDeletionBlocks(int numBlocks, String containerId);
-
-  /**
-   * Increase the read count of the container.
-   * @param containerName - Name of the container.
-   */
-  void incrReadCount(String containerName);
-
-  /**
-   * Increse the read counter for bytes read from the container.
-   * @param containerName - Name of the container.
-   * @param readBytes - bytes read from the container.
-   */
-  void incrReadBytes(String containerName, long readBytes);
-
-
-  /**
-   * Increase the write count of the container.
-   * @param containerName - Name of the container.
-   */
-  void incrWriteCount(String containerName);
-
-  /**
-   * Increase the write counter for bytes write into the container.
-   * @param containerName - Name of the container.
-   * @param writeBytes - bytes write into the container.
-   */
-  void incrWriteBytes(String containerName, long writeBytes);
-
-  /**
-   * Increase the bytes used by the container.
-   * @param containerName - Name of the container.
-   * @param used - additional bytes used by the container.
-   * @return the current bytes used.
-   */
-  long incrBytesUsed(String containerName, long used);
-
-  /**
-   * Decrease the bytes used by the container.
-   * @param containerName - Name of the container.
-   * @param used - additional bytes reclaimed by the container.
-   * @return the current bytes used.
-   */
-  long decrBytesUsed(String containerName, long used);
-
-  /**
-   * Get the bytes used by the container.
-   * @param containerName - Name of the container.
-   * @return the current bytes used by the container.
-   */
-  long getBytesUsed(String containerName);
-
-  /**
-   * Get the number of keys in the container.
-   * @param containerName - Name of the container.
-   * @return the current key count.
-   */
-  long getNumKeys(String containerName);
-
-  /**
-   * Get the container report state to send via HB to SCM.
-   * @return container report state.
-   */
-  ReportState getContainerReportState();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerReportManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerReportManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerReportManager.java
deleted file mode 100644
index 7d2ba0a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerReportManager.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-
-/**
- * Interface for container report manager operations.
- */
-public interface ContainerReportManager {
-
-  /**
-   * Get the container report state.
-   * @return the container report state.
-   */
-  ReportState getContainerReportState();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/KeyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/KeyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/KeyManager.java
deleted file mode 100644
index a613d2a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/KeyManager.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * KeyManager deals with Key Operations in the container Level.
- */
-public interface KeyManager {
-  /**
-   * Puts or overwrites a key.
-   *
-   * @param pipeline - Pipeline.
-   * @param data     - Key Data.
-   * @throws IOException
-   */
-  void putKey(Pipeline pipeline, KeyData data) throws IOException;
-
-  /**
-   * Gets an existing key.
-   *
-   * @param data - Key Data.
-   * @return Key Data.
-   * @throws IOException
-   */
-  KeyData getKey(KeyData data) throws IOException;
-
-  /**
-   * Deletes an existing Key.
-   *
-   * @param pipeline - Pipeline.
-   * @param keyName  Key Data.
-   * @throws StorageContainerException
-   */
-  void deleteKey(Pipeline pipeline, String keyName)
-      throws IOException;
-
-  /**
-   * List keys in a container.
-   *
-   * @param pipeline - pipeline.
-   * @param prefix   - Prefix in needed.
-   * @param startKey  - Key to start from, EMPTY_STRING to begin.
-   * @param count    - Number of keys to return.
-   * @return List of Keys that match the criteria.
-   */
-  List<KeyData> listKey(Pipeline pipeline, String prefix, String startKey,
-      int count) throws IOException;
-
-  /**
-   * Shutdown keyManager.
-   */
-  void shutdown();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/package-info.java
deleted file mode 100644
index d83bf95..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/package-info.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.interfaces;
-/**
- This package contains common ozone container interfaces.
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/package-info.java
deleted file mode 100644
index 1638a36..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common;
-/**
-  Common Container Layer. At this layer the abstractions are:
-
- 1. Containers - Both data and metadata containers.
- 2. Keys - Key/Value pairs that live inside a container.
- 3. Chunks - Keys can be composed of many chunks.
-
- Ozone uses these abstractions to build Volumes, Buckets and Keys.
-
- **/
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
deleted file mode 100644
index de55d96..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ /dev/null
@@ -1,385 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
-    .CloseContainerHandler;
-import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.CommandDispatcher;
-import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.ContainerReportHandler;
-import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DeleteBlocksCommandHandler;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * State Machine Class.
- */
-public class DatanodeStateMachine implements Closeable {
-  @VisibleForTesting
-  static final Logger LOG =
-      LoggerFactory.getLogger(DatanodeStateMachine.class);
-  private final ExecutorService executorService;
-  private final Configuration conf;
-  private final SCMConnectionManager connectionManager;
-  private final long heartbeatFrequency;
-  private StateContext context;
-  private final OzoneContainer container;
-  private DatanodeID datanodeID = null;
-  private final CommandDispatcher commandDispatcher;
-  private long commandsHandled;
-  private AtomicLong nextHB;
-  private Thread stateMachineThread = null;
-  private Thread cmdProcessThread = null;
-
-  /**
-   * Constructs a a datanode state machine.
-   *
-   * @param datanodeID - DatanodeID used to identify a datanode
-   * @param conf - Configuration.
-   */
-  public DatanodeStateMachine(DatanodeID datanodeID,
-      Configuration conf) throws IOException {
-    this.conf = conf;
-    executorService = HadoopExecutors.newCachedThreadPool(
-                new ThreadFactoryBuilder().setDaemon(true)
-            .setNameFormat("Datanode State Machine Thread - %d").build());
-    connectionManager = new SCMConnectionManager(conf);
-    context = new StateContext(this.conf, DatanodeStates.getInitState(), this);
-    heartbeatFrequency = TimeUnit.SECONDS.toMillis(
-        OzoneClientUtils.getScmHeartbeatInterval(conf));
-    container = new OzoneContainer(datanodeID, new OzoneConfiguration(conf));
-    this.datanodeID = datanodeID;
-    nextHB = new AtomicLong(Time.monotonicNow());
-
-     // When we add new handlers just adding a new handler here should do the
-     // trick.
-    commandDispatcher = CommandDispatcher.newBuilder()
-        .addHandler(new ContainerReportHandler())
-        .addHandler(new CloseContainerHandler())
-        .addHandler(new DeleteBlocksCommandHandler(
-            container.getContainerManager(), conf))
-        .setConnectionManager(connectionManager)
-        .setContainer(container)
-        .setContext(context)
-        .build();
-  }
-
-  public void setDatanodeID(DatanodeID datanodeID) {
-    this.datanodeID = datanodeID;
-  }
-
-  /**
-   *
-   * Return DatanodeID if set, return null otherwise.
-   *
-   * @return datanodeID
-   */
-  public DatanodeID getDatanodeID() {
-    return this.datanodeID;
-  }
-
-  /**
-   * Returns the Connection manager for this state machine.
-   *
-   * @return - SCMConnectionManager.
-   */
-  public SCMConnectionManager getConnectionManager() {
-    return connectionManager;
-  }
-
-  public OzoneContainer getContainer() {
-    return this.container;
-  }
-
-  /**
-   * Runs the state machine at a fixed frequency.
-   */
-  private void start() throws IOException {
-    long now = 0;
-
-    container.start();
-    initCommandHandlerThread(conf);
-    while (context.getState() != DatanodeStates.SHUTDOWN) {
-      try {
-        LOG.debug("Executing cycle Number : {}", context.getExecutionCount());
-        nextHB.set(Time.monotonicNow() + heartbeatFrequency);
-        context.setReportState(container.getNodeReport());
-        context.setContainerReportState(container.getContainerReportState());
-        context.execute(executorService, heartbeatFrequency,
-            TimeUnit.MILLISECONDS);
-        now = Time.monotonicNow();
-        if (now < nextHB.get()) {
-          Thread.sleep(nextHB.get() - now);
-        }
-      } catch (InterruptedException e) {
-        // Ignore this exception.
-      } catch (Exception e) {
-        LOG.error("Unable to finish the execution.", e);
-      }
-    }
-  }
-
-  /**
-   * Gets the current context.
-   *
-   * @return StateContext
-   */
-  public StateContext getContext() {
-    return context;
-  }
-
-  /**
-   * Sets the current context.
-   *
-   * @param context - Context
-   */
-  public void setContext(StateContext context) {
-    this.context = context;
-  }
-
-  /**
-   * Closes this stream and releases any system resources associated with it. If
-   * the stream is already closed then invoking this method has no effect.
-   * <p>
-   * <p> As noted in {@link AutoCloseable#close()}, cases where the close may
-   * fail require careful attention. It is strongly advised to relinquish the
-   * underlying resources and to internally <em>mark</em> the {@code Closeable}
-   * as closed, prior to throwing the {@code IOException}.
-   *
-   * @throws IOException if an I/O error occurs
-   */
-  @Override
-  public void close() throws IOException {
-    if (stateMachineThread != null) {
-      stateMachineThread.interrupt();
-    }
-    if (cmdProcessThread != null) {
-      cmdProcessThread.interrupt();
-    }
-    context.setState(DatanodeStates.getLastState());
-    executorService.shutdown();
-    try {
-      if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
-        executorService.shutdownNow();
-      }
-
-      if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
-        LOG.error("Unable to shutdown state machine properly.");
-      }
-    } catch (InterruptedException e) {
-      LOG.error("Error attempting to shutdown.", e);
-      executorService.shutdownNow();
-      Thread.currentThread().interrupt();
-    }
-
-    if (connectionManager != null) {
-      connectionManager.close();
-    }
-
-    if(container != null) {
-      container.stop();
-    }
-  }
-
-  /**
-   * States that a datanode  can be in. GetNextState will move this enum from
-   * getInitState to getLastState.
-   */
-  public enum DatanodeStates {
-    INIT(1),
-    RUNNING(2),
-    SHUTDOWN(3);
-    private final int value;
-
-    /**
-     * Constructs ContainerStates.
-     *
-     * @param value  Enum Value
-     */
-    DatanodeStates(int value) {
-      this.value = value;
-    }
-
-    /**
-     * Returns the first State.
-     *
-     * @return First State.
-     */
-    public static DatanodeStates getInitState() {
-      return INIT;
-    }
-
-    /**
-     * The last state of endpoint states.
-     *
-     * @return last state.
-     */
-    public static DatanodeStates getLastState() {
-      return SHUTDOWN;
-    }
-
-    /**
-     * returns the numeric value associated with the endPoint.
-     *
-     * @return int.
-     */
-    public int getValue() {
-      return value;
-    }
-
-    /**
-     * Returns the next logical state that endPoint should move to. This
-     * function assumes the States are sequentially numbered.
-     *
-     * @return NextState.
-     */
-    public DatanodeStates getNextState() {
-      if (this.value < getLastState().getValue()) {
-        int stateValue = this.getValue() + 1;
-        for (DatanodeStates iter : values()) {
-          if (stateValue == iter.getValue()) {
-            return iter;
-          }
-        }
-      }
-      return getLastState();
-    }
-  }
-
-  /**
-   * Start datanode state machine as a single thread daemon.
-   */
-  public void startDaemon() {
-    Runnable startStateMachineTask = () -> {
-      try {
-        start();
-        LOG.info("Ozone container server started.");
-      } catch (Exception ex) {
-        LOG.error("Unable to start the DatanodeState Machine", ex);
-      }
-    };
-    stateMachineThread =  new ThreadFactoryBuilder()
-        .setDaemon(true)
-        .setNameFormat("Datanode State Machine Thread - %d")
-        .build().newThread(startStateMachineTask);
-    stateMachineThread.start();
-  }
-
-  /**
-   * Stop the daemon thread of the datanode state machine.
-   */
-  public synchronized void stopDaemon() {
-    try {
-      context.setState(DatanodeStates.SHUTDOWN);
-      this.close();
-      LOG.info("Ozone container server stopped.");
-    } catch (IOException e) {
-      LOG.error("Stop ozone container server failed.", e);
-    }
-  }
-
-  /**
-   *
-   * Check if the datanode state machine daemon is stopped.
-   *
-   * @return True if datanode state machine daemon is stopped
-   * and false otherwise.
-   */
-  @VisibleForTesting
-  public boolean isDaemonStopped() {
-    return this.executorService.isShutdown()
-        && this.getContext().getExecutionCount() == 0
-        && this.getContext().getState() == DatanodeStates.SHUTDOWN;
-  }
-
-  /**
-   * Create a command handler thread.
-   *
-   * @param config
-   */
-  private void initCommandHandlerThread(Configuration config) {
-
-    /**
-     * Task that periodically checks if we have any outstanding commands.
-     * It is assumed that commands can be processed slowly and in order.
-     * This assumption might change in future. Right now due to this assumption
-     * we have single command  queue process thread.
-     */
-    Runnable processCommandQueue = () -> {
-      long now;
-      while (getContext().getState() != DatanodeStates.SHUTDOWN) {
-        SCMCommand command = getContext().getNextCommand();
-        if (command != null) {
-          commandDispatcher.handle(command);
-          commandsHandled++;
-        } else {
-          try {
-            // Sleep till the next HB + 1 second.
-            now = Time.monotonicNow();
-            if (nextHB.get() > now) {
-              Thread.sleep((nextHB.get() - now) + 1000L);
-            }
-          } catch (InterruptedException e) {
-            // Ignore this exception.
-          }
-        }
-      }
-    };
-
-    // We will have only one thread for command processing in a datanode.
-    cmdProcessThread = getCommandHandlerThread(processCommandQueue);
-    cmdProcessThread.start();
-  }
-
-  private Thread getCommandHandlerThread(Runnable processCommandQueue) {
-    Thread handlerThread = new Thread(processCommandQueue);
-    handlerThread.setDaemon(true);
-    handlerThread.setName("Command processor thread");
-    handlerThread.setUncaughtExceptionHandler((Thread t, Throwable e) -> {
-      // Let us just restart this thread after logging a critical error.
-      // if this thread is not running we cannot handle commands from SCM.
-      LOG.error("Critical Error : Command processor thread encountered an " +
-          "error. Thread: {}", t.toString(), e);
-      getCommandHandlerThread(processCommandQueue).start();
-    });
-    return handlerThread;
-  }
-
-  /**
-   * Returns the number of commands handled  by the datanode.
-   * @return  count
-   */
-  @VisibleForTesting
-  public long getCommandHandled() {
-    return commandsHandled;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java
deleted file mode 100644
index 4f1f47f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.protocol.VersionResponse;
-import org.apache.hadoop.ozone.protocolPB
-    .StorageContainerDatanodeProtocolClientSideTranslatorPB;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.time.ZonedDateTime;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-/**
- * Endpoint is used as holder class that keeps state around the RPC endpoint.
- */
-public class EndpointStateMachine
-    implements Closeable, EndpointStateMachineMBean {
-  static final Logger
-      LOG = LoggerFactory.getLogger(EndpointStateMachine.class);
-  private final StorageContainerDatanodeProtocolClientSideTranslatorPB endPoint;
-  private final AtomicLong missedCount;
-  private final InetSocketAddress address;
-  private final Lock lock;
-  private final Configuration conf;
-  private EndPointStates state;
-  private VersionResponse version;
-  private ZonedDateTime lastSuccessfulHeartbeat;
-
-  /**
-   * Constructs RPC Endpoints.
-   *
-   * @param endPoint - RPC endPoint.
-   */
-  public EndpointStateMachine(InetSocketAddress address,
-      StorageContainerDatanodeProtocolClientSideTranslatorPB endPoint,
-      Configuration conf) {
-    this.endPoint = endPoint;
-    this.missedCount = new AtomicLong(0);
-    this.address = address;
-    state = EndPointStates.getInitState();
-    lock = new ReentrantLock();
-    this.conf = conf;
-  }
-
-  /**
-   * Takes a lock on this EndPoint so that other threads don't use this while we
-   * are trying to communicate via this endpoint.
-   */
-  public void lock() {
-    lock.lock();
-  }
-
-  /**
-   * Unlocks this endpoint.
-   */
-  public void unlock() {
-    lock.unlock();
-  }
-
-  /**
-   * Returns the version that we read from the server if anyone asks .
-   *
-   * @return - Version Response.
-   */
-  public VersionResponse getVersion() {
-    return version;
-  }
-
-  /**
-   * Sets the Version reponse we recieved from the SCM.
-   *
-   * @param version VersionResponse
-   */
-  public void setVersion(VersionResponse version) {
-    this.version = version;
-  }
-
-  /**
-   * Returns the current State this end point is in.
-   *
-   * @return - getState.
-   */
-  public EndPointStates getState() {
-    return state;
-  }
-
-  @Override
-  public int getVersionNumber() {
-    if (version != null) {
-      return version.getProtobufMessage().getSoftwareVersion();
-    } else {
-      return -1;
-    }
-  }
-
-  /**
-   * Sets the endpoint state.
-   *
-   * @param epState - end point state.
-   */
-  public EndPointStates setState(EndPointStates epState) {
-    this.state = epState;
-    return this.state;
-  }
-
-  /**
-   * Closes the connection.
-   *
-   * @throws IOException
-   */
-  @Override
-  public void close() throws IOException {
-    if (endPoint != null) {
-      endPoint.close();
-    }
-  }
-
-  /**
-   * We maintain a count of how many times we missed communicating with a
-   * specific SCM. This is not made atomic since the access to this is always
-   * guarded by the read or write lock. That is, it is serialized.
-   */
-  public void incMissed() {
-    this.missedCount.incrementAndGet();
-  }
-
-  /**
-   * Returns the value of the missed count.
-   *
-   * @return int
-   */
-  public long getMissedCount() {
-    return this.missedCount.get();
-  }
-
-  @Override
-  public String getAddressString() {
-    return getAddress().toString();
-  }
-
-  public void zeroMissedCount() {
-    this.missedCount.set(0);
-  }
-
-  /**
-   * Returns the InetAddress of the endPoint.
-   *
-   * @return - EndPoint.
-   */
-  public InetSocketAddress getAddress() {
-    return this.address;
-  }
-
-  /**
-   * Returns real RPC endPoint.
-   *
-   * @return rpc client.
-   */
-  public StorageContainerDatanodeProtocolClientSideTranslatorPB
-      getEndPoint() {
-    return endPoint;
-  }
-
-  /**
-   * Returns the string that represents this endpoint.
-   *
-   * @return - String
-   */
-  public String toString() {
-    return address.toString();
-  }
-
-  /**
-   * Logs exception if needed.
-   *  @param ex         - Exception
-   */
-  public void logIfNeeded(Exception ex) {
-    LOG.trace("Incrementing the Missed count. Ex : {}", ex);
-    this.incMissed();
-    if (this.getMissedCount() % OzoneClientUtils.getLogWarnInterval(conf) ==
-        0) {
-      LOG.warn("Unable to communicate to SCM server at {}. We have not been " +
-              "able to communicate to this SCM server for past {} seconds.",
-          this.getAddress().getHostString() + ":" + this.getAddress().getPort(),
-          this.getMissedCount() * OzoneClientUtils.getScmHeartbeatInterval(
-              this.conf));
-    }
-  }
-
-
-  /**
-   * States that an Endpoint can be in.
-   * <p>
-   * This is a sorted list of states that EndPoint will traverse.
-   * <p>
-   * GetNextState will move this enum from getInitState to getLastState.
-   */
-  public enum EndPointStates {
-    GETVERSION(1),
-    REGISTER(2),
-    HEARTBEAT(3),
-    SHUTDOWN(4); // if you add value after this please edit getLastState too.
-    private final int value;
-
-    /**
-     * Constructs endPointStates.
-     *
-     * @param value  state.
-     */
-    EndPointStates(int value) {
-      this.value = value;
-    }
-
-    /**
-     * Returns the first State.
-     *
-     * @return First State.
-     */
-    public static EndPointStates getInitState() {
-      return GETVERSION;
-    }
-
-    /**
-     * The last state of endpoint states.
-     *
-     * @return last state.
-     */
-    public static EndPointStates getLastState() {
-      return SHUTDOWN;
-    }
-
-    /**
-     * returns the numeric value associated with the endPoint.
-     *
-     * @return int.
-     */
-    public int getValue() {
-      return value;
-    }
-
-    /**
-     * Returns the next logical state that endPoint should move to.
-     * The next state is computed by adding 1 to the current state.
-     *
-     * @return NextState.
-     */
-    public EndPointStates getNextState() {
-      if (this.getValue() < getLastState().getValue()) {
-        int stateValue = this.getValue() + 1;
-        for (EndPointStates iter : values()) {
-          if (stateValue == iter.getValue()) {
-            return iter;
-          }
-        }
-      }
-      return getLastState();
-    }
-  }
-
-  public long getLastSuccessfulHeartbeat() {
-    return lastSuccessfulHeartbeat == null ?
-        0 :
-        lastSuccessfulHeartbeat.toEpochSecond();
-  }
-
-  public void setLastSuccessfulHeartbeat(
-      ZonedDateTime lastSuccessfulHeartbeat) {
-    this.lastSuccessfulHeartbeat = lastSuccessfulHeartbeat;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachineMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachineMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachineMBean.java
deleted file mode 100644
index 4f64bde..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachineMBean.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine;
-
-
-/**
- * JMX representation of an EndpointStateMachine.
- */
-public interface EndpointStateMachineMBean {
-
-  long getMissedCount();
-
-  String getAddressString();
-
-  EndpointStateMachine.EndPointStates getState();
-
-  int getVersionNumber();
-
-  long getLastSuccessfulHeartbeat();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java
deleted file mode 100644
index a6767c2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.protocolPB
-    .StorageContainerDatanodeProtocolClientSideTranslatorPB;
-import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolPB;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.ObjectName;
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.*;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-/**
- * SCMConnectionManager - Acts as a class that manages the membership
- * information of the SCMs that we are working with.
- */
-public class SCMConnectionManager
-    implements Closeable, SCMConnectionManagerMXBean {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(SCMConnectionManager.class);
-
-  private final ReadWriteLock mapLock;
-  private final Map<InetSocketAddress, EndpointStateMachine> scmMachines;
-
-  private final int rpcTimeout;
-  private final Configuration conf;
-  private final ObjectName jmxBean;
-
-  public SCMConnectionManager(Configuration conf) {
-    this.mapLock = new ReentrantReadWriteLock();
-    Long timeOut = OzoneClientUtils.getScmRpcTimeOutInMilliseconds(conf);
-    this.rpcTimeout = timeOut.intValue();
-    this.scmMachines = new HashMap<>();
-    this.conf = conf;
-    jmxBean = MBeans.register("OzoneDataNode",
-        "SCMConnectionManager",
-        this);
-  }
-
-
-  /**
-   * Returns Config.
-   *
-   * @return ozoneConfig.
-   */
-  public Configuration getConf() {
-    return conf;
-  }
-
-  /**
-   * Get RpcTimeout.
-   *
-   * @return - Return RPC timeout.
-   */
-  public int getRpcTimeout() {
-    return rpcTimeout;
-  }
-
-
-  /**
-   * Takes a read lock.
-   */
-  public void readLock() {
-    this.mapLock.readLock().lock();
-  }
-
-  /**
-   * Releases the read lock.
-   */
-  public void readUnlock() {
-    this.mapLock.readLock().unlock();
-  }
-
-  /**
-   * Takes the write lock.
-   */
-  public void writeLock() {
-    this.mapLock.writeLock().lock();
-  }
-
-  /**
-   * Releases the write lock.
-   */
-  public void writeUnlock() {
-    this.mapLock.writeLock().unlock();
-  }
-
-  /**
-   * adds a new SCM machine to the target set.
-   *
-   * @param address - Address of the SCM machine to send heatbeat to.
-   * @throws IOException
-   */
-  public void addSCMServer(InetSocketAddress address) throws IOException {
-    writeLock();
-    try {
-      if (scmMachines.containsKey(address)) {
-        LOG.warn("Trying to add an existing SCM Machine to Machines group. " +
-            "Ignoring the request.");
-        return;
-      }
-      RPC.setProtocolEngine(conf, StorageContainerDatanodeProtocolPB.class,
-          ProtobufRpcEngine.class);
-      long version =
-          RPC.getProtocolVersion(StorageContainerDatanodeProtocolPB.class);
-
-      StorageContainerDatanodeProtocolPB rpcProxy = RPC.getProxy(
-          StorageContainerDatanodeProtocolPB.class, version,
-          address, UserGroupInformation.getCurrentUser(), conf,
-          NetUtils.getDefaultSocketFactory(conf), getRpcTimeout());
-
-      StorageContainerDatanodeProtocolClientSideTranslatorPB rpcClient =
-          new StorageContainerDatanodeProtocolClientSideTranslatorPB(rpcProxy);
-
-      EndpointStateMachine endPoint =
-          new EndpointStateMachine(address, rpcClient, conf);
-      scmMachines.put(address, endPoint);
-    } finally {
-      writeUnlock();
-    }
-  }
-
-  /**
-   * Removes a  SCM machine for the target set.
-   *
-   * @param address - Address of the SCM machine to send heatbeat to.
-   * @throws IOException
-   */
-  public void removeSCMServer(InetSocketAddress address) throws IOException {
-    writeLock();
-    try {
-      if (!scmMachines.containsKey(address)) {
-        LOG.warn("Trying to remove a non-existent SCM machine. " +
-            "Ignoring the request.");
-        return;
-      }
-
-      EndpointStateMachine endPoint = scmMachines.get(address);
-      endPoint.close();
-      scmMachines.remove(address);
-    } finally {
-      writeUnlock();
-    }
-  }
-
-  /**
-   * Returns all known RPCEndpoints.
-   *
-   * @return - List of RPC Endpoints.
-   */
-  public Collection<EndpointStateMachine> getValues() {
-    return scmMachines.values();
-  }
-
-  @Override
-  public void close() throws IOException {
-    getValues().forEach(endpointStateMachine
-        -> IOUtils.cleanupWithLogger(LOG, endpointStateMachine));
-    MBeans.unregister(jmxBean);
-  }
-
-  @Override
-  public List<EndpointStateMachineMBean> getSCMServers() {
-    readLock();
-    try {
-      return Collections
-          .unmodifiableList(new ArrayList<>(scmMachines.values()));
-
-    } finally {
-      readUnlock();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManagerMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManagerMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManagerMXBean.java
deleted file mode 100644
index 25ef163..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManagerMXBean.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine;
-
-import java.util.List;
-
-/**
- * JMX information about the connected SCM servers.
- */
-public interface SCMConnectionManagerMXBean {
-
-  List<EndpointStateMachineMBean> getSCMServers();
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[20/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/closer/ContainerCloser.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/closer/ContainerCloser.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/closer/ContainerCloser.java
deleted file mode 100644
index 9371387..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/closer/ContainerCloser.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.ozone.scm.container.closer;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_CONTAINER_REPORT_INTERVAL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT;
-
-/**
- * A class that manages closing of containers. This allows transition from a
- * open but full container to a closed container, to which no data is written.
- */
-public class ContainerCloser {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ContainerCloser.class);
-  private static final long MULTIPLIER = 3L;
-  private static final int CLEANUP_WATER_MARK = 1000;
-  private final NodeManager nodeManager;
-  private final Map<String, Long> commandIssued;
-  private final Configuration configuration;
-  private final AtomicInteger mapCount;
-  private final long reportInterval;
-  private final AtomicInteger threadRunCount;
-  private final AtomicBoolean isRunning;
-
-  /**
-   * Constructs the ContainerCloser class.
-   *
-   * @param nodeManager - NodeManager
-   * @param conf -   Configuration
-   */
-  public ContainerCloser(NodeManager nodeManager, Configuration conf) {
-    Preconditions.checkNotNull(nodeManager);
-    Preconditions.checkNotNull(conf);
-    this.nodeManager = nodeManager;
-    this.configuration = conf;
-    this.commandIssued = new ConcurrentHashMap<>();
-    this.mapCount = new AtomicInteger(0);
-    this.threadRunCount = new AtomicInteger(0);
-    this.isRunning = new AtomicBoolean(false);
-    this.reportInterval = this.configuration.getTimeDuration(
-        OZONE_CONTAINER_REPORT_INTERVAL,
-        OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT, TimeUnit.SECONDS);
-    Preconditions.checkState(this.reportInterval > 0,
-        "report interval has to be greater than 0");
-  }
-
-  @VisibleForTesting
-  public static int getCleanupWaterMark() {
-    return CLEANUP_WATER_MARK;
-  }
-
-  /**
-   * Sends a Container Close command to the data nodes where this container
-   * lives.
-   *
-   * @param info - ContainerInfo.
-   */
-  public void close(OzoneProtos.SCMContainerInfo info) {
-
-    if (commandIssued.containsKey(info.getContainerName())) {
-      // We check if we issued a close command in last 3 * reportInterval secs.
-      long commandQueueTime = commandIssued.get(info.getContainerName());
-      long currentTime = TimeUnit.MILLISECONDS.toSeconds(Time.monotonicNow());
-      if (currentTime > commandQueueTime + (MULTIPLIER * reportInterval)) {
-        commandIssued.remove(info.getContainerName());
-        mapCount.decrementAndGet();
-      } else {
-        // Ignore this request, since we just issued a close command. We
-        // should wait instead of sending a command to datanode again.
-        return;
-      }
-    }
-
-    // if we reached here, it means that we have not issued a command to the
-    // data node in last (3 times report interval). We are presuming that is
-    // enough time to close the container. Let us go ahead and queue a close
-    // to all the datanodes that participate in the container.
-    //
-    // Three important things to note here:
-    //
-    // 1. It is ok to send this command multiple times to a datanode. Close
-    // container is an idempotent command, if the container is already closed
-    // then we have no issues.
-    //
-    // 2. The container close command is issued to all datanodes. But
-    // depending on the pipeline type, some of the datanodes might ignore it.
-    //
-    // 3. SCM will see that datanode is closed from container reports, but it
-    // is possible that datanodes might get close commands since
-    // this queue can be emptied by a datanode after a close report is send
-    // to SCM. In that case also, data node will ignore this command.
-
-    OzoneProtos.Pipeline pipeline = info.getPipeline();
-    for (HdfsProtos.DatanodeIDProto datanodeID :
-        pipeline.getPipelineChannel().getMembersList()) {
-      nodeManager.addDatanodeCommand(DatanodeID.getFromProtoBuf(datanodeID),
-          new CloseContainerCommand(info.getContainerName()));
-    }
-    if (!commandIssued.containsKey(info.getContainerName())) {
-      commandIssued.put(info.getContainerName(),
-          TimeUnit.MILLISECONDS.toSeconds(Time.monotonicNow()));
-      mapCount.incrementAndGet();
-    }
-    // run the hash map cleaner thread if needed, non-blocking call.
-    runCleanerThreadIfNeeded();
-  }
-
-  private void runCleanerThreadIfNeeded() {
-    // Let us check if we should run a cleaner thread, not using map.size
-    // since it runs a loop in the case of the concurrentMap.
-    if (mapCount.get() > CLEANUP_WATER_MARK &&
-        isRunning.compareAndSet(false, true)) {
-      Runnable entryCleaner = () -> {
-        LOG.debug("Starting close container Hash map cleaner.");
-        try {
-          for (Map.Entry<String, Long> entry : commandIssued.entrySet()) {
-            long commandQueueTime = entry.getValue();
-            if (commandQueueTime + (MULTIPLIER * reportInterval) >
-                TimeUnit.MILLISECONDS.toSeconds(Time.monotonicNow())) {
-
-              // It is possible for this remove to fail due to race conditions.
-              // No big deal we will cleanup next time.
-              commandIssued.remove(entry.getKey());
-              mapCount.decrementAndGet();
-            }
-          }
-          isRunning.compareAndSet(true, false);
-          LOG.debug("Finished running, close container Hash map cleaner.");
-        } catch (Exception ex) {
-          LOG.error("Unable to finish cleaning the closed containers map.", ex);
-        }
-      };
-
-      // Launch the cleaner thread when we need instead of having a daemon
-      // thread that is sleeping all the time. We need to set the Daemon to
-      // true to avoid blocking clean exits.
-      Thread cleanerThread = new ThreadFactoryBuilder()
-          .setDaemon(true)
-          .setNameFormat("Closed Container Cleaner Thread - %d")
-          .build().newThread(entryCleaner);
-      threadRunCount.incrementAndGet();
-      cleanerThread.start();
-    }
-  }
-
-  @VisibleForTesting
-  public int getThreadRunCount() {
-    return threadRunCount.get();
-  }
-
-  @VisibleForTesting
-  public int getCloseCount() {
-    return mapCount.get();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/closer/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/closer/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/closer/package-info.java
deleted file mode 100644
index 2d0f257..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/closer/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-/**
- * This package has class that close a container. That is move a container from
- * open state to close state.
- */
-package org.apache.hadoop.ozone.scm.container.closer;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/package-info.java
deleted file mode 100644
index 4ac490c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container;
-/**
- * This package contains routines to manage the container location and
- * mapping inside SCM
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/ContainerPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/ContainerPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/ContainerPlacementPolicy.java
deleted file mode 100644
index 0cf1fde..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/ContainerPlacementPolicy.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement.algorithms;
-
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * A ContainerPlacementPolicy support choosing datanodes to build replication
- * pipeline with specified constraints.
- */
-public interface ContainerPlacementPolicy {
-
-  /**
-   * Given the replication factor and size required, return set of datanodes
-   * that satisfy the nodes and size requirement.
-   * @param nodesRequired - number of datanodes required.
-   * @param sizeRequired - size required for the container or block.
-   * @return list of datanodes chosen.
-   * @throws IOException
-   */
-  List<DatanodeID> chooseDatanodes(int nodesRequired, long sizeRequired)
-      throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMCommonPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMCommonPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMCommonPolicy.java
deleted file mode 100644
index b5a1a13..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMCommonPolicy.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement.algorithms;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeMetric;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Random;
-import java.util.stream.Collectors;
-
-/**
- * SCM CommonPolicy implements a set of invariants which are common
- * for all container placement policies, acts as the repository of helper
- * functions which are common to placement policies.
- */
-public abstract class SCMCommonPolicy implements ContainerPlacementPolicy {
-  @VisibleForTesting
-  static final Logger LOG =
-      LoggerFactory.getLogger(SCMCommonPolicy.class);
-  private final NodeManager nodeManager;
-  private final Random rand;
-  private final Configuration conf;
-
-  /**
-   * Constructs SCM Common Policy Class.
-   *
-   * @param nodeManager NodeManager
-   * @param conf Configuration class.
-   */
-  public SCMCommonPolicy(NodeManager nodeManager, Configuration conf) {
-    this.nodeManager = nodeManager;
-    this.rand = new Random();
-    this.conf = conf;
-  }
-
-  /**
-   * Return node manager.
-   *
-   * @return node manager
-   */
-  public NodeManager getNodeManager() {
-    return nodeManager;
-  }
-
-  /**
-   * Returns the Random Object.
-   *
-   * @return rand
-   */
-  public Random getRand() {
-    return rand;
-  }
-
-  /**
-   * Get Config.
-   *
-   * @return Configuration
-   */
-  public Configuration getConf() {
-    return conf;
-  }
-
-  /**
-   * Given the replication factor and size required, return set of datanodes
-   * that satisfy the nodes and size requirement.
-   * <p>
-   * Here are some invariants of container placement.
-   * <p>
-   * 1. We place containers only on healthy nodes.
-   * 2. We place containers on nodes with enough space for that container.
-   * 3. if a set of containers are requested, we either meet the required
-   * number of nodes or we fail that request.
-   *
-   * @param nodesRequired - number of datanodes required.
-   * @param sizeRequired - size required for the container or block.
-   * @return list of datanodes chosen.
-   * @throws SCMException SCM exception.
-   */
-
-  public List<DatanodeID> chooseDatanodes(int nodesRequired, final long
-      sizeRequired) throws SCMException {
-    List<DatanodeID> healthyNodes =
-        nodeManager.getNodes(OzoneProtos.NodeState.HEALTHY);
-    String msg;
-    if (healthyNodes.size() == 0) {
-      msg = "No healthy node found to allocate container.";
-      LOG.error(msg);
-      throw new SCMException(msg, SCMException.ResultCodes
-          .FAILED_TO_FIND_HEALTHY_NODES);
-    }
-
-    if (healthyNodes.size() < nodesRequired) {
-      msg = String.format("Not enough healthy nodes to allocate container. %d "
-              + " datanodes required. Found %d",
-          nodesRequired, healthyNodes.size());
-      LOG.error(msg);
-      throw new SCMException(msg,
-          SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
-    }
-    List<DatanodeID> healthyList = healthyNodes.stream().filter(d ->
-        hasEnoughSpace(d, sizeRequired)).collect(Collectors.toList());
-
-    if (healthyList.size() < nodesRequired) {
-      msg = String.format("Unable to find enough nodes that meet the space " +
-              "requirement of %d bytes in healthy node set." +
-              " Nodes required: %d Found: %d",
-          sizeRequired, nodesRequired, healthyList.size());
-      LOG.error(msg);
-      throw new SCMException(msg,
-          SCMException.ResultCodes.FAILED_TO_FIND_NODES_WITH_SPACE);
-    }
-
-    return healthyList;
-  }
-
-  /**
-   * Returns true if this node has enough space to meet our requirement.
-   *
-   * @param datanodeID DatanodeID
-   * @return true if we have enough space.
-   */
-  private boolean hasEnoughSpace(DatanodeID datanodeID, long sizeRequired) {
-    SCMNodeMetric nodeMetric = nodeManager.getNodeStat(datanodeID);
-    return (nodeMetric != null) && nodeMetric.get().getRemaining()
-        .hasResources(sizeRequired);
-  }
-
-  /**
-   * This function invokes the derived classes chooseNode Function to build a
-   * list of nodes. Then it verifies that invoked policy was able to return
-   * expected number of nodes.
-   *
-   * @param nodesRequired - Nodes Required
-   * @param healthyNodes - List of Nodes in the result set.
-   * @return List of Datanodes that can be used for placement.
-   * @throws SCMException
-   */
-  public List<DatanodeID> getResultSet(int nodesRequired, List<DatanodeID>
-      healthyNodes) throws SCMException {
-    List<DatanodeID> results = new LinkedList<>();
-    for (int x = 0; x < nodesRequired; x++) {
-      // invoke the choose function defined in the derived classes.
-      DatanodeID nodeId = chooseNode(healthyNodes);
-      if (nodeId != null) {
-        results.add(nodeId);
-      }
-    }
-
-    if (results.size() < nodesRequired) {
-      LOG.error("Unable to find the required number of healthy nodes that " +
-              "meet the criteria. Required nodes: {}, Found nodes: {}",
-          nodesRequired, results.size());
-      throw new SCMException("Unable to find required number of nodes.",
-          SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
-    }
-    return results;
-  }
-
-  /**
-   * Choose a datanode according to the policy, this function is implemented
-   * by the actual policy class. For example, PlacementCapacity or
-   * PlacementRandom.
-   *
-   * @param healthyNodes - Set of healthy nodes we can choose from.
-   * @return DatanodeID
-   */
-  public abstract DatanodeID chooseNode(List<DatanodeID> healthyNodes);
-
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
deleted file mode 100644
index de9bb74..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement.algorithms;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeMetric;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- * Container placement policy that randomly choose datanodes with remaining
- * space to satisfy the size constraints.
- * <p>
- * The Algorithm is as follows, Pick 2 random nodes from a given pool of nodes
- * and then pick the node which lower utilization. This leads to a higher
- * probability of nodes with lower utilization to be picked.
- * <p>
- * For those wondering why we choose two nodes randomly and choose the node
- * with lower utilization. There are links to this original papers in
- * HDFS-11564.
- * <p>
- * A brief summary -- We treat the nodes from a scale of lowest utilized to
- * highest utilized, there are (s * ( s + 1)) / 2 possibilities to build
- * distinct pairs of nodes.  There are s - k pairs of nodes in which the rank
- * k node is less than the couple. So probability of a picking a node is
- * (2 * (s -k)) / (s * (s - 1)).
- * <p>
- * In English, There is a much higher probability of picking less utilized nodes
- * as compared to nodes with higher utilization since we pick 2 nodes and
- * then pick the node with lower utilization.
- * <p>
- * This avoids the issue of users adding new nodes into the cluster and HDFS
- * sending all traffic to those nodes if we only use a capacity based
- * allocation scheme. Unless those nodes are part of the set of the first 2
- * nodes then newer nodes will not be in the running to get the container.
- * <p>
- * This leads to an I/O pattern where the lower utilized nodes are favoured
- * more than higher utilized nodes, but part of the I/O will still go to the
- * older higher utilized nodes.
- * <p>
- * With this algorithm in place, our hope is that balancer tool needs to do
- * little or no work and the cluster will achieve a balanced distribution
- * over time.
- */
-public final class SCMContainerPlacementCapacity extends SCMCommonPolicy {
-  @VisibleForTesting
-  static final Logger LOG =
-      LoggerFactory.getLogger(SCMContainerPlacementCapacity.class);
-
-  /**
-   * Constructs a Container Placement with considering only capacity.
-   * That is this policy tries to place containers based on node weight.
-   *
-   * @param nodeManager Node Manager
-   * @param conf Configuration
-   */
-  public SCMContainerPlacementCapacity(final NodeManager nodeManager,
-      final Configuration conf) {
-    super(nodeManager, conf);
-  }
-
-  /**
-   * Called by SCM to choose datanodes.
-   *
-   * @param nodesRequired - number of datanodes required.
-   * @param sizeRequired - size required for the container or block.
-   * @return List of datanodes.
-   * @throws SCMException  SCMException
-   */
-  @Override
-  public List<DatanodeID> chooseDatanodes(final int nodesRequired,
-      final long sizeRequired) throws SCMException {
-    List<DatanodeID> healthyNodes =
-        super.chooseDatanodes(nodesRequired, sizeRequired);
-    if (healthyNodes.size() == nodesRequired) {
-      return healthyNodes;
-    }
-    return getResultSet(nodesRequired, healthyNodes);
-  }
-
-  /**
-   * Find a node from the healthy list and return it after removing it from the
-   * list that we are operating on.
-   *
-   * @param healthyNodes - List of healthy nodes that meet the size
-   * requirement.
-   * @return DatanodeID that is chosen.
-   */
-  @Override
-  public DatanodeID chooseNode(List<DatanodeID> healthyNodes) {
-    int firstNodeNdx = getRand().nextInt(healthyNodes.size());
-    int secondNodeNdx = getRand().nextInt(healthyNodes.size());
-
-    DatanodeID chosenID;
-    // There is a possibility that both numbers will be same.
-    // if that is so, we just return the node.
-    if (firstNodeNdx == secondNodeNdx) {
-      chosenID = healthyNodes.get(firstNodeNdx);
-    } else {
-      DatanodeID firstNodeID = healthyNodes.get(firstNodeNdx);
-      DatanodeID secondNodeID = healthyNodes.get(secondNodeNdx);
-      SCMNodeMetric firstNodeMetric =
-          getNodeManager().getNodeStat(firstNodeID);
-      SCMNodeMetric secondNodeMetric =
-          getNodeManager().getNodeStat(secondNodeID);
-      chosenID = firstNodeMetric.isGreater(secondNodeMetric.get())
-          ? firstNodeID : secondNodeID;
-    }
-    healthyNodes.remove(chosenID);
-    return chosenID;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMContainerPlacementRandom.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
deleted file mode 100644
index b145b14..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement.algorithms;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- * Container placement policy that randomly chooses healthy datanodes.
- * This is very similar to current HDFS placement. That is we
- * just randomly place containers without any considerations of utilization.
- * <p>
- * That means we rely on balancer to achieve even distribution of data.
- * Balancer will need to support containers as a feature before this class
- * can be practically used.
- */
-public final class SCMContainerPlacementRandom extends SCMCommonPolicy
-    implements ContainerPlacementPolicy {
-  @VisibleForTesting
-  static final Logger LOG =
-      LoggerFactory.getLogger(SCMContainerPlacementRandom.class);
-
-  /**
-   * Construct a random Block Placement policy.
-   *
-   * @param nodeManager nodeManager
-   * @param conf Config
-   */
-  public SCMContainerPlacementRandom(final NodeManager nodeManager,
-      final Configuration conf) {
-    super(nodeManager, conf);
-  }
-
-  /**
-   * Choose datanodes called by the SCM to choose the datanode.
-   *
-   * @param nodesRequired - number of datanodes required.
-   * @param sizeRequired - size required for the container or block.
-   * @return List of Datanodes.
-   * @throws SCMException  SCMException
-   */
-  @Override
-  public List<DatanodeID> chooseDatanodes(final int nodesRequired,
-      final long sizeRequired) throws SCMException {
-    List<DatanodeID> healthyNodes =
-        super.chooseDatanodes(nodesRequired, sizeRequired);
-
-    if (healthyNodes.size() == nodesRequired) {
-      return healthyNodes;
-    }
-    return getResultSet(nodesRequired, healthyNodes);
-  }
-
-  /**
-   * Just chose a node randomly and remove it from the set of nodes we can
-   * chose from.
-   *
-   * @param healthyNodes - all healthy datanodes.
-   * @return one randomly chosen datanode that from two randomly chosen datanode
-   */
-  public DatanodeID chooseNode(final List<DatanodeID> healthyNodes) {
-    DatanodeID selectedNode =
-        healthyNodes.get(getRand().nextInt(healthyNodes.size()));
-    healthyNodes.remove(selectedNode);
-    return selectedNode;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/package-info.java
deleted file mode 100644
index d6280df..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container.placement.algorithms;
-// Various placement algorithms.
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/ContainerStat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/ContainerStat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/ContainerStat.java
deleted file mode 100644
index 810b8fd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/ContainerStat.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.container.placement.metrics;
-
-import java.io.IOException;
-
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Preconditions;
-
-/**
- * This class represents the SCM container stat.
- */
-public class ContainerStat {
-  /**
-   * The maximum container size.
-   */
-  @JsonProperty("Size")
-  private LongMetric size;
-
-  /**
-   * The number of bytes used by the container.
-   */
-  @JsonProperty("Used")
-  private LongMetric used;
-
-  /**
-   * The number of keys in the container.
-   */
-  @JsonProperty("KeyCount")
-  private LongMetric keyCount;
-
-  /**
-   * The number of bytes read from the container.
-   */
-  @JsonProperty("ReadBytes")
-  private LongMetric readBytes;
-
-  /**
-   * The number of bytes write into the container.
-   */
-  @JsonProperty("WriteBytes")
-  private LongMetric writeBytes;
-
-  /**
-   * The number of times the container is read.
-   */
-  @JsonProperty("ReadCount")
-  private LongMetric readCount;
-
-  /**
-   * The number of times the container is written into.
-   */
-  @JsonProperty("WriteCount")
-  private LongMetric writeCount;
-
-  public ContainerStat() {
-    this(0L, 0L, 0L, 0L, 0L, 0L, 0L);
-  }
-
-  public ContainerStat(long size, long used, long keyCount, long readBytes,
-      long writeBytes, long readCount, long writeCount) {
-    Preconditions.checkArgument(size >= 0,
-        "Container size cannot be " + "negative.");
-    Preconditions.checkArgument(used >= 0,
-        "Used space cannot be " + "negative.");
-    Preconditions.checkArgument(keyCount >= 0,
-        "Key count cannot be " + "negative");
-    Preconditions.checkArgument(readBytes >= 0,
-        "Read bytes read cannot be " + "negative.");
-    Preconditions.checkArgument(readBytes >= 0,
-        "Write bytes cannot be " + "negative.");
-    Preconditions.checkArgument(readCount >= 0,
-        "Read count cannot be " + "negative.");
-    Preconditions.checkArgument(writeCount >= 0,
-        "Write count cannot be " + "negative");
-
-    this.size = new LongMetric(size);
-    this.used = new LongMetric(used);
-    this.keyCount = new LongMetric(keyCount);
-    this.readBytes = new LongMetric(readBytes);
-    this.writeBytes = new LongMetric(writeBytes);
-    this.readCount = new LongMetric(readCount);
-    this.writeCount = new LongMetric(writeCount);
-  }
-
-  public LongMetric getSize() {
-    return size;
-  }
-
-  public LongMetric getUsed() {
-    return used;
-  }
-
-  public LongMetric getKeyCount() {
-    return keyCount;
-  }
-
-  public LongMetric getReadBytes() {
-    return readBytes;
-  }
-
-  public LongMetric getWriteBytes() {
-    return writeBytes;
-  }
-
-  public LongMetric getReadCount() {
-    return readCount;
-  }
-
-  public LongMetric getWriteCount() {
-    return writeCount;
-  }
-
-  public void add(ContainerStat stat) {
-    if (stat == null) {
-      return;
-    }
-
-    this.size.add(stat.getSize().get());
-    this.used.add(stat.getUsed().get());
-    this.keyCount.add(stat.getKeyCount().get());
-    this.readBytes.add(stat.getReadBytes().get());
-    this.writeBytes.add(stat.getWriteBytes().get());
-    this.readCount.add(stat.getReadCount().get());
-    this.writeCount.add(stat.getWriteCount().get());
-  }
-
-  public void subtract(ContainerStat stat) {
-    if (stat == null) {
-      return;
-    }
-
-    this.size.subtract(stat.getSize().get());
-    this.used.subtract(stat.getUsed().get());
-    this.keyCount.subtract(stat.getKeyCount().get());
-    this.readBytes.subtract(stat.getReadBytes().get());
-    this.writeBytes.subtract(stat.getWriteBytes().get());
-    this.readCount.subtract(stat.getReadCount().get());
-    this.writeCount.subtract(stat.getWriteCount().get());
-  }
-
-  public String toJsonString() {
-    try {
-      return JsonUtils.toJsonString(this);
-    } catch (IOException ignored) {
-      return null;
-    }
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/DatanodeMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/DatanodeMetric.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/DatanodeMetric.java
deleted file mode 100644
index cc829c2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/DatanodeMetric.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement.metrics;
-
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-
-/**
- * DatanodeMetric acts as the basis for all the metric that is used in
- * comparing 2 datanodes.
- */
-public interface DatanodeMetric<T, S> extends Comparable<T> {
-
-  /**
-   * Some syntactic sugar over Comparable interface. This makes code easier to
-   * read.
-   *
-   * @param o - Other Object
-   * @return - True if *this* object is greater than argument.
-   */
-  boolean isGreater(T o);
-
-  /**
-   * Inverse of isGreater.
-   *
-   * @param o - other object.
-   * @return True if *this* object is Lesser than argument.
-   */
-  boolean isLess(T o);
-
-  /**
-   * Returns true if the object has same values. Because of issues with
-   * equals, and loss of type information this interface supports isEqual.
-   *
-   * @param o object to compare.
-   * @return True, if the values match.
-   */
-  boolean isEqual(T o);
-
-  /**
-   * A resourceCheck, defined by resourceNeeded.
-   * For example, S could be bytes required
-   * and DatanodeMetric can reply by saying it can be met or not.
-   *
-   * @param resourceNeeded -  ResourceNeeded in its own metric.
-   * @return boolean, True if this resource requirement can be met.
-   */
-  boolean hasResources(S resourceNeeded) throws SCMException;
-
-  /**
-   * Returns the metric.
-   *
-   * @return T, the object that represents this metric.
-   */
-  T get();
-
-  /**
-   * Sets the value of this metric.
-   *
-   * @param value - value of the metric.
-   */
-  void set(T value);
-
-  /**
-   * Adds a value of to the base.
-   * @param value - value
-   */
-  void add(T value);
-
-  /**
-   * subtract a value.
-   * @param value value
-   */
-  void subtract(T value);
-
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/LongMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/LongMetric.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/LongMetric.java
deleted file mode 100644
index dbcd9f4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/LongMetric.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container.placement.metrics;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-
-/**
- * An helper class for all metrics based on Longs.
- */
-@JsonAutoDetect(fieldVisibility = Visibility.ANY)
-public class LongMetric implements DatanodeMetric<Long, Long> {
-  private Long value;
-
-  /**
-   * Constructs a long Metric.
-   *
-   * @param value Value for this metric.
-   */
-  public LongMetric(Long value) {
-    this.value = value;
-  }
-
-  /**
-   * Some syntactic sugar over Comparable interface. This makes code easier to
-   * read.
-   *
-   * @param o - Other Object
-   * @return - True if *this* object is greater than argument.
-   */
-  @Override
-  public boolean isGreater(Long o) {
-    return compareTo(o) > 0;
-  }
-
-  /**
-   * Inverse of isGreater.
-   *
-   * @param o - other object.
-   * @return True if *this* object is Lesser than argument.
-   */
-  @Override
-  public boolean isLess(Long o) {
-    return compareTo(o) < 0;
-  }
-
-  /**
-   * Returns true if the object has same values. Because of issues with
-   * equals, and loss of type information this interface supports isEqual.
-   *
-   * @param o object to compare.
-   * @return True, if the values match.
-   */
-  @Override
-  public boolean isEqual(Long o) {
-    return compareTo(o) == 0;
-  }
-
-  /**
-   * A resourceCheck, defined by resourceNeeded.
-   * For example, S could be bytes required
-   * and DatanodeMetric can reply by saying it can be met or not.
-   *
-   * @param resourceNeeded -  ResourceNeeded in its own metric.
-   * @return boolean, True if this resource requirement can be met.
-   */
-  @Override
-  public boolean hasResources(Long resourceNeeded) {
-    return isGreater(resourceNeeded);
-  }
-
-  /**
-   * Returns the metric.
-   *
-   * @return T, the object that represents this metric.
-   */
-  @Override
-  public Long get() {
-    return this.value;
-  }
-
-  /**
-   * Sets the value of this metric.
-   *
-   * @param setValue - value of the metric.
-   */
-  @Override
-  public void set(Long setValue) {
-    this.value = setValue;
-
-  }
-
-  /**
-   * Adds a value of to the base.
-   *
-   * @param addValue - value
-   */
-  @Override
-  public void add(Long addValue) {
-    this.value += addValue;
-  }
-
-  /**
-   * subtract a value.
-   *
-   * @param subValue value
-   */
-  @Override
-  public void subtract(Long subValue) {
-    this.value -= subValue;
-  }
-
-  /**
-   * Compares this object with the specified object for order.  Returns a
-   * negative integer, zero, or a positive integer as this object is less
-   * than, equal to, or greater than the specified object.
-   *
-   * @param o the object to be compared.
-   * @return a negative integer, zero, or a positive integer as this object is
-   * less than, equal to, or greater than the specified object.
-   * @throws NullPointerException if the specified object is null
-   * @throws ClassCastException   if the specified object's type prevents it
-   *                              from being compared to this object.
-   */
-  @Override
-  public int compareTo(Long o) {
-    return Long.compare(this.value, o);
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-
-    LongMetric that = (LongMetric) o;
-
-    return value != null ? value.equals(that.value) : that.value == null;
-  }
-
-  @Override
-  public int hashCode() {
-    return value != null ? value.hashCode() : 0;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/NodeStat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/NodeStat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/NodeStat.java
deleted file mode 100644
index bda61f8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/NodeStat.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement.metrics;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * Interface that defines Node Stats.
- */
-interface NodeStat {
-  /**
-   * Get capacity of the node.
-   * @return capacity of the node.
-   */
-  LongMetric getCapacity();
-
-  /**
-   * Get the used space of the node.
-   * @return the used space of the node.
-   */
-  LongMetric getScmUsed();
-
-  /**
-   * Get the remaining space of the node.
-   * @return the remaining space of the node.
-   */
-  LongMetric getRemaining();
-
-  /**
-   * Set the total/used/remaining space.
-   * @param capacity - total space.
-   * @param used - used space.
-   * @param remain - remaining space.
-   */
-  @VisibleForTesting
-  void set(long capacity, long used, long remain);
-
-  /**
-   * Adding of the stat.
-   * @param stat - stat to be added.
-   * @return updated node stat.
-   */
-  NodeStat add(NodeStat stat);
-
-  /**
-   * Subtract of the stat.
-   * @param stat - stat to be subtracted.
-   * @return updated nodestat.
-   */
-  NodeStat subtract(NodeStat stat);
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMMetrics.java
deleted file mode 100644
index bdfec2d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMMetrics.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.container.placement.metrics;
-
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.annotation.Metrics;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
-
-/**
- * This class is for maintaining StorageContainerManager statistics.
- */
-@Metrics(about="Storage Container Manager Metrics", context="dfs")
-public class SCMMetrics {
-  public static final String SOURCE_NAME =
-      SCMMetrics.class.getSimpleName();
-
-  /**
-   * Container stat metrics, the meaning of following metrics
-   * can be found in {@link ContainerStat}.
-   */
-  @Metric private MutableGaugeLong lastContainerReportSize;
-  @Metric private MutableGaugeLong lastContainerReportUsed;
-  @Metric private MutableGaugeLong lastContainerReportKeyCount;
-  @Metric private MutableGaugeLong lastContainerReportReadBytes;
-  @Metric private MutableGaugeLong lastContainerReportWriteBytes;
-  @Metric private MutableGaugeLong lastContainerReportReadCount;
-  @Metric private MutableGaugeLong lastContainerReportWriteCount;
-
-  @Metric private MutableCounterLong containerReportSize;
-  @Metric private MutableCounterLong containerReportUsed;
-  @Metric private MutableCounterLong containerReportKeyCount;
-  @Metric private MutableCounterLong containerReportReadBytes;
-  @Metric private MutableCounterLong containerReportWriteBytes;
-  @Metric private MutableCounterLong containerReportReadCount;
-  @Metric private MutableCounterLong containerReportWriteCount;
-
-  public SCMMetrics() {
-  }
-
-  public static SCMMetrics create() {
-    MetricsSystem ms = DefaultMetricsSystem.instance();
-    return ms.register(SOURCE_NAME, "Storage Container Manager Metrics",
-        new SCMMetrics());
-  }
-
-  public void setLastContainerReportSize(long size) {
-    this.lastContainerReportSize.set(size);
-  }
-
-  public void setLastContainerReportUsed(long used) {
-    this.lastContainerReportUsed.set(used);
-  }
-
-  public void setLastContainerReportKeyCount(long keyCount) {
-    this.lastContainerReportKeyCount.set(keyCount);
-  }
-
-  public void setLastContainerReportReadBytes(long readBytes) {
-    this.lastContainerReportReadBytes.set(readBytes);
-  }
-
-  public void setLastContainerReportWriteBytes(long writeBytes) {
-    this.lastContainerReportWriteBytes.set(writeBytes);
-  }
-
-  public void setLastContainerReportReadCount(long readCount) {
-    this.lastContainerReportReadCount.set(readCount);
-  }
-
-  public void setLastContainerReportWriteCount(long writeCount) {
-    this.lastContainerReportWriteCount.set(writeCount);
-  }
-
-  public void incrContainerReportSize(long size) {
-    this.containerReportSize.incr(size);
-  }
-
-  public void incrContainerReportUsed(long used) {
-    this.containerReportUsed.incr(used);
-  }
-
-  public void incrContainerReportKeyCount(long keyCount) {
-    this.containerReportKeyCount.incr(keyCount);
-  }
-
-  public void incrContainerReportReadBytes(long readBytes) {
-    this.containerReportReadBytes.incr(readBytes);
-  }
-
-  public void incrContainerReportWriteBytes(long writeBytes) {
-    this.containerReportWriteBytes.incr(writeBytes);
-  }
-
-  public void incrContainerReportReadCount(long readCount) {
-    this.containerReportReadCount.incr(readCount);
-  }
-
-  public void incrContainerReportWriteCount(long writeCount) {
-    this.containerReportWriteCount.incr(writeCount);
-  }
-
-  public void setLastContainerStat(ContainerStat newStat) {
-    this.lastContainerReportSize.set(newStat.getSize().get());
-    this.lastContainerReportUsed.set(newStat.getUsed().get());
-    this.lastContainerReportKeyCount.set(newStat.getKeyCount().get());
-    this.lastContainerReportReadBytes.set(newStat.getReadBytes().get());
-    this.lastContainerReportWriteBytes.set(newStat.getWriteBytes().get());
-    this.lastContainerReportReadCount.set(newStat.getReadCount().get());
-    this.lastContainerReportWriteCount.set(newStat.getWriteCount().get());
-  }
-
-  public void incrContainerStat(ContainerStat deltaStat) {
-    this.containerReportSize.incr(deltaStat.getSize().get());
-    this.containerReportUsed.incr(deltaStat.getUsed().get());
-    this.containerReportKeyCount.incr(deltaStat.getKeyCount().get());
-    this.containerReportReadBytes.incr(deltaStat.getReadBytes().get());
-    this.containerReportWriteBytes.incr(deltaStat.getWriteBytes().get());
-    this.containerReportReadCount.incr(deltaStat.getReadCount().get());
-    this.containerReportWriteCount.incr(deltaStat.getWriteCount().get());
-  }
-
-  public void decrContainerStat(ContainerStat deltaStat) {
-    this.containerReportSize.incr(-1 * deltaStat.getSize().get());
-    this.containerReportUsed.incr(-1 * deltaStat.getUsed().get());
-    this.containerReportKeyCount.incr(-1 * deltaStat.getKeyCount().get());
-    this.containerReportReadBytes.incr(-1 * deltaStat.getReadBytes().get());
-    this.containerReportWriteBytes.incr(-1 * deltaStat.getWriteBytes().get());
-    this.containerReportReadCount.incr(-1 * deltaStat.getReadCount().get());
-    this.containerReportWriteCount.incr(-1 * deltaStat.getWriteCount().get());
-  }
-
-  public void unRegister() {
-    MetricsSystem ms = DefaultMetricsSystem.instance();
-    ms.unregisterSource(SOURCE_NAME);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMNodeMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMNodeMetric.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMNodeMetric.java
deleted file mode 100644
index ae01361..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMNodeMetric.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement.metrics;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-/**
- * SCM Node Metric that is used in the placement classes.
- */
-public class SCMNodeMetric implements DatanodeMetric<SCMNodeStat, Long> {
-  private SCMNodeStat stat;
-
-  /**
-   * Constructs an SCMNode Metric.
-   *
-   * @param stat - SCMNodeStat.
-   */
-  public SCMNodeMetric(SCMNodeStat stat) {
-    this.stat = stat;
-  }
-
-  /**
-   * Set the capacity, used and remaining space on a datanode.
-   *
-   * @param capacity in bytes
-   * @param used in bytes
-   * @param remaining in bytes
-   */
-  @VisibleForTesting
-  public SCMNodeMetric(long capacity, long used, long remaining) {
-    this.stat = new SCMNodeStat();
-    this.stat.set(capacity, used, remaining);
-  }
-
-  /**
-   *
-   * @param o - Other Object
-   * @return - True if *this* object is greater than argument.
-   */
-  @Override
-  public boolean isGreater(SCMNodeStat o) {
-    Preconditions.checkNotNull(o, "Argument cannot be null");
-
-    // if zero, replace with 1 for the division to work.
-    long thisDenominator = (this.stat.getCapacity().get() == 0)
-        ? 1 : this.stat.getCapacity().get();
-    long otherDenominator = (o.getCapacity().get() == 0)
-        ? 1 : o.getCapacity().get();
-
-    float thisNodeWeight =
-        stat.getScmUsed().get() / (float) thisDenominator;
-
-    float oNodeWeight =
-        o.getScmUsed().get() / (float) otherDenominator;
-
-    if (Math.abs(thisNodeWeight - oNodeWeight) > 0.000001) {
-      return thisNodeWeight > oNodeWeight;
-    }
-    // if these nodes are have similar weight then return the node with more
-    // free space as the greater node.
-    return stat.getRemaining().isGreater(o.getRemaining().get());
-  }
-
-  /**
-   * Inverse of isGreater.
-   *
-   * @param o - other object.
-   * @return True if *this* object is Lesser than argument.
-   */
-  @Override
-  public boolean isLess(SCMNodeStat o) {
-    Preconditions.checkNotNull(o, "Argument cannot be null");
-
-    // if zero, replace with 1 for the division to work.
-    long thisDenominator = (this.stat.getCapacity().get() == 0)
-        ? 1 : this.stat.getCapacity().get();
-    long otherDenominator = (o.getCapacity().get() == 0)
-        ? 1 : o.getCapacity().get();
-
-    float thisNodeWeight =
-        stat.getScmUsed().get() / (float) thisDenominator;
-
-    float oNodeWeight =
-        o.getScmUsed().get() / (float) otherDenominator;
-
-    if (Math.abs(thisNodeWeight - oNodeWeight) > 0.000001) {
-      return thisNodeWeight < oNodeWeight;
-    }
-
-    // if these nodes are have similar weight then return the node with less
-    // free space as the lesser node.
-    return stat.getRemaining().isLess(o.getRemaining().get());
-  }
-
-  /**
-   * Returns true if the object has same values. Because of issues with
-   * equals, and loss of type information this interface supports isEqual.
-   *
-   * @param o object to compare.
-   * @return True, if the values match.
-   * TODO : Consider if it makes sense to add remaining to this equation.
-   */
-  @Override
-  public boolean isEqual(SCMNodeStat o) {
-    float thisNodeWeight = stat.getScmUsed().get() / (float)
-        stat.getCapacity().get();
-    float oNodeWeight = o.getScmUsed().get() / (float) o.getCapacity().get();
-    return Math.abs(thisNodeWeight - oNodeWeight) < 0.000001;
-  }
-
-  /**
-   * A resourceCheck, defined by resourceNeeded.
-   * For example, S could be bytes required
-   * and DatanodeMetric can reply by saying it can be met or not.
-   *
-   * @param resourceNeeded -  ResourceNeeded in its own metric.
-   * @return boolean, True if this resource requirement can be met.
-   */
-  @Override
-  public boolean hasResources(Long resourceNeeded) {
-    return false;
-  }
-
-  /**
-   * Returns the metric.
-   *
-   * @return T, the object that represents this metric.
-   */
-  @Override
-  public SCMNodeStat get() {
-    return stat;
-  }
-
-  /**
-   * Sets the value of this metric.
-   *
-   * @param value - value of the metric.
-   */
-  @Override
-  public void set(SCMNodeStat value) {
-    stat.set(value.getCapacity().get(), value.getScmUsed().get(),
-        value.getRemaining().get());
-  }
-
-  /**
-   * Adds a value of to the base.
-   *
-   * @param value - value
-   */
-  @Override
-  public void add(SCMNodeStat value) {
-    stat.add(value);
-  }
-
-  /**
-   * subtract a value.
-   *
-   * @param value value
-   */
-  @Override
-  public void subtract(SCMNodeStat value) {
-    stat.subtract(value);
-  }
-
-  /**
-   * Compares this object with the specified object for order.  Returns a
-   * negative integer, zero, or a positive integer as this object is less
-   * than, equal to, or greater than the specified object.
-   *
-   * @param o the object to be compared.
-   * @return a negative integer, zero, or a positive integer as this object is
-   * less than, equal to, or greater than the specified object.
-   * @throws NullPointerException if the specified object is null
-   * @throws ClassCastException   if the specified object's type prevents it
-   *                              from being compared to this object.
-   */
-  @Override
-  public int compareTo(SCMNodeStat o) {
-    if (isEqual(o)) {
-      return 0;
-    }
-    if (isGreater(o)) {
-      return 1;
-    } else {
-      return -1;
-    }
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-
-    SCMNodeMetric that = (SCMNodeMetric) o;
-
-    return stat != null ? stat.equals(that.stat) : that.stat == null;
-  }
-
-  @Override
-  public int hashCode() {
-    return stat != null ? stat.hashCode() : 0;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMNodeStat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMNodeStat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMNodeStat.java
deleted file mode 100644
index 64f76a6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMNodeStat.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement.metrics;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-/**
- * This class represents the SCM node stat.
- */
-public class SCMNodeStat implements NodeStat {
-  private LongMetric capacity;
-  private LongMetric scmUsed;
-  private LongMetric remaining;
-
-  public SCMNodeStat() {
-    this(0L, 0L, 0L);
-  }
-
-  public SCMNodeStat(SCMNodeStat other) {
-    this(other.capacity.get(), other.scmUsed.get(), other.remaining.get());
-  }
-
-  public SCMNodeStat(long capacity, long used, long remaining) {
-    Preconditions.checkArgument(capacity >= 0, "Capacity cannot be " +
-        "negative.");
-    Preconditions.checkArgument(used >= 0, "used space cannot be " +
-        "negative.");
-    Preconditions.checkArgument(remaining >= 0, "remaining cannot be " +
-        "negative");
-    this.capacity = new LongMetric(capacity);
-    this.scmUsed = new LongMetric(used);
-    this.remaining = new LongMetric(remaining);
-  }
-
-  /**
-   * @return the total configured capacity of the node.
-   */
-  public LongMetric getCapacity() {
-    return capacity;
-  }
-
-  /**
-   * @return the total SCM used space on the node.
-   */
-  public LongMetric getScmUsed() {
-    return scmUsed;
-  }
-
-  /**
-   * @return the total remaining space available on the node.
-   */
-  public LongMetric getRemaining() {
-    return remaining;
-  }
-
-  /**
-   * Set the capacity, used and remaining space on a datanode.
-   *
-   * @param newCapacity in bytes
-   * @param newUsed in bytes
-   * @param newRemaining in bytes
-   */
-  @VisibleForTesting
-  public void set(long newCapacity, long newUsed, long newRemaining) {
-    Preconditions.checkNotNull(newCapacity, "Capacity cannot be null");
-    Preconditions.checkNotNull(newUsed, "used cannot be null");
-    Preconditions.checkNotNull(newRemaining, "remaining cannot be null");
-
-    Preconditions.checkArgument(newCapacity >= 0, "Capacity cannot be " +
-        "negative.");
-    Preconditions.checkArgument(newUsed >= 0, "used space cannot be " +
-        "negative.");
-    Preconditions.checkArgument(newRemaining >= 0, "remaining cannot be " +
-        "negative");
-
-    this.capacity = new LongMetric(newCapacity);
-    this.scmUsed = new LongMetric(newUsed);
-    this.remaining = new LongMetric(newRemaining);
-  }
-
-  /**
-   * Adds a new nodestat to existing values of the node.
-   *
-   * @param stat Nodestat.
-   * @return SCMNodeStat
-   */
-  public SCMNodeStat add(NodeStat stat) {
-    this.capacity.set(this.getCapacity().get() + stat.getCapacity().get());
-    this.scmUsed.set(this.getScmUsed().get() + stat.getScmUsed().get());
-    this.remaining.set(this.getRemaining().get() + stat.getRemaining().get());
-    return this;
-  }
-
-  /**
-   * Subtracts the stat values from the existing NodeStat.
-   *
-   * @param stat SCMNodeStat.
-   * @return Modified SCMNodeStat
-   */
-  public SCMNodeStat subtract(NodeStat stat) {
-    this.capacity.set(this.getCapacity().get() - stat.getCapacity().get());
-    this.scmUsed.set(this.getScmUsed().get() - stat.getScmUsed().get());
-    this.remaining.set(this.getRemaining().get() - stat.getRemaining().get());
-    return this;
-  }
-
-  @Override
-  public boolean equals(Object to) {
-    if (to instanceof SCMNodeStat) {
-      SCMNodeStat tempStat = (SCMNodeStat) to;
-      return capacity.isEqual(tempStat.getCapacity().get()) &&
-          scmUsed.isEqual(tempStat.getScmUsed().get()) &&
-          remaining.isEqual(tempStat.getRemaining().get());
-    }
-    return false;
-  }
-
-  @Override
-  public int hashCode() {
-    return Long.hashCode(capacity.get() ^ scmUsed.get() ^ remaining.get());
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/package-info.java
deleted file mode 100644
index 5135095..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/package-info.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container.placement.metrics;
-
-// Various metrics supported by Datanode and used by SCM in the placement
-// strategy.
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/package-info.java
deleted file mode 100644
index 43676ef..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement;
-// Classes related to container placement.
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/ContainerSupervisor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/ContainerSupervisor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/ContainerSupervisor.java
deleted file mode 100644
index c063b8b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/ContainerSupervisor.java
+++ /dev/null
@@ -1,333 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container.replication;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.scm.node.NodePoolManager;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.PriorityQueue;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import static com.google.common.util.concurrent.Uninterruptibles
-    .sleepUninterruptibly;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_MAX_CONTAINER_REPORT_THREADS;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS;
-import static org.apache.hadoop.scm.ScmConfigKeys
-    .OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT;
-
-/**
- * This class takes a set of container reports that belong to a pool and then
- * computes the replication levels for each container.
- */
-public class ContainerSupervisor implements Closeable {
-  public static final Logger LOG =
-      LoggerFactory.getLogger(ContainerSupervisor.class);
-
-  private final NodePoolManager poolManager;
-  private final HashSet<String> poolNames;
-  private final PriorityQueue<PeriodicPool> poolQueue;
-  private final NodeManager nodeManager;
-  private final long containerProcessingLag;
-  private final AtomicBoolean runnable;
-  private final ExecutorService executorService;
-  private final long maxPoolWait;
-  private long poolProcessCount;
-  private final List<InProgressPool> inProgressPoolList;
-  private final AtomicInteger threadFaultCount;
-  private final int inProgressPoolMaxCount;
-
-  private final ReadWriteLock inProgressPoolListLock;
-
-  /**
-   * Returns the number of times we have processed pools.
-   * @return long
-   */
-  public long getPoolProcessCount() {
-    return poolProcessCount;
-  }
-
-
-  /**
-   * Constructs a class that computes Replication Levels.
-   *
-   * @param conf - OzoneConfiguration
-   * @param nodeManager - Node Manager
-   * @param poolManager - Pool Manager
-   */
-  public ContainerSupervisor(Configuration conf, NodeManager nodeManager,
-                             NodePoolManager poolManager) {
-    Preconditions.checkNotNull(poolManager);
-    Preconditions.checkNotNull(nodeManager);
-    this.containerProcessingLag =
-        conf.getTimeDuration(OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL,
-            OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT,
-            TimeUnit.SECONDS
-        ) * 1000;
-    int maxContainerReportThreads =
-        conf.getInt(OZONE_SCM_MAX_CONTAINER_REPORT_THREADS,
-            OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT
-        );
-    this.maxPoolWait =
-        conf.getTimeDuration(OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT,
-            OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT,
-            TimeUnit.MILLISECONDS);
-    this.inProgressPoolMaxCount = conf.getInt(
-        OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS,
-        OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT);
-    this.poolManager = poolManager;
-    this.nodeManager = nodeManager;
-    this.poolNames = new HashSet<>();
-    this.poolQueue = new PriorityQueue<>();
-    this.runnable = new AtomicBoolean(true);
-    this.threadFaultCount = new AtomicInteger(0);
-    this.executorService = HadoopExecutors.newCachedThreadPool(
-        new ThreadFactoryBuilder().setDaemon(true)
-            .setNameFormat("Container Reports Processing Thread - %d")
-            .build(), maxContainerReportThreads);
-    this.inProgressPoolList = new LinkedList<>();
-    this.inProgressPoolListLock = new ReentrantReadWriteLock();
-
-    initPoolProcessThread();
-  }
-
-  /**
-   * Returns the number of pools that are under process right now.
-   * @return  int - Number of pools that are in process.
-   */
-  public int getInProgressPoolCount() {
-    return inProgressPoolList.size();
-  }
-
-  /**
-   * Exits the background thread.
-   */
-  public void setExit() {
-    this.runnable.set(false);
-  }
-
-  /**
-   * Adds or removes pools from names that we need to process.
-   *
-   * There are two different cases that we need to process.
-   * The case where some pools are being added and some times we have to
-   * handle cases where pools are removed.
-   */
-  private void refreshPools() {
-    List<String> pools = this.poolManager.getNodePools();
-    if (pools != null) {
-
-      HashSet<String> removedPools =
-          computePoolDifference(this.poolNames, new HashSet<>(pools));
-
-      HashSet<String> addedPools =
-          computePoolDifference(new HashSet<>(pools), this.poolNames);
-      // TODO: Support remove pool API in pool manager so that this code
-      // path can be tested. This never happens in the current code base.
-      for (String poolName : removedPools) {
-        for (PeriodicPool periodicPool : poolQueue) {
-          if (periodicPool.getPoolName().compareTo(poolName) == 0) {
-            poolQueue.remove(periodicPool);
-          }
-        }
-      }
-      // Remove the pool names that we have in the list.
-      this.poolNames.removeAll(removedPools);
-
-      for (String poolName : addedPools) {
-        poolQueue.add(new PeriodicPool(poolName));
-      }
-
-      // Add to the pool names we are tracking.
-      poolNames.addAll(addedPools);
-    }
-
-  }
-
-  /**
-   * Handle the case where pools are added.
-   *
-   * @param newPools - New Pools list
-   * @param oldPool - oldPool List.
-   */
-  private HashSet<String> computePoolDifference(HashSet<String> newPools,
-      Set<String> oldPool) {
-    Preconditions.checkNotNull(newPools);
-    Preconditions.checkNotNull(oldPool);
-    HashSet<String> newSet = new HashSet<>(newPools);
-    newSet.removeAll(oldPool);
-    return newSet;
-  }
-
-  private void initPoolProcessThread() {
-
-    /*
-     * Task that runs to check if we need to start a pool processing job.
-     * if so we create a pool reconciliation job and find out of all the
-     * expected containers are on the nodes.
-     */
-    Runnable processPools = () -> {
-      while (runnable.get()) {
-        // Make sure that we don't have any new pools.
-        refreshPools();
-        while (inProgressPoolList.size() < inProgressPoolMaxCount) {
-          PeriodicPool pool = poolQueue.poll();
-          if (pool != null) {
-            if (pool.getLastProcessedTime() + this.containerProcessingLag >
-                Time.monotonicNow()) {
-              LOG.debug("Not within the time window for processing: {}",
-                  pool.getPoolName());
-              // we might over sleep here, not a big deal.
-              sleepUninterruptibly(this.containerProcessingLag,
-                  TimeUnit.MILLISECONDS);
-            }
-            LOG.debug("Adding pool {} to container processing queue",
-                pool.getPoolName());
-            InProgressPool inProgressPool = new InProgressPool(maxPoolWait,
-                pool, this.nodeManager, this.poolManager, this.executorService);
-            inProgressPool.startReconciliation();
-            inProgressPoolListLock.writeLock().lock();
-            try {
-              inProgressPoolList.add(inProgressPool);
-            } finally {
-              inProgressPoolListLock.writeLock().unlock();
-            }
-            poolProcessCount++;
-          } else {
-            break;
-          }
-        }
-        sleepUninterruptibly(this.maxPoolWait, TimeUnit.MILLISECONDS);
-        inProgressPoolListLock.readLock().lock();
-        try {
-          for (InProgressPool inProgressPool : inProgressPoolList) {
-            inProgressPool.finalizeReconciliation();
-            poolQueue.add(inProgressPool.getPool());
-          }
-        } finally {
-          inProgressPoolListLock.readLock().unlock();
-        }
-        inProgressPoolListLock.writeLock().lock();
-        try {
-          inProgressPoolList.clear();
-        } finally {
-          inProgressPoolListLock.writeLock().unlock();
-        }
-      }
-    };
-
-    // We will have only one thread for pool processing.
-    Thread poolProcessThread = new Thread(processPools);
-    poolProcessThread.setDaemon(true);
-    poolProcessThread.setName("Pool replica thread");
-    poolProcessThread.setUncaughtExceptionHandler((Thread t, Throwable e) -> {
-      // Let us just restart this thread after logging a critical error.
-      // if this thread is not running we cannot handle commands from SCM.
-      LOG.error("Critical Error : Pool replica thread encountered an " +
-          "error. Thread: {} Error Count : {}", t.toString(), e,
-          threadFaultCount.incrementAndGet());
-      poolProcessThread.start();
-      // TODO : Add a config to restrict how many times we will restart this
-      // thread in a single session.
-    });
-    poolProcessThread.start();
-  }
-
-  /**
-   * Adds a container report to appropriate inProgress Pool.
-   * @param containerReport  -- Container report for a specific container from
-   * a datanode.
-   */
-  public void handleContainerReport(
-      ContainerReportsRequestProto containerReport) {
-    DatanodeID datanodeID = DatanodeID.getFromProtoBuf(
-        containerReport.getDatanodeID());
-    inProgressPoolListLock.readLock().lock();
-    try {
-      String poolName = poolManager.getNodePool(datanodeID);
-      for (InProgressPool ppool : inProgressPoolList) {
-        if (ppool.getPoolName().equalsIgnoreCase(poolName)) {
-          ppool.handleContainerReport(containerReport);
-          return;
-        }
-      }
-      // TODO: Decide if we can do anything else with this report.
-      LOG.debug("Discarding the container report for pool {}. " +
-              "That pool is not currently in the pool reconciliation process." +
-              " Container Name: {}", poolName, containerReport.getDatanodeID());
-    } catch (SCMException e) {
-      LOG.warn("Skipping processing container report from datanode {}, "
-              + "cause: failed to get the corresponding node pool",
-          datanodeID.toString(), e);
-    } finally {
-      inProgressPoolListLock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Get in process pool list, used for testing.
-   * @return List of InProgressPool
-   */
-  @VisibleForTesting
-  public List<InProgressPool> getInProcessPoolList() {
-    return inProgressPoolList;
-  }
-
-  /**
-   * Shutdown the Container Replication Manager.
-   * @throws IOException if an I/O error occurs
-   */
-  @Override
-  public void close() throws IOException {
-    setExit();
-    HadoopExecutors.shutdown(executorService, LOG, 5, TimeUnit.SECONDS);
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[49/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java
new file mode 100644
index 0000000..e7df0cf
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java
@@ -0,0 +1,334 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.cblock.jscsiHelper;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableRate;
+
+/**
+ * This class is for maintaining  the various Cblock Target statistics
+ * and publishing them through the metrics interfaces.
+ * This also registers the JMX MBean for RPC.
+ *
+ * This class maintains stats like cache hit and miss ratio
+ * as well as the latency time of read and write ops.
+ */
+public class CBlockTargetMetrics {
+  // IOPS based Metrics
+  @Metric private MutableCounterLong numReadOps;
+  @Metric private MutableCounterLong numWriteOps;
+  @Metric private MutableCounterLong numReadCacheHits;
+  @Metric private MutableCounterLong numReadCacheMiss;
+  @Metric private MutableCounterLong numDirectBlockWrites;
+
+  // Cblock internal Metrics
+  @Metric private MutableCounterLong numDirtyLogBlockRead;
+  @Metric private MutableCounterLong numBytesDirtyLogRead;
+  @Metric private MutableCounterLong numBytesDirtyLogWritten;
+  @Metric private MutableCounterLong numBlockBufferFlushCompleted;
+  @Metric private MutableCounterLong numBlockBufferFlushTriggered;
+  @Metric private MutableCounterLong numBlockBufferUpdates;
+  @Metric private MutableCounterLong numRetryLogBlockRead;
+  @Metric private MutableCounterLong numBytesRetryLogRead;
+
+  // Failure Metrics
+  @Metric private MutableCounterLong numReadLostBlocks;
+  @Metric private MutableCounterLong numFailedReadBlocks;
+  @Metric private MutableCounterLong numWriteIOExceptionRetryBlocks;
+  @Metric private MutableCounterLong numWriteGenericExceptionRetryBlocks;
+  @Metric private MutableCounterLong numFailedDirectBlockWrites;
+  @Metric private MutableCounterLong numIllegalDirtyLogFiles;
+  @Metric private MutableCounterLong numFailedDirtyLogFileDeletes;
+  @Metric private MutableCounterLong numFailedBlockBufferFlushes;
+  @Metric private MutableCounterLong numInterruptedBufferWaits;
+  @Metric private MutableCounterLong numFailedRetryLogFileWrites;
+  @Metric private MutableCounterLong numWriteMaxRetryBlocks;
+  @Metric private MutableCounterLong numFailedReleaseLevelDB;
+
+  // Latency based Metrics
+  @Metric private MutableRate dbReadLatency;
+  @Metric private MutableRate containerReadLatency;
+  @Metric private MutableRate dbWriteLatency;
+  @Metric private MutableRate containerWriteLatency;
+  @Metric private MutableRate blockBufferFlushLatency;
+  @Metric private MutableRate directBlockWriteLatency;
+
+  public CBlockTargetMetrics() {
+  }
+
+  public static CBlockTargetMetrics create() {
+    MetricsSystem ms = DefaultMetricsSystem.instance();
+    return ms.register("CBlockTargetMetrics",
+        "CBlock Target Metrics",
+        new CBlockTargetMetrics());
+  }
+
+  public void incNumReadOps() {
+    numReadOps.incr();
+  }
+
+  public void incNumWriteOps() {
+    numWriteOps.incr();
+  }
+
+  public void incNumReadCacheHits() {
+    numReadCacheHits.incr();
+  }
+
+  public void incNumReadCacheMiss() {
+    numReadCacheMiss.incr();
+  }
+
+  public void incNumReadLostBlocks() {
+    numReadLostBlocks.incr();
+  }
+
+  public void incNumDirectBlockWrites() {
+    numDirectBlockWrites.incr();
+  }
+
+  public void incNumWriteIOExceptionRetryBlocks() {
+    numWriteIOExceptionRetryBlocks.incr();
+  }
+
+  public void incNumWriteGenericExceptionRetryBlocks() {
+    numWriteGenericExceptionRetryBlocks.incr();
+  }
+
+  public void incNumFailedDirectBlockWrites() {
+    numFailedDirectBlockWrites.incr();
+  }
+
+  public void incNumFailedReadBlocks() {
+    numFailedReadBlocks.incr();
+  }
+
+  public void incNumBlockBufferFlushCompleted() {
+    numBlockBufferFlushCompleted.incr();
+  }
+
+  public void incNumBlockBufferFlushTriggered() {
+    numBlockBufferFlushTriggered.incr();
+  }
+
+  public void incNumDirtyLogBlockRead() {
+    numDirtyLogBlockRead.incr();
+  }
+
+  public void incNumBytesDirtyLogRead(int bytes) {
+    numBytesDirtyLogRead.incr(bytes);
+  }
+
+  public void incNumBlockBufferUpdates() {
+    numBlockBufferUpdates.incr();
+  }
+
+  public void incNumRetryLogBlockRead() {
+    numRetryLogBlockRead.incr();
+  }
+
+  public void incNumBytesRetryLogRead(int bytes) {
+    numBytesRetryLogRead.incr(bytes);
+  }
+
+  public void incNumBytesDirtyLogWritten(int bytes) {
+    numBytesDirtyLogWritten.incr(bytes);
+  }
+
+  public void incNumFailedBlockBufferFlushes() {
+    numFailedBlockBufferFlushes.incr();
+  }
+
+  public void incNumInterruptedBufferWaits() {
+    numInterruptedBufferWaits.incr();
+  }
+
+  public void incNumIllegalDirtyLogFiles() {
+    numIllegalDirtyLogFiles.incr();
+  }
+
+  public void incNumFailedDirtyLogFileDeletes() {
+    numFailedDirtyLogFileDeletes.incr();
+  }
+
+  public void incNumFailedRetryLogFileWrites() {
+    numFailedRetryLogFileWrites.incr();
+  }
+
+  public void incNumWriteMaxRetryBlocks() {
+    numWriteMaxRetryBlocks.incr();
+  }
+
+  public void incNumFailedReleaseLevelDB() {
+    numFailedReleaseLevelDB.incr();
+  }
+
+  public void updateDBReadLatency(long latency) {
+    dbReadLatency.add(latency);
+  }
+
+  public void updateContainerReadLatency(long latency) {
+    containerReadLatency.add(latency);
+  }
+
+  public void updateDBWriteLatency(long latency) {
+    dbWriteLatency.add(latency);
+  }
+
+  public void updateContainerWriteLatency(long latency) {
+    containerWriteLatency.add(latency);
+  }
+
+  public void updateDirectBlockWriteLatency(long latency) {
+    directBlockWriteLatency.add(latency);
+  }
+
+  public void updateBlockBufferFlushLatency(long latency) {
+    blockBufferFlushLatency.add(latency);
+  }
+
+  @VisibleForTesting
+  public long getNumReadOps() {
+    return numReadOps.value();
+  }
+
+  @VisibleForTesting
+  public long getNumWriteOps() {
+    return numWriteOps.value();
+  }
+
+  @VisibleForTesting
+  public long getNumReadCacheHits() {
+    return numReadCacheHits.value();
+  }
+
+  @VisibleForTesting
+  public long getNumReadCacheMiss() {
+    return numReadCacheMiss.value();
+  }
+
+  @VisibleForTesting
+  public long getNumReadLostBlocks() {
+    return numReadLostBlocks.value();
+  }
+
+  @VisibleForTesting
+  public long getNumDirectBlockWrites() {
+    return numDirectBlockWrites.value();
+  }
+
+  @VisibleForTesting
+  public long getNumFailedDirectBlockWrites() {
+    return numFailedDirectBlockWrites.value();
+  }
+
+  @VisibleForTesting
+  public long getNumFailedReadBlocks() {
+    return numFailedReadBlocks.value();
+  }
+
+  @VisibleForTesting
+  public long getNumWriteIOExceptionRetryBlocks() {
+    return numWriteIOExceptionRetryBlocks.value();
+  }
+
+  @VisibleForTesting
+  public long getNumWriteGenericExceptionRetryBlocks() {
+    return numWriteGenericExceptionRetryBlocks.value();
+  }
+
+  @VisibleForTesting
+  public long getNumBlockBufferFlushCompleted() {
+    return numBlockBufferFlushCompleted.value();
+  }
+
+  @VisibleForTesting
+  public long getNumBlockBufferFlushTriggered() {
+    return numBlockBufferFlushTriggered.value();
+  }
+
+  @VisibleForTesting
+  public long getNumDirtyLogBlockRead() {
+    return numDirtyLogBlockRead.value();
+  }
+
+  @VisibleForTesting
+  public long getNumBytesDirtyLogReads() {
+    return numBytesDirtyLogRead.value();
+  }
+
+  @VisibleForTesting
+  public long getNumBlockBufferUpdates() {
+    return numBlockBufferUpdates.value();
+  }
+
+  @VisibleForTesting
+  public long getNumRetryLogBlockRead() {
+    return numRetryLogBlockRead.value();
+  }
+
+  @VisibleForTesting
+  public long getNumBytesRetryLogReads() {
+    return numBytesRetryLogRead.value();
+  }
+
+  @VisibleForTesting
+  public long getNumBytesDirtyLogWritten() {
+    return numBytesDirtyLogWritten.value();
+  }
+
+  @VisibleForTesting
+  public long getNumFailedBlockBufferFlushes() {
+    return numFailedBlockBufferFlushes.value();
+  }
+
+  @VisibleForTesting
+  public long getNumInterruptedBufferWaits() {
+    return numInterruptedBufferWaits.value();
+  }
+
+  @VisibleForTesting
+  public long getNumIllegalDirtyLogFiles() {
+    return numIllegalDirtyLogFiles.value();
+  }
+
+  @VisibleForTesting
+  public long getNumFailedDirtyLogFileDeletes() {
+    return numFailedDirtyLogFileDeletes.value();
+  }
+
+  @VisibleForTesting
+  public long getNumFailedRetryLogFileWrites() {
+    return numFailedRetryLogFileWrites.value();
+  }
+
+  @VisibleForTesting
+  public long getNumWriteMaxRetryBlocks() {
+    return numWriteMaxRetryBlocks.value();
+  }
+
+  @VisibleForTesting
+  public long getNumFailedReleaseLevelDB() {
+    return numFailedReleaseLevelDB.value();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java
new file mode 100644
index 0000000..75e013e
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.cblock.proto.MountVolumeResponse;
+import org.apache.hadoop.cblock.util.KeyUtil;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.jscsi.target.Configuration;
+import org.jscsi.target.Target;
+import org.jscsi.target.TargetServer;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+/**
+ * This class extends JSCSI target server, which is a ISCSI target that can be
+ * recognized by a remote machine with ISCSI installed.
+ */
+public final class CBlockTargetServer extends TargetServer {
+  private final OzoneConfiguration conf;
+  private final CBlockManagerHandler cBlockManagerHandler;
+  private final XceiverClientManager xceiverClientManager;
+  private final ContainerCacheFlusher containerCacheFlusher;
+  private final CBlockTargetMetrics metrics;
+
+  public CBlockTargetServer(OzoneConfiguration ozoneConfig,
+      Configuration jscsiConf,
+      CBlockManagerHandler cBlockManagerHandler,
+      CBlockTargetMetrics metrics)
+      throws IOException {
+    super(jscsiConf);
+    this.cBlockManagerHandler = cBlockManagerHandler;
+    this.xceiverClientManager = new XceiverClientManager(ozoneConfig);
+    this.conf = ozoneConfig;
+    this.containerCacheFlusher = new ContainerCacheFlusher(this.conf,
+        xceiverClientManager, metrics);
+    this.metrics = metrics;
+    LOGGER.info("Starting flusher thread.");
+    Thread flushListenerThread = new Thread(containerCacheFlusher);
+    flushListenerThread.setDaemon(true);
+    flushListenerThread.start();
+  }
+
+  public static void main(String[] args) throws Exception {
+  }
+
+  @Override
+  public boolean isValidTargetName(String checkTargetName) {
+    if (!KeyUtil.isValidVolumeKey(checkTargetName)) {
+      return false;
+    }
+    String userName = KeyUtil.getUserNameFromVolumeKey(checkTargetName);
+    String volumeName = KeyUtil.getVolumeFromVolumeKey(checkTargetName);
+    if (userName == null || volumeName == null) {
+      return false;
+    }
+    try {
+      MountVolumeResponse result =
+          cBlockManagerHandler.mountVolume(userName, volumeName);
+      if (!result.getIsValid()) {
+        LOGGER.error("Not a valid volume:" + checkTargetName);
+        return false;
+      }
+      String volumeKey = KeyUtil.getVolumeKey(result.getUserName(),
+          result.getVolumeName());
+      if (!targets.containsKey(volumeKey)) {
+        LOGGER.info("Mounting Volume. username: {} volume:{}",
+            userName, volumeName);
+        CBlockIStorageImpl ozoneStore = CBlockIStorageImpl.newBuilder()
+            .setUserName(userName)
+            .setVolumeName(volumeName)
+            .setVolumeSize(result.getVolumeSize())
+            .setBlockSize(result.getBlockSize())
+            .setContainerList(result.getContainerList())
+            .setClientManager(xceiverClientManager)
+            .setConf(this.conf)
+            .setFlusher(containerCacheFlusher)
+            .setCBlockTargetMetrics(metrics)
+            .build();
+        Target target = new Target(volumeKey, volumeKey, ozoneStore);
+        targets.put(volumeKey, target);
+      }
+    } catch (IOException e) {
+      LOGGER.error("Can not connect to server when validating target!"
+          + e.getMessage());
+    }
+    return targets.containsKey(checkTargetName);
+  }
+
+  @Override
+  public String[] getTargetNames() {
+    try {
+      if (cBlockManagerHandler != null) {
+        return cBlockManagerHandler.listVolumes().
+            stream().map(
+              volumeInfo -> volumeInfo.getUserName() + ":" + volumeInfo
+                .getVolumeName()).toArray(String[]::new);
+      } else {
+        return new String[0];
+      }
+    } catch (IOException e) {
+      LOGGER.error("Can't list existing volumes", e);
+      return new String[0];
+    }
+  }
+
+  @VisibleForTesting
+  public HashMap<String, Target> getTargets() {
+    return targets;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java
new file mode 100644
index 0000000..292662e
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java
@@ -0,0 +1,599 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper;
+
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Longs;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.cblock.CBlockConfigKeys;
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import org.apache.hadoop.cblock.jscsiHelper.cache.impl.AsyncBlockWriter;
+import org.apache.hadoop.cblock.jscsiHelper.cache.impl.DiskBlock;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.utils.LevelDBStore;
+import org.iq80.leveldb.Options;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_CACHE_KEEP_ALIVE;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_CACHE_MAX_POOL_SIZE;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_CACHE_QUEUE_SIZE_KB;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_CACHE_THREAD_PRIORITY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_DEFAULT;
+
+/**
+ * Class that writes to remote containers.
+ */
+public class ContainerCacheFlusher implements Runnable {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ContainerCacheFlusher.class);
+  private final LinkedBlockingQueue<Message> messageQueue;
+  private final ThreadPoolExecutor threadPoolExecutor;
+  private final ArrayBlockingQueue<Runnable> workQueue;
+  private final ConcurrentMap<String, RefCountedDB> dbMap;
+  private final ByteBuffer blockIDBuffer;
+  private final ConcurrentMap<String, Pipeline[]> pipelineMap;
+  private final AtomicLong remoteIO;
+  private final XceiverClientManager xceiverClientManager;
+  private final CBlockTargetMetrics metrics;
+  private AtomicBoolean shutdown;
+  private final long levelDBCacheSize;
+  private final int maxRetryCount;
+  private final String tracePrefix;
+
+  private final ConcurrentMap<String, FinishCounter> finishCountMap;
+
+  /**
+   * Constructs the writers to remote queue.
+   */
+  public ContainerCacheFlusher(Configuration config,
+      XceiverClientManager xceiverClientManager,
+      CBlockTargetMetrics metrics) {
+    int queueSize = config.getInt(DFS_CBLOCK_CACHE_QUEUE_SIZE_KB,
+        DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT) * 1024;
+    int corePoolSize = config.getInt(DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE,
+        DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT);
+    int maxPoolSize = config.getInt(DFS_CBLOCK_CACHE_MAX_POOL_SIZE,
+        DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT);
+    long keepAlive = config.getTimeDuration(DFS_CBLOCK_CACHE_KEEP_ALIVE,
+        DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT, TimeUnit.SECONDS);
+    int threadPri = config.getInt(DFS_CBLOCK_CACHE_THREAD_PRIORITY,
+        DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT);
+    int blockBufferSize = config.getInt(DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE,
+        DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT) * (Long.SIZE / Byte.SIZE);
+    levelDBCacheSize = config.getInt(DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_KEY,
+        DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_DEFAULT) * OzoneConsts.MB;
+
+    LOG.info("Cache: Core Pool Size: {}", corePoolSize);
+    LOG.info("Cache: Keep Alive: {}", keepAlive);
+    LOG.info("Cache: Max Pool Size: {}", maxPoolSize);
+    LOG.info("Cache: Thread Pri: {}", threadPri);
+    LOG.info("Cache: BlockBuffer Size: {}", blockBufferSize);
+
+    shutdown = new AtomicBoolean(false);
+    messageQueue = new LinkedBlockingQueue<>();
+    workQueue = new ArrayBlockingQueue<>(queueSize, true);
+
+    ThreadFactory workerThreadFactory = new ThreadFactoryBuilder()
+        .setNameFormat("Cache Block Writer Thread #%d")
+        .setDaemon(true)
+        .setPriority(threadPri)
+        .build();
+    threadPoolExecutor = new ThreadPoolExecutor(corePoolSize, maxPoolSize,
+        keepAlive, TimeUnit.SECONDS, workQueue, workerThreadFactory,
+        new ThreadPoolExecutor.AbortPolicy());
+    threadPoolExecutor.prestartAllCoreThreads();
+
+    dbMap = new ConcurrentHashMap<>();
+    pipelineMap = new ConcurrentHashMap<>();
+    blockIDBuffer = ByteBuffer.allocateDirect(blockBufferSize);
+    this.xceiverClientManager = xceiverClientManager;
+    this.metrics = metrics;
+    this.remoteIO = new AtomicLong();
+
+    this.finishCountMap = new ConcurrentHashMap<>();
+    this.maxRetryCount =
+        config.getInt(CBlockConfigKeys.DFS_CBLOCK_CACHE_MAX_RETRY_KEY,
+            CBlockConfigKeys.DFS_CBLOCK_CACHE_MAX_RETRY_DEFAULT);
+    this.tracePrefix = getTracePrefix();
+  }
+
+  private void checkExistingLog(String prefixFileName, File dbPath) {
+    if (!dbPath.exists()) {
+      LOG.debug("No existing dirty log found at {}", dbPath);
+      return;
+    }
+    LOG.debug("Need to check and requeue existing dirty log {}", dbPath);
+    HashMap<String, ArrayList<String>> allFiles = new HashMap<>();
+    traverse(prefixFileName, dbPath, allFiles);
+    for (Map.Entry<String, ArrayList<String>> entry : allFiles.entrySet()) {
+      String parentPath = entry.getKey();
+      for (String fileName : entry.getValue()) {
+        LOG.info("found {} {} with prefix {}",
+            parentPath, fileName, prefixFileName);
+        processDirtyBlocks(parentPath, fileName);
+      }
+    }
+  }
+
+  private void traverse(String prefixFileName, File path,
+                        HashMap<String, ArrayList<String>> files) {
+    if (path.isFile()) {
+      if (path.getName().startsWith(prefixFileName)) {
+        LOG.debug("found this {} with {}", path.getParent(), path.getName());
+        if (!files.containsKey(path.getParent())) {
+          files.put(path.getParent(), new ArrayList<>());
+        }
+        files.get(path.getParent()).add(path.getName());
+      }
+    } else {
+      File[] listFiles = path.listFiles();
+      if (listFiles != null) {
+        for (File subPath : listFiles) {
+          traverse(prefixFileName, subPath, files);
+        }
+      }
+    }
+  }
+
+  /**
+   * Gets the CBlockTargetMetrics.
+   *
+   * @return CBlockTargetMetrics
+   */
+  public CBlockTargetMetrics getTargetMetrics() {
+    return metrics;
+  }
+
+  /**
+   * Gets the  getXceiverClientManager.
+   *
+   * @return XceiverClientManager
+   */
+  public XceiverClientManager getXceiverClientManager() {
+    return xceiverClientManager;
+  }
+
+  /**
+   * Shutdown this instance.
+   */
+  public void shutdown() {
+    this.shutdown.set(true);
+    threadPoolExecutor.shutdown();
+  }
+
+  public long incrementRemoteIO() {
+    return remoteIO.incrementAndGet();
+  }
+
+  /**
+   * Processes a block cache file and queues those blocks for the remote I/O.
+   *
+   * @param dbPath - Location where the DB can be found.
+   * @param fileName - Block Cache File Name
+   */
+  public void processDirtyBlocks(String dbPath, String fileName) {
+    LOG.info("Adding {}/{} to queue. Queue Length: {}", dbPath, fileName,
+        messageQueue.size());
+    this.messageQueue.add(new Message(dbPath, fileName));
+  }
+
+  public Logger getLOG() {
+    return LOG;
+  }
+
+  /**
+   * Opens a DB if needed or returns a handle to an already open DB.
+   *
+   * @param dbPath -- dbPath
+   * @return the levelDB on the given path.
+   * @throws IOException
+   */
+  public synchronized LevelDBStore openDB(String dbPath)
+      throws IOException {
+    if (dbMap.containsKey(dbPath)) {
+      RefCountedDB refDB = dbMap.get(dbPath);
+      refDB.open();
+      return refDB.db;
+    } else {
+      Options options = new Options();
+      options.cacheSize(levelDBCacheSize);
+      options.createIfMissing(true);
+      LevelDBStore cacheDB = new LevelDBStore(
+          new File(getDBFileName(dbPath)), options);
+      RefCountedDB refDB = new RefCountedDB(dbPath, cacheDB);
+      dbMap.put(dbPath, refDB);
+      return cacheDB;
+    }
+  }
+
+  /**
+   * Updates the container map. This data never changes so we will update this
+   * during restarts and it should not hurt us.
+   *
+   * Once a CBlockLocalCache cache is registered, requeue dirty/retry log files
+   * for the volume
+   *
+   * @param dbPath - DbPath
+   * @param containerList - Container List.
+   */
+  public void register(String dbPath, Pipeline[] containerList) {
+    File dbFile = Paths.get(dbPath).toFile();
+    pipelineMap.put(dbPath, containerList);
+    checkExistingLog(AsyncBlockWriter.DIRTY_LOG_PREFIX, dbFile);
+    checkExistingLog(AsyncBlockWriter.RETRY_LOG_PREFIX, dbFile);
+  }
+
+  private String getDBFileName(String dbPath) {
+    return dbPath + ".db";
+  }
+
+  public LevelDBStore getCacheDB(String dbPath) throws IOException {
+    return openDB(dbPath);
+  }
+
+  public void releaseCacheDB(String dbPath) {
+    try {
+      closeDB(dbPath);
+    } catch (Exception e) {
+      metrics.incNumFailedReleaseLevelDB();
+      LOG.error("LevelDB close failed, dbPath:" + dbPath, e);
+    }
+  }
+  /**
+   * Close the DB if we don't have any outstanding references.
+   *
+   * @param dbPath - dbPath
+   * @throws IOException
+   */
+  public synchronized void closeDB(String dbPath) throws IOException {
+    if (dbMap.containsKey(dbPath)) {
+      RefCountedDB refDB = dbMap.get(dbPath);
+      int count = refDB.close();
+      if (count == 0) {
+        dbMap.remove(dbPath);
+      }
+    }
+  }
+
+  Pipeline getPipeline(String dbPath, long blockId) {
+    Pipeline[] containerList = pipelineMap.get(dbPath);
+    Preconditions.checkNotNull(containerList);
+    int containerIdx = (int) blockId % containerList.length;
+    long cBlockIndex =
+        Longs.fromByteArray(containerList[containerIdx].getData());
+    if (cBlockIndex > 0) {
+      // This catches the case when we get a wrong container in the ordering
+      // of the containers.
+      Preconditions.checkState(containerIdx % cBlockIndex == 0,
+          "The container ID computed should match with the container index " +
+              "returned from cBlock Server.");
+    }
+    return containerList[containerIdx];
+  }
+
+  public void incFinishCount(String fileName) {
+    if (!finishCountMap.containsKey(fileName)) {
+      LOG.error("No record for such file:" + fileName);
+      return;
+    }
+    finishCountMap.get(fileName).incCount();
+    if (finishCountMap.get(fileName).isFileDeleted()) {
+      finishCountMap.remove(fileName);
+    }
+  }
+
+  /**
+   * When an object implementing interface <code>Runnable</code> is used
+   * to create a thread, starting the thread causes the object's
+   * <code>run</code> method to be called in that separately executing
+   * thread.
+   * <p>
+   * The general contract of the method <code>run</code> is that it may
+   * take any action whatsoever.
+   *
+   * @see Thread#run()
+   */
+  @Override
+  public void run() {
+    while (!this.shutdown.get()) {
+      try {
+        Message message = messageQueue.take();
+        LOG.debug("Got message to process -- DB Path : {} , FileName; {}",
+            message.getDbPath(), message.getFileName());
+        String fullPath = Paths.get(message.getDbPath(),
+            message.getFileName()).toString();
+        String[] fileNameParts = message.getFileName().split("\\.");
+        Preconditions.checkState(fileNameParts.length > 1);
+        String fileType = fileNameParts[0];
+        boolean isDirtyLogFile =
+            fileType.equalsIgnoreCase(AsyncBlockWriter.DIRTY_LOG_PREFIX);
+        ReadableByteChannel fileChannel = new FileInputStream(fullPath)
+            .getChannel();
+        // TODO: We can batch and unique the IOs here. First getting the code
+        // to work, we will add those later.
+        int bytesRead = fileChannel.read(blockIDBuffer);
+        fileChannel.close();
+        LOG.debug("Read blockID log of size: {} position {} remaining {}",
+            bytesRead, blockIDBuffer.position(), blockIDBuffer.remaining());
+        // current position of in the buffer in bytes, divided by number of
+        // bytes per long (which is calculated by number of bits per long
+        // divided by number of bits per byte) gives the number of blocks
+        int blockCount = blockIDBuffer.position()/(Long.SIZE / Byte.SIZE);
+        if (isDirtyLogFile) {
+          getTargetMetrics().incNumBytesDirtyLogRead(bytesRead);
+        } else {
+          getTargetMetrics().incNumBytesRetryLogRead(bytesRead);
+        }
+        if (finishCountMap.containsKey(message.getFileName())) {
+          // In theory this should never happen. But if it happened,
+          // we need to know it...
+          getTargetMetrics().incNumIllegalDirtyLogFiles();
+          LOG.error("Adding DirtyLog file again {} current count {} new {}",
+              message.getFileName(),
+              finishCountMap.get(message.getFileName()).expectedCount,
+              blockCount);
+        }
+        finishCountMap.put(message.getFileName(),
+            new FinishCounter(blockCount, message.getDbPath(),
+                message.getFileName(), this));
+        // should be flip instead of rewind, because we also need to make sure
+        // the end position is correct.
+        blockIDBuffer.flip();
+        LOG.debug("Remaining blocks count {} and {}", blockIDBuffer.remaining(),
+            blockCount);
+        while (blockIDBuffer.remaining() >= (Long.SIZE / Byte.SIZE)) {
+          long blockID = blockIDBuffer.getLong();
+          int retryCount = 0;
+          if (isDirtyLogFile) {
+            getTargetMetrics().incNumDirtyLogBlockRead();
+          } else {
+            getTargetMetrics().incNumRetryLogBlockRead();
+            Preconditions.checkState(fileNameParts.length == 4);
+            retryCount = Integer.parseInt(fileNameParts[3]);
+          }
+          LogicalBlock block = new DiskBlock(blockID, null, false);
+          BlockWriterTask blockWriterTask = new BlockWriterTask(block, this,
+              message.getDbPath(), retryCount, message.getFileName(),
+              maxRetryCount);
+          threadPoolExecutor.submit(blockWriterTask);
+        }
+        blockIDBuffer.clear();
+      } catch (InterruptedException e) {
+        LOG.info("ContainerCacheFlusher is interrupted.", e);
+      } catch (FileNotFoundException e) {
+        LOG.error("Unable to find the dirty blocks file. This will cause " +
+            "data errors. Please stop using this volume.", e);
+      } catch (IOException e) {
+        LOG.error("Unable to read the dirty blocks file. This will cause " +
+            "data errors. Please stop using this volume.", e);
+      } catch (Exception e) {
+        LOG.error("Generic exception.", e);
+      }
+    }
+    LOG.info("Exiting flusher");
+  }
+
+  /**
+   * Tries to get the local host IP Address as trace prefix
+   * for creating trace IDs, otherwise uses a random UUID for it.
+   */
+  private static String getTracePrefix() {
+    String tmp;
+    try {
+      tmp = InetAddress.getLocalHost().getHostAddress();
+    } catch (UnknownHostException ex) {
+      tmp = UUID.randomUUID().toString();
+      LOG.error("Unable to read the host address. Using a GUID for " +
+          "hostname:{} ", tmp, ex);
+    }
+    return tmp;
+  }
+
+  /**
+   * We create a trace ID to make it easy to debug issues.
+   * A trace ID is in IPAddress:UserName:VolumeName:blockID:second format.
+   *
+   * This will get written down on the data node if we get any failures, so
+   * with this trace ID we can correlate cBlock failures across machines.
+   *
+   * @param blockID - Block ID
+   * @return trace ID
+   */
+  public String getTraceID(File dbPath, long blockID) {
+    String volumeName = dbPath.getName();
+    String userName = dbPath.getParentFile().getName();
+    // mapping to seconds to make the string smaller.
+    return tracePrefix + ":" + userName + ":" + volumeName
+        + ":" + blockID + ":" + Time.monotonicNow() / 1000;
+  }
+
+  /**
+   * Keeps a Reference counted DB that we close only when the total Reference
+   * has gone to zero.
+   */
+  private static class RefCountedDB {
+    private LevelDBStore db;
+    private AtomicInteger refcount;
+    private String dbPath;
+
+    /**
+     * RefCountedDB DB ctor.
+     *
+     * @param dbPath - DB path.
+     * @param db - LevelDBStore db
+     */
+    RefCountedDB(String dbPath, LevelDBStore db) {
+      this.db = db;
+      this.refcount = new AtomicInteger(1);
+      this.dbPath = dbPath;
+    }
+
+    /**
+     * close the DB if possible.
+     */
+    public int close() throws IOException {
+      int count = this.refcount.decrementAndGet();
+      if (count == 0) {
+        LOG.info("Closing the LevelDB. {} ", this.dbPath);
+        db.close();
+      }
+      return count;
+    }
+
+    public void open() {
+      this.refcount.incrementAndGet();
+    }
+  }
+
+  /**
+   * The message held in processing queue.
+   */
+  private static class Message {
+    private String dbPath;
+    private String fileName;
+
+    /**
+     * A message that holds the info about which path dirty blocks log and
+     * which path contains db.
+     *
+     * @param dbPath
+     * @param fileName
+     */
+    Message(String dbPath, String fileName) {
+      this.dbPath = dbPath;
+      this.fileName = fileName;
+    }
+
+    public String getDbPath() {
+      return dbPath;
+    }
+
+    public void setDbPath(String dbPath) {
+      this.dbPath = dbPath;
+    }
+
+    public String getFileName() {
+      return fileName;
+    }
+
+    public void setFileName(String fileName) {
+      this.fileName = fileName;
+    }
+  }
+
+  private static class FinishCounter {
+    private final long expectedCount;
+    private final String dbPath;
+    private final String dirtyLogPath;
+    private final AtomicLong currentCount;
+    private AtomicBoolean fileDeleted;
+    private final ContainerCacheFlusher flusher;
+
+    FinishCounter(long expectedCount, String dbPath,
+        String dirtyLogPath, ContainerCacheFlusher flusher) throws IOException {
+      this.expectedCount = expectedCount;
+      this.dbPath = dbPath;
+      this.dirtyLogPath = dirtyLogPath;
+      this.currentCount = new AtomicLong(0);
+      this.fileDeleted = new AtomicBoolean(false);
+      this.flusher = flusher;
+    }
+
+    public boolean isFileDeleted() {
+      return fileDeleted.get();
+    }
+
+    public void incCount() {
+      long count = this.currentCount.incrementAndGet();
+      if (count >= expectedCount) {
+        String filePath = String.format("%s/%s", dbPath, dirtyLogPath);
+        LOG.debug(
+            "Deleting {} with count {} {}", filePath, count, expectedCount);
+        try {
+          Path path = Paths.get(filePath);
+          Files.delete(path);
+          // the following part tries to remove the directory if it is empty
+          // but not sufficient, because the .db directory still exists....
+          // TODO how to handle the .db directory?
+          /*Path parent = path.getParent();
+          if (parent.toFile().listFiles().length == 0) {
+            Files.delete(parent);
+          }*/
+          fileDeleted.set(true);
+        } catch (Exception e) {
+          flusher.getTargetMetrics().incNumFailedDirtyLogFileDeletes();
+          LOG.error("Error deleting dirty log file:" + filePath, e);
+        }
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java
new file mode 100644
index 0000000..f164f38
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_ISCSI_ADVERTISED_IP;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_ISCSI_ADVERTISED_PORT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT;
+
+import org.apache.hadoop.cblock.CblockUtils;
+import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB;
+import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.scm.client.ContainerOperationClient;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.jscsi.target.Configuration;
+
+import java.net.InetSocketAddress;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_PORT_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_PORT_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY;
+import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
+import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT;
+import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY;
+import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT;
+import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_KEY;
+import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY;
+import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT;
+import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY;
+
+/**
+ * This class runs the target server process.
+ */
+public final class SCSITargetDaemon {
+  public static void main(String[] args) throws Exception {
+    CblockUtils.activateConfigs();
+    OzoneConfiguration ozoneConf = new OzoneConfiguration();
+
+    RPC.setProtocolEngine(ozoneConf, CBlockClientServerProtocolPB.class,
+        ProtobufRpcEngine.class);
+    long containerSizeGB = ozoneConf.getInt(DFS_CBLOCK_CONTAINER_SIZE_GB_KEY,
+        DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT);
+    ContainerOperationClient.setContainerSizeB(
+        containerSizeGB * OzoneConsts.GB);
+    String jscsiServerAddress = ozoneConf.get(
+        DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY,
+        DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT);
+    String cbmIPAddress = ozoneConf.get(
+        DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY,
+        DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT
+    );
+    int cbmPort = ozoneConf.getInt(
+        DFS_CBLOCK_JSCSI_PORT_KEY,
+        DFS_CBLOCK_JSCSI_PORT_DEFAULT
+    );
+
+    String scmAddress = ozoneConf.get(OZONE_SCM_CLIENT_BIND_HOST_KEY,
+        OZONE_SCM_CLIENT_BIND_HOST_DEFAULT);
+    int scmClientPort = ozoneConf.getInt(OZONE_SCM_CLIENT_PORT_KEY,
+        OZONE_SCM_CLIENT_PORT_DEFAULT);
+    int scmDatanodePort = ozoneConf.getInt(OZONE_SCM_DATANODE_PORT_KEY,
+        OZONE_SCM_DATANODE_PORT_DEFAULT);
+
+    String scmClientAddress = scmAddress + ":" + scmClientPort;
+    String scmDataodeAddress = scmAddress + ":" + scmDatanodePort;
+
+    ozoneConf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scmClientAddress);
+    ozoneConf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, scmDataodeAddress);
+
+    InetSocketAddress cbmAddress = new InetSocketAddress(
+        cbmIPAddress, cbmPort);
+    long version = RPC.getProtocolVersion(
+        CBlockServiceProtocolPB.class);
+    CBlockClientProtocolClientSideTranslatorPB cbmClient =
+        new CBlockClientProtocolClientSideTranslatorPB(
+            RPC.getProxy(CBlockClientServerProtocolPB.class, version,
+                cbmAddress, UserGroupInformation.getCurrentUser(), ozoneConf,
+                NetUtils.getDefaultSocketFactory(ozoneConf), 5000)
+        );
+    CBlockManagerHandler cbmHandler = new CBlockManagerHandler(cbmClient);
+
+    String advertisedAddress = ozoneConf.
+        getTrimmed(DFS_CBLOCK_ISCSI_ADVERTISED_IP, jscsiServerAddress);
+
+    int advertisedPort = ozoneConf.
+        getInt(DFS_CBLOCK_ISCSI_ADVERTISED_PORT,
+            DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT);
+
+    Configuration jscsiConfig =
+        new Configuration(jscsiServerAddress,
+            advertisedAddress,
+            advertisedPort);
+    DefaultMetricsSystem.initialize("CBlockMetrics");
+    CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+    CBlockTargetServer targetServer = new CBlockTargetServer(
+        ozoneConf, jscsiConfig, cbmHandler, metrics);
+
+    targetServer.call();
+  }
+
+  private SCSITargetDaemon() {
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java
new file mode 100644
index 0000000..300b2ae
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper.cache;
+
+import java.io.IOException;
+
+/**
+ * Defines the interface for cache implementations. The cache will be called
+ * by cblock storage module when it performs IO operations.
+ */
+public interface CacheModule {
+  /**
+   * check if the key is cached, if yes, returned the cached object.
+   * otherwise, load from data source. Then put it into cache.
+   *
+   * @param blockID
+   * @return the target block.
+   */
+  LogicalBlock get(long blockID) throws IOException;
+
+  /**
+   * put the value of the key into cache.
+   * @param blockID
+   * @param value
+   */
+  void put(long blockID, byte[] value) throws IOException;
+
+  void flush() throws IOException;
+
+  void start() throws IOException;
+
+  void stop() throws IOException;
+
+  void close() throws IOException;
+
+  boolean isDirtyCache();
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java
new file mode 100644
index 0000000..470826f
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper.cache;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Logical Block is the data structure that we write to the cache,
+ * the key and data gets written to remote contianers. Rest is used for
+ * book keeping for the cache.
+ */
+public interface LogicalBlock {
+  /**
+   * Returns the data stream of this block.
+   * @return - ByteBuffer
+   */
+  ByteBuffer getData();
+
+  /**
+   * Frees the byte buffer since we don't need it any more.
+   */
+  void clearData();
+
+  /**
+   * Returns the Block ID for this Block.
+   * @return long - BlockID
+   */
+  long getBlockID();
+
+  /**
+   * Flag that tells us if this block has been persisted to container.
+   * @return whether this block is now persistent
+   */
+  boolean isPersisted();
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java
new file mode 100644
index 0000000..992578f
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
+
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Longs;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.apache.hadoop.scm.XceiverClientSpi;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.utils.LevelDBStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * A Queue that is used to write blocks asynchronously to the container.
+ */
+public class AsyncBlockWriter {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(AsyncBlockWriter.class);
+
+  /**
+   * XceiverClientManager is used to get client connections to a set of
+   * machines.
+   */
+  private final XceiverClientManager xceiverClientManager;
+
+  /**
+   * This lock is used as a signal to re-queuing thread. The requeue thread
+   * wakes up as soon as it is signaled some blocks are in the retry queue.
+   * We try really aggressively since this new block will automatically move
+   * to the end of the queue.
+   * <p>
+   * In the event a container is unavailable for a long time, we can either
+   * fail all writes or remap and let the writes succeed. The easier
+   * semantics is to fail the volume until the container is recovered by SCM.
+   */
+  private final Lock lock;
+  private final Condition notEmpty;
+  /**
+   * The cache this writer is operating against.
+   */
+  private final CBlockLocalCache parentCache;
+  private final BlockBufferManager blockBufferManager;
+  public final static String DIRTY_LOG_PREFIX = "DirtyLog";
+  public static final String RETRY_LOG_PREFIX = "RetryLog";
+  private AtomicLong localIoCount;
+
+  /**
+   * Constructs an Async Block Writer.
+   *
+   * @param config - Config
+   * @param cache - Parent Cache for this writer
+   */
+  public AsyncBlockWriter(Configuration config, CBlockLocalCache cache) {
+
+    Preconditions.checkNotNull(cache, "Cache cannot be null.");
+    Preconditions.checkNotNull(cache.getCacheDB(), "DB cannot be null.");
+    localIoCount = new AtomicLong();
+    lock = new ReentrantLock();
+    notEmpty = lock.newCondition();
+    parentCache = cache;
+    xceiverClientManager = cache.getClientManager();
+    blockBufferManager = new BlockBufferManager(config, parentCache);
+  }
+
+  public void start() throws IOException {
+    File logDir = new File(parentCache.getDbPath().toString());
+    if (!logDir.exists() && !logDir.mkdirs()) {
+      LOG.error("Unable to create the log directory, Critical error cannot " +
+          "continue. Log Dir : {}", logDir);
+      throw new IllegalStateException("Cache Directory create failed, Cannot " +
+          "continue. Log Dir: {}" + logDir);
+    }
+    blockBufferManager.start();
+  }
+
+  /**
+   * Return the log to write to.
+   *
+   * @return Logger.
+   */
+  public static Logger getLOG() {
+    return LOG;
+  }
+
+  /**
+   * Get the CacheDB.
+   *
+   * @return LevelDB Handle
+   */
+  LevelDBStore getCacheDB() {
+    return parentCache.getCacheDB();
+  }
+
+  /**
+   * Returns the client manager.
+   *
+   * @return XceiverClientManager
+   */
+  XceiverClientManager getXceiverClientManager() {
+    return xceiverClientManager;
+  }
+
+  /**
+   * Incs the localIoPacket Count that has gone into this device.
+   */
+  public long incrementLocalIO() {
+    return localIoCount.incrementAndGet();
+  }
+
+  /**
+   * Return the local io counts to this device.
+   * @return the count of io
+   */
+  public long getLocalIOCount() {
+    return localIoCount.get();
+  }
+
+  /**
+   * Writes a block to LevelDB store and queues a work item for the system to
+   * sync the block to containers.
+   *
+   * @param block - Logical Block
+   */
+  public void writeBlock(LogicalBlock block) throws IOException {
+    byte[] keybuf = Longs.toByteArray(block.getBlockID());
+    String traceID = parentCache.getTraceID(block.getBlockID());
+    if (parentCache.isShortCircuitIOEnabled()) {
+      long startTime = Time.monotonicNow();
+      getCacheDB().put(keybuf, block.getData().array());
+      incrementLocalIO();
+      long endTime = Time.monotonicNow();
+      parentCache.getTargetMetrics().updateDBWriteLatency(
+                                                    endTime - startTime);
+      if (parentCache.isTraceEnabled()) {
+        String datahash = DigestUtils.sha256Hex(block.getData().array());
+        parentCache.getTracer().info(
+            "Task=WriterTaskDBPut,BlockID={},Time={},SHA={}",
+            block.getBlockID(), endTime - startTime, datahash);
+      }
+      block.clearData();
+      blockBufferManager.addToBlockBuffer(block.getBlockID());
+    } else {
+      Pipeline pipeline = parentCache.getPipeline(block.getBlockID());
+      String containerName = pipeline.getContainerName();
+      XceiverClientSpi client = null;
+      try {
+        long startTime = Time.monotonicNow();
+        client = parentCache.getClientManager()
+            .acquireClient(parentCache.getPipeline(block.getBlockID()));
+        ContainerProtocolCalls.writeSmallFile(client, containerName,
+            Long.toString(block.getBlockID()), block.getData().array(),
+            traceID);
+        long endTime = Time.monotonicNow();
+        if (parentCache.isTraceEnabled()) {
+          String datahash = DigestUtils.sha256Hex(block.getData().array());
+          parentCache.getTracer().info(
+              "Task=DirectWriterPut,BlockID={},Time={},SHA={}",
+              block.getBlockID(), endTime - startTime, datahash);
+        }
+        parentCache.getTargetMetrics().
+            updateDirectBlockWriteLatency(endTime - startTime);
+        parentCache.getTargetMetrics().incNumDirectBlockWrites();
+      } catch (Exception ex) {
+        parentCache.getTargetMetrics().incNumFailedDirectBlockWrites();
+        LOG.error("Direct I/O writing of block:{} traceID:{} to "
+            + "container {} failed", block.getBlockID(), traceID,
+            containerName, ex);
+        throw ex;
+      } finally {
+        if (client != null) {
+          parentCache.getClientManager().releaseClient(client);
+        }
+        block.clearData();
+      }
+    }
+  }
+
+  /**
+   * Shutdown by writing any pending I/O to dirtylog buffer.
+   */
+  public void shutdown() {
+    blockBufferManager.shutdown();
+  }
+  /**
+   * Returns tracer.
+   *
+   * @return Tracer
+   */
+  Logger getTracer() {
+    return parentCache.getTracer();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java
new file mode 100644
index 0000000..c61a7a4
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
+
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Paths;
+
+/**
+ * This task is responsible for flushing the BlockIDBuffer
+ * to Dirty Log File. This Dirty Log file is used later by
+ * ContainerCacheFlusher when the data is written to container
+ */
+public class BlockBufferFlushTask implements Runnable {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(BlockBufferFlushTask.class);
+  private final CBlockLocalCache parentCache;
+  private final BlockBufferManager bufferManager;
+  private final ByteBuffer blockIDBuffer;
+
+  BlockBufferFlushTask(ByteBuffer blockIDBuffer, CBlockLocalCache parentCache,
+                       BlockBufferManager manager) {
+    this.parentCache = parentCache;
+    this.bufferManager = manager;
+    this.blockIDBuffer = blockIDBuffer;
+  }
+
+  /**
+   * When an object implementing interface <code>Runnable</code> is used
+   * to create a thread, starting the thread causes the object's
+   * <code>run</code> method to be called in that separately executing
+   * thread.
+   * <p>
+   * The general contract of the method <code>run</code> is that it may
+   * take any action whatsoever.
+   *
+   * @see Thread#run()
+   */
+  @Override
+  public void run() {
+    try {
+      writeBlockBufferToFile(blockIDBuffer);
+    } catch (Exception e) {
+      parentCache.getTargetMetrics().incNumFailedBlockBufferFlushes();
+      LOG.error("Unable to sync the Block map to disk with "
+          + (blockIDBuffer.position() / Long.SIZE) + "entries "
+          + "-- NOTE: This might cause a data loss or corruption", e);
+    } finally {
+      bufferManager.releaseBuffer(blockIDBuffer);
+    }
+  }
+
+  /**
+   * Write Block Buffer to file.
+   *
+   * @param buffer - ByteBuffer
+   * @throws IOException
+   */
+  private void writeBlockBufferToFile(ByteBuffer buffer)
+      throws IOException {
+    long startTime = Time.monotonicNow();
+    boolean append = false;
+
+    // If there is nothing written to blockId buffer,
+    // then skip flushing of blockId buffer
+    if (buffer.position() == 0) {
+      return;
+    }
+
+    buffer.flip();
+    String fileName =
+        String.format("%s.%s", AsyncBlockWriter.DIRTY_LOG_PREFIX,
+                      Time.monotonicNow());
+    String log = Paths.get(parentCache.getDbPath().toString(), fileName)
+        .toString();
+
+    FileChannel channel = new FileOutputStream(log, append).getChannel();
+    int bytesWritten = channel.write(buffer);
+    channel.close();
+    buffer.clear();
+    parentCache.processDirtyMessage(fileName);
+    long endTime = Time.monotonicNow();
+    if (parentCache.isTraceEnabled()) {
+      parentCache.getTracer().info(
+          "Task=DirtyBlockLogWrite,Time={} bytesWritten={}",
+          endTime - startTime, bytesWritten);
+    }
+
+    parentCache.getTargetMetrics().incNumBlockBufferFlushCompleted();
+    parentCache.getTargetMetrics().incNumBytesDirtyLogWritten(bytesWritten);
+    parentCache.getTargetMetrics().
+        updateBlockBufferFlushLatency(endTime - startTime);
+    LOG.debug("Block buffer writer bytesWritten:{} Time:{}",
+        bytesWritten, endTime - startTime);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java
new file mode 100644
index 0000000..5d3209c
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.ThreadFactory;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+    DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+    DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+    DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+    DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+    DFS_CBLOCK_CACHE_KEEP_ALIVE;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+    DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+    DFS_CBLOCK_CACHE_THREAD_PRIORITY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+    DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT;
+
+/**
+ * This class manages the block ID buffer.
+ * Block ID Buffer keeps a list of blocks which are in leveldb cache
+ * This buffer is used later when the blocks are flushed to container
+ *
+ * Two blockIDBuffers are maintained so that write are not blocked when
+ * DirtyLog is being written. Once a blockIDBuffer is full, it will be
+ * enqueued for DirtyLog write while the other buffer accepts new write.
+ * Once the DirtyLog write is done, the buffer is returned back to the pool.
+ *
+ * There are three triggers for blockIDBuffer flush
+ * 1) BlockIDBuffer is full,
+ * 2) Time period defined for blockIDBuffer flush has elapsed.
+ * 3) Shutdown
+ */
+public class BlockBufferManager {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(BlockBufferManager.class);
+
+  private enum FlushReason {
+    BUFFER_FULL,
+    SHUTDOWN,
+    TIMER
+  };
+
+  private final int blockBufferSize;
+  private final CBlockLocalCache parentCache;
+  private final ScheduledThreadPoolExecutor scheduledExecutor;
+  private final ThreadPoolExecutor threadPoolExecutor;
+  private final long intervalSeconds;
+  private final ArrayBlockingQueue<ByteBuffer> acquireQueue;
+  private final ArrayBlockingQueue<Runnable> workQueue;
+  private ByteBuffer currentBuffer;
+
+  BlockBufferManager(Configuration config, CBlockLocalCache parentCache) {
+    this.parentCache = parentCache;
+    this.scheduledExecutor = new ScheduledThreadPoolExecutor(1);
+
+    this.intervalSeconds =
+        config.getTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL,
+            DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL_DEFAULT,
+            TimeUnit.SECONDS);
+
+    long keepAlive = config.getTimeDuration(DFS_CBLOCK_CACHE_KEEP_ALIVE,
+        DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT,
+        TimeUnit.SECONDS);
+    this.workQueue = new ArrayBlockingQueue<>(2, true);
+    int threadPri = config.getInt(DFS_CBLOCK_CACHE_THREAD_PRIORITY,
+        DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT);
+    ThreadFactory workerThreadFactory = new ThreadFactoryBuilder()
+        .setNameFormat("Cache Block Buffer Manager Thread #%d")
+        .setDaemon(true)
+        .setPriority(threadPri)
+        .build();
+    /*
+     * starting a thread pool with core pool size of 1 and maximum of 2 threads
+     * as there are maximum of 2 buffers which can be flushed at the same time.
+     */
+    this.threadPoolExecutor = new ThreadPoolExecutor(1, 2,
+        keepAlive, TimeUnit.SECONDS, workQueue, workerThreadFactory,
+        new ThreadPoolExecutor.AbortPolicy());
+
+    this.blockBufferSize = config.getInt(DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE,
+        DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT) * (Long.SIZE / Byte.SIZE);
+    this.acquireQueue = new ArrayBlockingQueue<>(2, true);
+
+    for (int i = 0; i < 2; i++) {
+      acquireQueue.add(ByteBuffer.allocate(blockBufferSize));
+    }
+    // get the first buffer to be used
+    this.currentBuffer = acquireQueue.remove();
+
+    LOG.info("BufferManager: Buffer Size:{} FlushIntervalSeconds:{}",
+        blockBufferSize, intervalSeconds);
+  }
+
+  // triggerBlockBufferFlush enqueues current ByteBuffer for flush and returns.
+  // This enqueue is asynchronous and hence triggerBlockBufferFlush will
+  // only block when there are no available buffers in acquireQueue
+  // Once the DirtyLog write is done, buffer is returned back to
+  // BlockBufferManager using releaseBuffer
+  private synchronized void triggerBlockBufferFlush(FlushReason reason) {
+    LOG.debug("Flush triggered because: " + reason.toString() +
+        " Num entries in buffer: " +
+        currentBuffer.position() / (Long.SIZE / Byte.SIZE) +
+        " Acquire Queue Size: " + acquireQueue.size());
+
+    parentCache.getTargetMetrics().incNumBlockBufferFlushTriggered();
+    BlockBufferFlushTask flushTask =
+        new BlockBufferFlushTask(currentBuffer, parentCache, this);
+    threadPoolExecutor.submit(flushTask);
+    try {
+      currentBuffer = acquireQueue.take();
+    } catch (InterruptedException ex) {
+      currentBuffer = null;
+      parentCache.getTargetMetrics().incNumInterruptedBufferWaits();
+      LOG.error("wait on take operation on acquire queue interrupted", ex);
+      Thread.currentThread().interrupt();
+    }
+  }
+
+  public synchronized void addToBlockBuffer(long blockId)  {
+    parentCache.getTargetMetrics().incNumBlockBufferUpdates();
+    currentBuffer.putLong(blockId);
+    // if no space left, flush this buffer
+    if (currentBuffer.remaining() == 0) {
+      triggerBlockBufferFlush(FlushReason.BUFFER_FULL);
+    }
+  }
+
+  public void releaseBuffer(ByteBuffer buffer) {
+    if (buffer.position() != 0) {
+      LOG.error("requeuing a non empty buffer with:{}",
+          "elements enqueued in the acquire queue",
+          buffer.position() / (Long.SIZE / Byte.SIZE));
+      buffer.reset();
+    }
+    // There should always be space in the queue to add an element
+    acquireQueue.add(buffer);
+  }
+
+  // Start a scheduled task to flush blockIDBuffer
+  public void start() {
+    Runnable scheduledTask = () -> triggerBlockBufferFlush(FlushReason.TIMER);
+    scheduledExecutor.scheduleWithFixedDelay(scheduledTask, intervalSeconds,
+                                        intervalSeconds, TimeUnit.SECONDS);
+    threadPoolExecutor.prestartAllCoreThreads();
+  }
+
+  public void shutdown() {
+    triggerBlockBufferFlush(FlushReason.SHUTDOWN);
+    scheduledExecutor.shutdown();
+    threadPoolExecutor.shutdown();
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java
new file mode 100644
index 0000000..1149164
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java
@@ -0,0 +1,577 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
+
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Longs;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher;
+import org.apache.hadoop.cblock.jscsiHelper.cache.CacheModule;
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics;
+import org.apache.hadoop.utils.LevelDBStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.FileStore;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_DISK_CACHE_PATH_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_TRACE_IO;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+    .DFS_CBLOCK_TRACE_IO_DEFAULT;
+
+/**
+ * A local cache used by the CBlock ISCSI server. This class is enabled or
+ * disabled via config settings.
+ */
+public class CBlockLocalCache implements CacheModule {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(CBlockLocalCache.class);
+  private static final Logger TRACER =
+      LoggerFactory.getLogger("TraceIO");
+
+  private final Configuration conf;
+  /**
+   * LevelDB cache file.
+   */
+  private final LevelDBStore cacheDB;
+
+  /**
+   * AsyncBlock writer updates the cacheDB and writes the blocks async to
+   * remote containers.
+   */
+  private final AsyncBlockWriter blockWriter;
+
+  /**
+   * Sync block reader tries to read from the cache and if we get a cache
+   * miss we will fetch the block from remote location. It will asynchronously
+   * update the cacheDB.
+   */
+  private final SyncBlockReader blockReader;
+  private final String userName;
+  private final String volumeName;
+
+  /**
+   * From a block ID we are able to get the pipeline by indexing this array.
+   */
+  private final Pipeline[] containerList;
+  private final int blockSize;
+  private XceiverClientManager clientManager;
+  /**
+   * If this flag is enabled then cache traces all I/O, all reads and writes
+   * are visible in the log with sha of the block written. Makes the system
+   * slower use it only for debugging or creating trace simulations.
+   */
+  private final boolean traceEnabled;
+  private final boolean enableShortCircuitIO;
+  private final long volumeSize;
+  private long currentCacheSize;
+  private File dbPath;
+  private final ContainerCacheFlusher flusher;
+  private CBlockTargetMetrics cblockTargetMetrics;
+
+  /**
+   * Get Db Path.
+   * @return the file instance of the db.
+   */
+  public File getDbPath() {
+    return dbPath;
+  }
+
+  /**
+   * Constructor for CBlockLocalCache invoked via the builder.
+   *
+   * @param conf -  Configuration
+   * @param volumeName - volume Name
+   * @param userName - user name
+   * @param containerPipelines - Pipelines that make up this contianer
+   * @param blockSize - blockSize
+   * @param flusher - flusher to flush data to container
+   * @throws IOException
+   */
+  CBlockLocalCache(
+      Configuration conf, String volumeName,
+      String userName, List<Pipeline> containerPipelines, int blockSize,
+      long volumeSize, ContainerCacheFlusher flusher) throws IOException {
+    this.conf = conf;
+    this.userName = userName;
+    this.volumeName = volumeName;
+    this.blockSize = blockSize;
+    this.flusher = flusher;
+    this.traceEnabled = conf.getBoolean(DFS_CBLOCK_TRACE_IO,
+        DFS_CBLOCK_TRACE_IO_DEFAULT);
+    this.enableShortCircuitIO = conf.getBoolean(
+        DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO,
+        DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO_DEFAULT);
+    dbPath = Paths.get(conf.get(DFS_CBLOCK_DISK_CACHE_PATH_KEY,
+        DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT), userName, volumeName).toFile();
+
+    if (!dbPath.exists() && !dbPath.mkdirs()) {
+      LOG.error("Unable to create the cache paths. Path: {}", dbPath);
+      throw new IllegalArgumentException("Unable to create paths. Path: " +
+          dbPath);
+    }
+    cacheDB = flusher.getCacheDB(dbPath.toString());
+    this.containerList = containerPipelines.toArray(new
+        Pipeline[containerPipelines.size()]);
+    this.volumeSize = volumeSize;
+
+    blockWriter = new AsyncBlockWriter(conf, this);
+    blockReader = new SyncBlockReader(conf, this);
+    if (this.traceEnabled) {
+      getTracer().info("Task=StartingCache");
+    }
+  }
+
+  private void setClientManager(XceiverClientManager manager) {
+    this.clientManager = manager;
+  }
+
+  private void setCblockTargetMetrics(CBlockTargetMetrics targetMetrics) {
+    this.cblockTargetMetrics = targetMetrics;
+  }
+
+  /**
+   * Returns new builder class that builds a CBlockLocalCache.
+   *
+   * @return Builder
+   */
+  public static Builder newBuilder() {
+    return new Builder();
+  }
+
+  public void processDirtyMessage(String fileName) {
+    flusher.processDirtyBlocks(dbPath.toString(), fileName);
+  }
+
+  /**
+   * Get usable disk space.
+   *
+   * @param dbPathString - Path to db
+   * @return long bytes remaining.
+   */
+  private static long getRemainingDiskSpace(String dbPathString) {
+    try {
+      URI fileUri = new URI("file:///");
+      Path dbPath = Paths.get(fileUri).resolve(dbPathString);
+      FileStore disk = Files.getFileStore(dbPath);
+      return disk.getUsableSpace();
+    } catch (URISyntaxException | IOException ex) {
+      LOG.error("Unable to get free space on for path :" + dbPathString);
+    }
+    return 0L;
+  }
+
+  /**
+   * Returns the Max current CacheSize.
+   *
+   * @return - Cache Size
+   */
+  public long getCurrentCacheSize() {
+    return currentCacheSize;
+  }
+
+  /**
+   * Sets the Maximum Cache Size.
+   *
+   * @param currentCacheSize - Max current Cache Size.
+   */
+  public void setCurrentCacheSize(long currentCacheSize) {
+    this.currentCacheSize = currentCacheSize;
+  }
+
+  /**
+   * True if block tracing is enabled.
+   *
+   * @return - bool
+   */
+  public boolean isTraceEnabled() {
+    return traceEnabled;
+  }
+
+  /**
+   * Checks if Short Circuit I/O is enabled.
+   *
+   * @return - true if it is enabled.
+   */
+  public boolean isShortCircuitIOEnabled() {
+    return enableShortCircuitIO;
+  }
+
+  /**
+   * Returns the default block size of this device.
+   *
+   * @return - int
+   */
+  public int getBlockSize() {
+    return blockSize;
+  }
+
+  /**
+   * Gets the client manager.
+   *
+   * @return XceiverClientManager
+   */
+  public XceiverClientManager getClientManager() {
+    return clientManager;
+  }
+
+  /**
+   * check if the key is cached, if yes, returned the cached object.
+   * otherwise, load from data source. Then put it into cache.
+   *
+   * @param blockID
+   * @return the block associated to the blockID
+   */
+  @Override
+  public LogicalBlock get(long blockID) throws IOException {
+    cblockTargetMetrics.incNumReadOps();
+    return blockReader.readBlock(blockID);
+  }
+
+  /**
+   * put the value of the key into cache and remote container.
+   *
+   * @param blockID - BlockID
+   * @param data - byte[]
+   */
+  @Override
+  public void put(long blockID, byte[] data) throws IOException {
+    cblockTargetMetrics.incNumWriteOps();
+    LogicalBlock block = new DiskBlock(blockID, data, false);
+    blockWriter.writeBlock(block);
+  }
+
+  @Override
+  public void flush() throws IOException {
+
+  }
+
+  @Override
+  public void start() throws IOException {
+    flusher.register(getDbPath().getPath(), containerList);
+    blockWriter.start();
+  }
+
+  @Override
+  public void stop() throws IOException {
+  }
+
+  @Override
+  public void close() throws IOException {
+    blockReader.shutdown();
+    blockWriter.shutdown();
+    this.flusher.releaseCacheDB(dbPath.toString());
+    if (this.traceEnabled) {
+      getTracer().info("Task=ShutdownCache");
+    }
+  }
+
+  /**
+   * Returns true if cache still has blocks pending to write.
+   *
+   * @return false if we have no pending blocks to write.
+   */
+  @Override
+  public boolean isDirtyCache() {
+    return false;
+  }
+
+  /**
+   * Returns the local cache DB.
+   *
+   * @return - DB
+   */
+  LevelDBStore getCacheDB() {
+    return this.cacheDB;
+  }
+
+  /**
+   * Returns the current userName.
+   *
+   * @return - UserName
+   */
+  String getUserName() {
+    return this.userName;
+  }
+
+  /**
+   * Returns the volume name.
+   *
+   * @return VolumeName.
+   */
+  String getVolumeName() {
+    return this.volumeName;
+  }
+
+  /**
+   * Returns the target metrics.
+   *
+   * @return CBlock Target Metrics.
+   */
+  CBlockTargetMetrics getTargetMetrics() {
+    return this.cblockTargetMetrics;
+  }
+
+  /**
+   * Returns the pipeline to use given a container.
+   *
+   * @param blockId - blockID
+   * @return - pipeline.
+   */
+  Pipeline getPipeline(long blockId) {
+    int containerIdx = (int) blockId % containerList.length;
+    long cBlockIndex =
+        Longs.fromByteArray(containerList[containerIdx].getData());
+    if (cBlockIndex > 0) {
+      // This catches the case when we get a wrong container in the ordering
+      // of the containers.
+      Preconditions.checkState(containerIdx % cBlockIndex == 0,
+          "The container ID computed should match with the container index " +
+              "returned from cBlock Server.");
+    }
+    return containerList[containerIdx];
+  }
+
+  String getTraceID(long blockID) {
+    return flusher.getTraceID(dbPath, blockID);
+  }
+
+  /**
+   * Returns tracer.
+   *
+   * @return - Logger
+   */
+  Logger getTracer() {
+    return TRACER;
+  }
+
+  /**
+   * Builder class for CBlocklocalCache.
+   */
+  public static class Builder {
+    private Configuration configuration;
+    private String userName;
+    private String volumeName;
+    private List<Pipeline> pipelines;
+    private XceiverClientManager clientManager;
+    private int blockSize;
+    private long volumeSize;
+    private ContainerCacheFlusher flusher;
+    private CBlockTargetMetrics metrics;
+
+    /**
+     * Ctor.
+     */
+    Builder() {
+    }
+
+    /**
+     * Computes a cache size based on the configuration and available disk
+     * space.
+     *
+     * @param configuration - Config
+     * @param volumeSize - Size of Volume
+     * @param blockSize - Size of the block
+     * @return - cache size in bytes.
+     */
+    private static long computeCacheSize(Configuration configuration,
+        long volumeSize, int blockSize) {
+      long cacheSize = 0;
+      String dbPath = configuration.get(DFS_CBLOCK_DISK_CACHE_PATH_KEY,
+          DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT);
+      if (StringUtils.isBlank(dbPath)) {
+        return cacheSize;
+      }
+      long spaceRemaining = getRemainingDiskSpace(dbPath);
+      double cacheRatio = 1.0;
+
+      if (spaceRemaining < volumeSize) {
+        cacheRatio = (double)spaceRemaining / volumeSize;
+      }
+
+      // if cache is going to be at least 10% of the volume size it is worth
+      // doing, otherwise skip creating the  cache.
+      if (cacheRatio >= 0.10) {
+        cacheSize = Double.doubleToLongBits(volumeSize * cacheRatio);
+      }
+      return cacheSize;
+    }
+
+    /**
+     * Sets the Config to be used by this cache.
+     *
+     * @param conf - Config
+     * @return Builder
+     */
+    public Builder setConfiguration(Configuration conf) {
+      this.configuration = conf;
+      return this;
+    }
+
+    /**
+     * Sets the user name who is the owner of this volume.
+     *
+     * @param user - name of the owner, please note this is not the current
+     * user name.
+     * @return - Builder
+     */
+    public Builder setUserName(String user) {
+      this.userName = user;
+      return this;
+    }
+
+    /**
+     * Sets the VolumeName.
+     *
+     * @param volume - Name of the volume
+     * @return Builder
+     */
+    public Builder setVolumeName(String volume) {
+      this.volumeName = volume;
+      return this;
+    }
+
+    /**
+     * Sets the Pipelines that form this volume.
+     *
+     * @param pipelineList - list of pipelines
+     * @return Builder
+     */
+    public Builder setPipelines(List<Pipeline> pipelineList) {
+      this.pipelines = pipelineList;
+      return this;
+    }
+
+    /**
+     * Sets the Client Manager that manages the communication with containers.
+     *
+     * @param xceiverClientManager - clientManager.
+     * @return - Builder
+     */
+    public Builder setClientManager(XceiverClientManager xceiverClientManager) {
+      this.clientManager = xceiverClientManager;
+      return this;
+    }
+
+    /**
+     * Sets the block size -- Typical sizes are 4KB, 8KB etc.
+     *
+     * @param size - BlockSize.
+     * @return - Builder
+     */
+    public Builder setBlockSize(int size) {
+      this.blockSize = size;
+      return this;
+    }
+
+    /**
+     * Sets the volumeSize.
+     *
+     * @param size - VolumeSize
+     * @return - Builder
+     */
+    public Builder setVolumeSize(long size) {
+      this.volumeSize = size;
+      return this;
+    }
+
+    /**
+     * Set flusher.
+     * @param containerCacheFlusher - cache Flusher
+     * @return Builder.
+     */
+    public Builder setFlusher(ContainerCacheFlusher containerCacheFlusher) {
+      this.flusher = containerCacheFlusher;
+      return this;
+    }
+
+    /**
+     * Sets the cblock Metrics.
+     *
+     * @param targetMetrics - CBlock Target Metrics
+     * @return - Builder
+     */
+    public Builder setCBlockTargetMetrics(CBlockTargetMetrics targetMetrics) {
+      this.metrics = targetMetrics;
+      return this;
+    }
+
+    /**
+     * Constructs a CBlockLocalCache.
+     *
+     * @return the CBlockLocalCache with the preset properties.
+     * @throws IOException
+     */
+    public CBlockLocalCache build() throws IOException {
+      Preconditions.checkNotNull(this.configuration, "A valid configuration " +
+          "is needed");
+      Preconditions.checkState(StringUtils.isNotBlank(userName), "A valid " +
+          "username is needed");
+      Preconditions.checkState(StringUtils.isNotBlank(volumeName), " A valid" +
+          " volume name is needed");
+      Preconditions.checkNotNull(this.pipelines, "Pipelines cannot be null");
+      Preconditions.checkState(this.pipelines.size() > 0, "At least one " +
+          "pipeline location is needed for a volume");
+
+      for (int x = 0; x < pipelines.size(); x++) {
+        Preconditions.checkNotNull(pipelines.get(x).getData(), "cBlock " +
+            "relies on private data on the pipeline, null data found.");
+      }
+
+      Preconditions.checkNotNull(clientManager, "Client Manager cannot be " +
+          "null");
+      Preconditions.checkState(blockSize > 0, " Block size has to be a " +
+          "number greater than 0");
+
+      Preconditions.checkState(volumeSize > 0, "Volume Size cannot be less " +
+          "than 1");
+      Preconditions.checkNotNull(this.flusher, "Flusher cannot be null.");
+
+      CBlockLocalCache cache = new CBlockLocalCache(this.configuration,
+          this.volumeName, this.userName, this.pipelines, blockSize,
+          volumeSize, flusher);
+      cache.setCblockTargetMetrics(this.metrics);
+      cache.setClientManager(this.clientManager);
+
+      // TODO : Support user configurable maximum size.
+      long cacheSize = computeCacheSize(this.configuration, this.volumeSize,
+          this.blockSize);
+      cache.setCurrentCacheSize(cacheSize);
+      return cache;
+    }
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[18/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/package-info.java
deleted file mode 100644
index 7686df3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm;
-
-/*
- * This package contains StorageContainerManager classes.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/PipelineManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/PipelineManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/PipelineManager.java
deleted file mode 100644
index 28e5267..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/PipelineManager.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.pipelines;
-
-
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.scm.container.common.helpers.PipelineChannel;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Manage Ozone pipelines.
- */
-public abstract class PipelineManager {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(PipelineManager.class);
-  private final List<PipelineChannel> activePipelineChannels;
-  private final AtomicInteger conduitsIndex;
-
-  public PipelineManager() {
-    activePipelineChannels = new LinkedList<>();
-    conduitsIndex = new AtomicInteger(0);
-  }
-
-  /**
-   * This function is called by the Container Manager while allocating a new
-   * container. The client specifies what kind of replication pipeline is
-   * needed and based on the replication type in the request appropriate
-   * Interface is invoked.
-   *
-   * @param containerName Name of the container
-   * @param replicationFactor - Replication Factor
-   * @return a Pipeline.
-   */
-  public synchronized final Pipeline getPipeline(String containerName,
-      ReplicationFactor replicationFactor, ReplicationType replicationType)
-      throws IOException {
-    /**
-     * In the Ozone world, we have a very simple policy.
-     *
-     * 1. Try to create a pipelineChannel if there are enough free nodes.
-     *
-     * 2. This allows all nodes to part of a pipelineChannel quickly.
-     *
-     * 3. if there are not enough free nodes, return conduits in a
-     * round-robin fashion.
-     *
-     * TODO: Might have to come up with a better algorithm than this.
-     * Create a new placement policy that returns conduits in round robin
-     * fashion.
-     */
-    PipelineChannel pipelineChannel =
-        allocatePipelineChannel(replicationFactor);
-    if (pipelineChannel != null) {
-      LOG.debug("created new pipelineChannel:{} for container:{}",
-          pipelineChannel.getName(), containerName);
-      activePipelineChannels.add(pipelineChannel);
-    } else {
-      pipelineChannel =
-          findOpenPipelineChannel(replicationType, replicationFactor);
-      if (pipelineChannel != null) {
-        LOG.debug("re-used pipelineChannel:{} for container:{}",
-            pipelineChannel.getName(), containerName);
-      }
-    }
-    if (pipelineChannel == null) {
-      LOG.error("Get pipelineChannel call failed. We are not able to find" +
-              "free nodes or operational pipelineChannel.");
-      return null;
-    } else {
-      return new Pipeline(containerName, pipelineChannel);
-    }
-  }
-
-  protected int getReplicationCount(ReplicationFactor factor) {
-    switch (factor) {
-    case ONE:
-      return 1;
-    case THREE:
-      return 3;
-    default:
-      throw new IllegalArgumentException("Unexpected replication count");
-    }
-  }
-
-  public abstract PipelineChannel allocatePipelineChannel(
-      ReplicationFactor replicationFactor) throws IOException;
-
-  /**
-   * Find a PipelineChannel that is operational.
-   *
-   * @return - Pipeline or null
-   */
-  private PipelineChannel findOpenPipelineChannel(
-      ReplicationType type, ReplicationFactor factor) {
-    PipelineChannel pipelineChannel = null;
-    final int sentinal = -1;
-    if (activePipelineChannels.size() == 0) {
-      LOG.error("No Operational conduits found. Returning null.");
-      return null;
-    }
-    int startIndex = getNextIndex();
-    int nextIndex = sentinal;
-    for (; startIndex != nextIndex; nextIndex = getNextIndex()) {
-      // Just walk the list in a circular way.
-      PipelineChannel temp =
-          activePipelineChannels
-              .get(nextIndex != sentinal ? nextIndex : startIndex);
-      // if we find an operational pipelineChannel just return that.
-      if ((temp.getLifeCycleState() == LifeCycleState.OPEN) &&
-          (temp.getFactor() == factor) && (temp.getType() == type)) {
-        pipelineChannel = temp;
-        break;
-      }
-    }
-    return pipelineChannel;
-  }
-
-  /**
-   * gets the next index of the PipelineChannel to get.
-   *
-   * @return index in the link list to get.
-   */
-  private int getNextIndex() {
-    return conduitsIndex.incrementAndGet() % activePipelineChannels.size();
-  }
-
-  /**
-   * Creates a pipeline from a specified set of Nodes.
-   * @param pipelineID - Name of the pipeline
-   * @param datanodes - The list of datanodes that make this pipeline.
-   */
-  public abstract void createPipeline(String pipelineID,
-      List<DatanodeID> datanodes) throws IOException;
-
-  /**
-   * Close the  pipeline with the given clusterId.
-   */
-  public abstract void closePipeline(String pipelineID) throws IOException;
-
-  /**
-   * list members in the pipeline .
-   * @return the datanode
-   */
-  public abstract List<DatanodeID> getMembers(String pipelineID)
-      throws IOException;
-
-  /**
-   * Update the datanode list of the pipeline.
-   */
-  public abstract void updatePipeline(String pipelineID,
-      List<DatanodeID> newDatanodes) throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/PipelineSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/PipelineSelector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/PipelineSelector.java
deleted file mode 100644
index 7759f9c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/PipelineSelector.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.pipelines;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.ozone.scm.container.placement.algorithms.ContainerPlacementPolicy;
-import org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementRandom;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.scm.pipelines.ratis.RatisManagerImpl;
-import org.apache.hadoop.ozone.scm.pipelines.standalone.StandaloneManagerImpl;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.container.common.helpers.PipelineChannel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Sends the request to the right pipeline manager.
- */
-public class PipelineSelector {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(PipelineSelector.class);
-  private final ContainerPlacementPolicy placementPolicy;
-  private final NodeManager nodeManager;
-  private final Configuration conf;
-  private final RatisManagerImpl ratisManager;
-  private final StandaloneManagerImpl standaloneManager;
-  private final long containerSize;
-
-  /**
-   * Constructs a pipeline Selector.
-   *
-   * @param nodeManager - node manager
-   * @param conf - Ozone Config
-   */
-  public PipelineSelector(NodeManager nodeManager, Configuration conf) {
-    this.nodeManager = nodeManager;
-    this.conf = conf;
-    this.placementPolicy = createContainerPlacementPolicy(nodeManager, conf);
-    this.containerSize = OzoneConsts.GB * this.conf.getInt(
-        ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB,
-        ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT);
-    this.standaloneManager =
-        new StandaloneManagerImpl(this.nodeManager, placementPolicy,
-            containerSize);
-    this.ratisManager =
-        new RatisManagerImpl(this.nodeManager, placementPolicy, containerSize,
-            conf);
-  }
-
-  /**
-   * Translates a list of nodes, ordered such that the first is the leader, into
-   * a corresponding {@link Pipeline} object.
-   *
-   * @param nodes - list of datanodes on which we will allocate the container.
-   * The first of the list will be the leader node.
-   * @return pipeline corresponding to nodes
-   */
-  public static PipelineChannel newPipelineFromNodes(List<DatanodeID> nodes,
-      LifeCycleState state, ReplicationType replicationType,
-      ReplicationFactor replicationFactor, String name) {
-    Preconditions.checkNotNull(nodes);
-    Preconditions.checkArgument(nodes.size() > 0);
-    String leaderId = nodes.get(0).getDatanodeUuid();
-    PipelineChannel
-        pipelineChannel = new PipelineChannel(leaderId, state, replicationType,
-        replicationFactor, name);
-    for (DatanodeID node : nodes) {
-      pipelineChannel.addMember(node);
-    }
-    return pipelineChannel;
-  }
-
-  /**
-   * Create pluggable container placement policy implementation instance.
-   *
-   * @param nodeManager - SCM node manager.
-   * @param conf - configuration.
-   * @return SCM container placement policy implementation instance.
-   */
-  @SuppressWarnings("unchecked")
-  private static ContainerPlacementPolicy createContainerPlacementPolicy(
-      final NodeManager nodeManager, final Configuration conf) {
-    Class<? extends ContainerPlacementPolicy> implClass =
-        (Class<? extends ContainerPlacementPolicy>) conf.getClass(
-            ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
-            SCMContainerPlacementRandom.class);
-
-    try {
-      Constructor<? extends ContainerPlacementPolicy> ctor =
-          implClass.getDeclaredConstructor(NodeManager.class,
-              Configuration.class);
-      return ctor.newInstance(nodeManager, conf);
-    } catch (RuntimeException e) {
-      throw e;
-    } catch (InvocationTargetException e) {
-      throw new RuntimeException(implClass.getName()
-          + " could not be constructed.", e.getCause());
-    } catch (Exception e) {
-      LOG.error("Unhandled exception occurred, Placement policy will not be " +
-          "functional.");
-      throw new IllegalArgumentException("Unable to load " +
-          "ContainerPlacementPolicy", e);
-    }
-  }
-
-  /**
-   * Return the pipeline manager from the replication type.
-   *
-   * @param replicationType - Replication Type Enum.
-   * @return pipeline Manager.
-   * @throws IllegalArgumentException If an pipeline type gets added
-   * and this function is not modified we will throw.
-   */
-  private PipelineManager getPipelineManager(ReplicationType replicationType)
-      throws IllegalArgumentException {
-    switch (replicationType) {
-    case RATIS:
-      return this.ratisManager;
-    case STAND_ALONE:
-      return this.standaloneManager;
-    case CHAINED:
-      throw new IllegalArgumentException("Not implemented yet");
-    default:
-      throw new IllegalArgumentException("Unexpected enum found. Does not" +
-          " know how to handle " + replicationType.toString());
-    }
-
-  }
-
-  /**
-   * This function is called by the Container Manager while allocating a new
-   * container. The client specifies what kind of replication pipeline is needed
-   * and based on the replication type in the request appropriate Interface is
-   * invoked.
-   */
-
-  public Pipeline getReplicationPipeline(ReplicationType replicationType,
-      OzoneProtos.ReplicationFactor replicationFactor, String containerName)
-      throws IOException {
-    PipelineManager manager = getPipelineManager(replicationType);
-    Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
-    LOG.debug("Getting replication pipeline for {} : Replication {}",
-        containerName, replicationFactor.toString());
-    return manager.
-        getPipeline(containerName, replicationFactor, replicationType);
-  }
-
-  /**
-   * Creates a pipeline from a specified set of Nodes.
-   */
-
-  public void createPipeline(ReplicationType replicationType, String
-      pipelineID, List<DatanodeID> datanodes) throws IOException {
-    PipelineManager manager = getPipelineManager(replicationType);
-    Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
-    LOG.debug("Creating a pipeline: {} with nodes:{}", pipelineID,
-        datanodes.stream().map(DatanodeID::toString)
-            .collect(Collectors.joining(",")));
-    manager.createPipeline(pipelineID, datanodes);
-  }
-
-  /**
-   * Close the  pipeline with the given clusterId.
-   */
-
-  public void closePipeline(ReplicationType replicationType, String
-      pipelineID) throws IOException {
-    PipelineManager manager = getPipelineManager(replicationType);
-    Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
-    LOG.debug("Closing pipeline. pipelineID: {}", pipelineID);
-    manager.closePipeline(pipelineID);
-  }
-
-  /**
-   * list members in the pipeline .
-   */
-
-  public List<DatanodeID> getDatanodes(ReplicationType replicationType,
-      String pipelineID) throws IOException {
-    PipelineManager manager = getPipelineManager(replicationType);
-    Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
-    LOG.debug("Getting data nodes from pipeline : {}", pipelineID);
-    return manager.getMembers(pipelineID);
-  }
-
-  /**
-   * Update the datanodes in the list of the pipeline.
-   */
-
-  public void updateDatanodes(ReplicationType replicationType, String
-      pipelineID, List<DatanodeID> newDatanodes) throws IOException {
-    PipelineManager manager = getPipelineManager(replicationType);
-    Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
-    LOG.debug("Updating pipeline: {} with new nodes:{}", pipelineID,
-        newDatanodes.stream().map(DatanodeID::toString)
-            .collect(Collectors.joining(",")));
-    manager.updatePipeline(pipelineID, newDatanodes);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/package-info.java
deleted file mode 100644
index c2a3b54..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/package-info.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.pipelines;
-/**
- Ozone supports the notion of different kind of pipelines.
- That means that we can have a replication pipeline build on
- Ratis, Standalone or some other protocol. All Pipeline managers
- the entities in charge of pipelines reside in the package.
-
- Here is the high level Arch.
-
- 1. A pipeline selector class is instantiated in the Container manager class.
-
- 2. A client when creating a container -- will specify what kind of
- replication type it wants to use. We support 2 types now, Ratis and StandAlone.
-
- 3. Based on the replication type, the pipeline selector class asks the
- corresponding pipeline manager for a pipeline.
-
- 4. We have supported the ability for clients to specify a set of nodes in
- the pipeline or rely in the pipeline manager to select the datanodes if they
- are not specified.
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/ratis/RatisManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/ratis/RatisManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/ratis/RatisManagerImpl.java
deleted file mode 100644
index 7903b0b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/ratis/RatisManagerImpl.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.pipelines.ratis;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState;
-import org.apache.hadoop.ozone.scm.container.placement.algorithms
-    .ContainerPlacementPolicy;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.scm.pipelines.PipelineManager;
-import org.apache.hadoop.ozone.scm.pipelines.PipelineSelector;
-import org.apache.hadoop.scm.XceiverClientRatis;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.container.common.helpers.PipelineChannel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-
-/**
- * Implementation of {@link PipelineManager}.
- *
- * TODO : Introduce a state machine.
- */
-public class RatisManagerImpl extends PipelineManager {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(RatisManagerImpl.class);
-  private static final String PREFIX = "Ratis-";
-  private final Configuration conf;
-  private final NodeManager nodeManager;
-  private final Set<DatanodeID> ratisMembers;
-
-  /**
-   * Constructs a Ratis Pipeline Manager.
-   *
-   * @param nodeManager
-   */
-  public RatisManagerImpl(NodeManager nodeManager,
-      ContainerPlacementPolicy placementPolicy, long size, Configuration conf) {
-    super();
-    this.conf = conf;
-    this.nodeManager = nodeManager;
-    ratisMembers = new HashSet<>();
-  }
-
-  /**
-   * Allocates a new ratis PipelineChannel from the free nodes.
-   *
-   * @param factor - One or Three
-   * @return PipelineChannel.
-   */
-  public PipelineChannel allocatePipelineChannel(ReplicationFactor factor) {
-    List<DatanodeID> newNodesList = new LinkedList<>();
-    List<DatanodeID> datanodes = nodeManager.getNodes(NodeState.HEALTHY);
-    int count = getReplicationCount(factor);
-    //TODO: Add Raft State to the Nodes, so we can query and skip nodes from
-    // data from datanode instead of maintaining a set.
-    for (DatanodeID datanode : datanodes) {
-      Preconditions.checkNotNull(datanode);
-      if (!ratisMembers.contains(datanode)) {
-        newNodesList.add(datanode);
-        if (newNodesList.size() == count) {
-          // once a datanode has been added to a pipeline, exclude it from
-          // further allocations
-          ratisMembers.addAll(newNodesList);
-          LOG.info("Allocating a new ratis pipelineChannel of size: {}", count);
-          // Start all channel names with "Ratis", easy to grep the logs.
-          String conduitName = PREFIX +
-              UUID.randomUUID().toString().substring(PREFIX.length());
-          PipelineChannel pipelineChannel =
-              PipelineSelector.newPipelineFromNodes(newNodesList,
-              LifeCycleState.OPEN, ReplicationType.RATIS, factor, conduitName);
-          Pipeline pipeline =
-              new Pipeline("setup", pipelineChannel);
-          try (XceiverClientRatis client =
-              XceiverClientRatis.newXceiverClientRatis(pipeline, conf)) {
-            client.createPipeline(pipeline.getPipelineName(), newNodesList);
-          } catch (IOException e) {
-            return null;
-          }
-          return pipelineChannel;
-        }
-      }
-    }
-    return null;
-  }
-
-  /**
-   * Creates a pipeline from a specified set of Nodes.
-   *
-   * @param pipelineID - Name of the pipeline
-   * @param datanodes - The list of datanodes that make this pipeline.
-   */
-  @Override
-  public void createPipeline(String pipelineID, List<DatanodeID> datanodes) {
-
-  }
-
-  /**
-   * Close the  pipeline with the given clusterId.
-   *
-   * @param pipelineID
-   */
-  @Override
-  public void closePipeline(String pipelineID) throws IOException {
-
-  }
-
-  /**
-   * list members in the pipeline .
-   *
-   * @param pipelineID
-   * @return the datanode
-   */
-  @Override
-  public List<DatanodeID> getMembers(String pipelineID) throws IOException {
-    return null;
-  }
-
-  /**
-   * Update the datanode list of the pipeline.
-   *
-   * @param pipelineID
-   * @param newDatanodes
-   */
-  @Override
-  public void updatePipeline(String pipelineID, List<DatanodeID> newDatanodes)
-      throws IOException {
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/ratis/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/ratis/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/ratis/package-info.java
deleted file mode 100644
index 6fe9b28..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/ratis/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.pipelines.ratis;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/standalone/StandaloneManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/standalone/StandaloneManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/standalone/StandaloneManagerImpl.java
deleted file mode 100644
index ef37926..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/standalone/StandaloneManagerImpl.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.pipelines.standalone;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState;
-import org.apache.hadoop.ozone.scm.container.placement.algorithms.ContainerPlacementPolicy;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.scm.pipelines.PipelineManager;
-import org.apache.hadoop.ozone.scm.pipelines.PipelineSelector;
-import org.apache.hadoop.scm.container.common.helpers.PipelineChannel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.UUID;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.LinkedList;
-
-/**
- * Standalone Manager Impl to prove that pluggable interface
- * works with current tests.
- */
-public class StandaloneManagerImpl extends PipelineManager {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(StandaloneManagerImpl.class);
-  private final NodeManager nodeManager;
-  private final ContainerPlacementPolicy placementPolicy;
-  private final long containerSize;
-  private final Set<DatanodeID> standAloneMembers;
-
-  /**
-   * Constructor for Standalone Node Manager Impl.
-   * @param nodeManager - Node Manager.
-   * @param placementPolicy - Placement Policy
-   * @param containerSize - Container Size.
-   */
-  public StandaloneManagerImpl(NodeManager nodeManager,
-      ContainerPlacementPolicy placementPolicy, long containerSize) {
-    super();
-    this.nodeManager = nodeManager;
-    this.placementPolicy = placementPolicy;
-    this.containerSize =  containerSize;
-    this.standAloneMembers = new HashSet<>();
-  }
-
-
-  /**
-   * Allocates a new standalone PipelineChannel from the free nodes.
-   *
-   * @param factor - One
-   * @return PipelineChannel.
-   */
-  public PipelineChannel allocatePipelineChannel(ReplicationFactor factor) {
-    List<DatanodeID> newNodesList = new LinkedList<>();
-    List<DatanodeID> datanodes = nodeManager.getNodes(NodeState.HEALTHY);
-    int count = getReplicationCount(factor);
-    for (DatanodeID datanode : datanodes) {
-      Preconditions.checkNotNull(datanode);
-      if (!standAloneMembers.contains(datanode)) {
-        newNodesList.add(datanode);
-        if (newNodesList.size() == count) {
-          // once a datanode has been added to a pipeline, exclude it from
-          // further allocations
-          standAloneMembers.addAll(newNodesList);
-          LOG.info("Allocating a new standalone pipeline channel of size: {}",
-              count);
-          String channelName =
-              "SA-" + UUID.randomUUID().toString().substring(3);
-          return PipelineSelector.newPipelineFromNodes(newNodesList,
-              LifeCycleState.OPEN, ReplicationType.STAND_ALONE,
-              ReplicationFactor.ONE, channelName);
-        }
-      }
-    }
-    return null;
-  }
-
-  /**
-   * Creates a pipeline from a specified set of Nodes.
-   *
-   * @param pipelineID - Name of the pipeline
-   * @param datanodes - The list of datanodes that make this pipeline.
-   */
-  @Override
-  public void createPipeline(String pipelineID, List<DatanodeID> datanodes) {
-    //return newPipelineFromNodes(datanodes, pipelineID);
-  }
-
-  /**
-   * Close the  pipeline with the given clusterId.
-   *
-   * @param pipelineID
-   */
-  @Override
-  public void closePipeline(String pipelineID) throws IOException {
-
-  }
-
-  /**
-   * list members in the pipeline .
-   *
-   * @param pipelineID
-   * @return the datanode
-   */
-  @Override
-  public List<DatanodeID> getMembers(String pipelineID) throws IOException {
-    return null;
-  }
-
-  /**
-   * Update the datanode list of the pipeline.
-   *
-   * @param pipelineID
-   * @param newDatanodes
-   */
-  @Override
-  public void updatePipeline(String pipelineID, List<DatanodeID>
-      newDatanodes) throws IOException {
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/standalone/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/standalone/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/standalone/package-info.java
deleted file mode 100644
index 7e6393a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/standalone/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.pipelines.standalone;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/ratis/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/ratis/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/ratis/package-info.java
deleted file mode 100644
index 27fd32b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/ratis/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.ratis;
-
-/**
- * This package contains classes related to Apache Ratis for SCM.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/Freon.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/Freon.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/Freon.java
deleted file mode 100644
index bec5d87..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/Freon.java
+++ /dev/null
@@ -1,1146 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.tools;
-
-import com.codahale.metrics.Histogram;
-import com.codahale.metrics.Snapshot;
-import com.codahale.metrics.UniformReservoir;
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.commons.lang.time.DurationFormatUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.*;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.util.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.Supplier;
-
-import static java.lang.Math.min;
-
-/**
- * Freon - A tool to populate ozone with data for testing.<br>
- * This is not a map-reduce program and this is not for benchmarking
- * Ozone write throughput.<br>
- * It supports both online and offline modes. Default mode is offline,
- * <i>-mode</i> can be used to change the mode.
- * <p>
- * In online mode, active internet connection is required,
- * common crawl data from AWS will be used.<br>
- * Default source is:<br>
- * https://commoncrawl.s3.amazonaws.com/crawl-data/
- * CC-MAIN-2017-17/warc.paths.gz<br>
- * (it contains the path to actual data segment)<br>
- * User can override this using <i>-source</i>.
- * The following values are derived from URL of Common Crawl data
- * <ul>
- * <li>Domain will be used as Volume</li>
- * <li>URL will be used as Bucket</li>
- * <li>FileName will be used as Key</li>
- * </ul></p>
- * In offline mode, the data will be random bytes and
- * size of data will be 10 KB.<br>
- * <ul>
- * <li>Default number of Volumes 10, <i>-numOfVolumes</i>
- * can be used to override</li>
- * <li>Default number of Buckets per Volume 1000, <i>-numOfBuckets</i>
- * can be used to override</li>
- * <li>Default number of Keys per Bucket 500000, <i>-numOfKeys</i>
- * can be used to override</li>
- * </ul>
- */
-public final class Freon extends Configured implements Tool {
-
-  enum FreonOps {
-    VOLUME_CREATE,
-    BUCKET_CREATE,
-    KEY_CREATE,
-    KEY_WRITE
-  }
-
-  private static final String HELP = "help";
-  private static final String MODE = "mode";
-  private static final String SOURCE = "source";
-  private static final String VALIDATE_WRITE = "validateWrites";
-  private static final String JSON_WRITE_DIRECTORY = "jsonDir";
-  private static final String NUM_OF_THREADS = "numOfThreads";
-  private static final String NUM_OF_VOLUMES = "numOfVolumes";
-  private static final String NUM_OF_BUCKETS = "numOfBuckets";
-  private static final String NUM_OF_KEYS = "numOfKeys";
-  private static final String KEY_SIZE = "keySize";
-  private static final String RATIS = "ratis";
-
-  private static final String MODE_DEFAULT = "offline";
-  private static final String SOURCE_DEFAULT =
-      "https://commoncrawl.s3.amazonaws.com/" +
-          "crawl-data/CC-MAIN-2017-17/warc.paths.gz";
-  private static final String NUM_OF_THREADS_DEFAULT = "10";
-  private static final String NUM_OF_VOLUMES_DEFAULT = "10";
-  private static final String NUM_OF_BUCKETS_DEFAULT = "1000";
-  private static final String NUM_OF_KEYS_DEFAULT = "500000";
-  private static final String DURATION_FORMAT = "HH:mm:ss,SSS";
-
-  private static final int KEY_SIZE_DEFAULT = 10240;
-  private static final int QUANTILES = 10;
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(Freon.class);
-
-  private boolean printUsage = false;
-  private boolean completed = false;
-  private boolean exception = false;
-
-  private String mode;
-  private String source;
-  private String numOfThreads;
-  private String numOfVolumes;
-  private String numOfBuckets;
-  private String numOfKeys;
-  private String jsonDir;
-  private boolean useRatis;
-  private ReplicationType type;
-  private ReplicationFactor factor;
-
-  private int threadPoolSize;
-  private int keySize;
-  private byte[] keyValue = null;
-
-  private boolean validateWrites;
-
-  private OzoneClient ozoneClient;
-  private ObjectStore objectStore;
-  private ExecutorService processor;
-
-  private long startTime;
-  private long jobStartTime;
-
-  private AtomicLong volumeCreationTime;
-  private AtomicLong bucketCreationTime;
-  private AtomicLong keyCreationTime;
-  private AtomicLong keyWriteTime;
-
-  private AtomicLong totalBytesWritten;
-
-  private AtomicInteger numberOfVolumesCreated;
-  private AtomicInteger numberOfBucketsCreated;
-  private AtomicLong numberOfKeysAdded;
-
-  private Long totalWritesValidated;
-  private Long writeValidationSuccessCount;
-  private Long writeValidationFailureCount;
-
-  private BlockingQueue<KeyValue> validationQueue;
-  private ArrayList<Histogram> histograms = new ArrayList<>();
-
-  @VisibleForTesting
-  Freon(Configuration conf) throws IOException {
-    startTime = System.nanoTime();
-    jobStartTime = System.currentTimeMillis();
-    volumeCreationTime = new AtomicLong();
-    bucketCreationTime = new AtomicLong();
-    keyCreationTime = new AtomicLong();
-    keyWriteTime = new AtomicLong();
-    totalBytesWritten = new AtomicLong();
-    numberOfVolumesCreated = new AtomicInteger();
-    numberOfBucketsCreated = new AtomicInteger();
-    numberOfKeysAdded = new AtomicLong();
-    ozoneClient = OzoneClientFactory.getClient(conf);
-    objectStore = ozoneClient.getObjectStore();
-    for (FreonOps ops : FreonOps.values()) {
-      histograms.add(ops.ordinal(), new Histogram(new UniformReservoir()));
-    }
-  }
-
-  /**
-   * @param args arguments
-   */
-  public static void main(String[] args) throws Exception {
-    Configuration conf = new OzoneConfiguration();
-    int res = ToolRunner.run(conf, new Freon(conf), args);
-    System.exit(res);
-  }
-
-  @Override
-  public int run(String[] args) throws Exception {
-    GenericOptionsParser parser = new GenericOptionsParser(getConf(),
-        getOptions(), args);
-    parseOptions(parser.getCommandLine());
-    if (printUsage) {
-      usage();
-      return 0;
-    }
-
-    keyValue =
-        DFSUtil.string2Bytes(RandomStringUtils.randomAscii(keySize - 36));
-
-    LOG.info("Number of Threads: " + numOfThreads);
-    threadPoolSize =
-        min(Integer.parseInt(numOfVolumes), Integer.parseInt(numOfThreads));
-    processor = Executors.newFixedThreadPool(threadPoolSize);
-    addShutdownHook();
-    if (mode.equals("online")) {
-      LOG.info("Mode: online");
-      throw new UnsupportedOperationException("Not yet implemented.");
-    } else {
-      LOG.info("Mode: offline");
-      LOG.info("Number of Volumes: {}.", numOfVolumes);
-      LOG.info("Number of Buckets per Volume: {}.", numOfBuckets);
-      LOG.info("Number of Keys per Bucket: {}.", numOfKeys);
-      LOG.info("Key size: {} bytes", keySize);
-      for (int i = 0; i < Integer.parseInt(numOfVolumes); i++) {
-        String volume = "vol-" + i + "-" +
-            RandomStringUtils.randomNumeric(5);
-        processor.submit(new OfflineProcessor(volume));
-      }
-    }
-    Thread validator = null;
-    if (validateWrites) {
-      totalWritesValidated = 0L;
-      writeValidationSuccessCount = 0L;
-      writeValidationFailureCount = 0L;
-
-      validationQueue =
-          new ArrayBlockingQueue<>(Integer.parseInt(numOfThreads));
-      validator = new Thread(new Validator());
-      validator.start();
-      LOG.info("Data validation is enabled.");
-    }
-    Thread progressbar = getProgressBarThread();
-    LOG.info("Starting progress bar Thread.");
-    progressbar.start();
-    processor.shutdown();
-    processor.awaitTermination(Integer.MAX_VALUE, TimeUnit.MILLISECONDS);
-    completed = true;
-    progressbar.join();
-    if (validateWrites) {
-      validator.join();
-    }
-    ozoneClient.close();
-    return 0;
-  }
-
-  private Options getOptions() {
-    Options options = new Options();
-
-    OptionBuilder.withDescription("prints usage.");
-    Option optHelp = OptionBuilder.create(HELP);
-
-    OptionBuilder.withArgName("online | offline");
-    OptionBuilder.hasArg();
-    OptionBuilder.withDescription("specifies the mode of " +
-        "Freon run.");
-    Option optMode = OptionBuilder.create(MODE);
-
-    OptionBuilder.withArgName("source url");
-    OptionBuilder.hasArg();
-    OptionBuilder.withDescription("specifies the URL of s3 " +
-        "commoncrawl warc file to be used when the mode is online.");
-    Option optSource = OptionBuilder.create(SOURCE);
-
-    OptionBuilder.withDescription("do random validation of " +
-        "data written into ozone, only subset of data is validated.");
-    Option optValidateWrite = OptionBuilder.create(VALIDATE_WRITE);
-
-
-    OptionBuilder.withDescription("directory where json is created");
-    OptionBuilder.hasArg();
-    Option optJsonDir = OptionBuilder.create(JSON_WRITE_DIRECTORY);
-
-    OptionBuilder.withArgName("value");
-    OptionBuilder.hasArg();
-    OptionBuilder.withDescription("number of threads to be launched " +
-        "for the run");
-    Option optNumOfThreads = OptionBuilder.create(NUM_OF_THREADS);
-
-    OptionBuilder.withArgName("value");
-    OptionBuilder.hasArg();
-    OptionBuilder.withDescription("specifies number of Volumes to be " +
-        "created in offline mode");
-    Option optNumOfVolumes = OptionBuilder.create(NUM_OF_VOLUMES);
-
-    OptionBuilder.withArgName("value");
-    OptionBuilder.hasArg();
-    OptionBuilder.withDescription("specifies number of Buckets to be " +
-        "created per Volume in offline mode");
-    Option optNumOfBuckets = OptionBuilder.create(NUM_OF_BUCKETS);
-
-    OptionBuilder.withArgName("value");
-    OptionBuilder.hasArg();
-    OptionBuilder.withDescription("specifies number of Keys to be " +
-        "created per Bucket in offline mode");
-    Option optNumOfKeys = OptionBuilder.create(NUM_OF_KEYS);
-
-    OptionBuilder.withArgName("value");
-    OptionBuilder.hasArg();
-    OptionBuilder.withDescription("specifies the size of Key in bytes to be " +
-        "created in offline mode");
-    Option optKeySize = OptionBuilder.create(KEY_SIZE);
-
-    OptionBuilder.withArgName(RATIS);
-    OptionBuilder.hasArg();
-    OptionBuilder.withDescription("Use Ratis as the default replication " +
-        "strategy");
-    Option optRatis = OptionBuilder.create(RATIS);
-
-    options.addOption(optHelp);
-    options.addOption(optMode);
-    options.addOption(optSource);
-    options.addOption(optValidateWrite);
-    options.addOption(optJsonDir);
-    options.addOption(optNumOfThreads);
-    options.addOption(optNumOfVolumes);
-    options.addOption(optNumOfBuckets);
-    options.addOption(optNumOfKeys);
-    options.addOption(optKeySize);
-    options.addOption(optRatis);
-    return options;
-  }
-
-  private void parseOptions(CommandLine cmdLine) {
-    printUsage = cmdLine.hasOption(HELP);
-
-    mode = cmdLine.getOptionValue(MODE, MODE_DEFAULT);
-
-    source = cmdLine.getOptionValue(SOURCE, SOURCE_DEFAULT);
-
-    numOfThreads =
-        cmdLine.getOptionValue(NUM_OF_THREADS, NUM_OF_THREADS_DEFAULT);
-
-    validateWrites = cmdLine.hasOption(VALIDATE_WRITE);
-
-    jsonDir = cmdLine.getOptionValue(JSON_WRITE_DIRECTORY);
-
-    numOfVolumes =
-        cmdLine.getOptionValue(NUM_OF_VOLUMES, NUM_OF_VOLUMES_DEFAULT);
-
-    numOfBuckets =
-        cmdLine.getOptionValue(NUM_OF_BUCKETS, NUM_OF_BUCKETS_DEFAULT);
-
-    numOfKeys = cmdLine.getOptionValue(NUM_OF_KEYS, NUM_OF_KEYS_DEFAULT);
-
-    keySize = cmdLine.hasOption(KEY_SIZE) ?
-        Integer.parseInt(cmdLine.getOptionValue(KEY_SIZE)) : KEY_SIZE_DEFAULT;
-    if (keySize < 1024) {
-      throw new IllegalArgumentException(
-          "keySize can not be less than 1024 bytes");
-    }
-
-    useRatis = cmdLine.hasOption(RATIS);
-
-    type = ReplicationType.STAND_ALONE;
-    factor = ReplicationFactor.ONE;
-
-    if (useRatis) {
-      type = ReplicationType.RATIS;
-      int replicationFactor = Integer.parseInt(cmdLine.getOptionValue(RATIS));
-      switch (replicationFactor) {
-      case 1:
-        factor = ReplicationFactor.ONE;
-        break;
-      case 3:
-        factor = ReplicationFactor.THREE;
-        break;
-      default:
-        throw new IllegalArgumentException("Illegal replication factor:"
-            + replicationFactor);
-      }
-    }
-  }
-
-  private void usage() {
-    System.out.println("Options supported are:");
-    System.out.println("-numOfThreads <value>           "
-        + "number of threads to be launched for the run.");
-    System.out.println("-validateWrites                 "
-        + "do random validation of data written into ozone, " +
-        "only subset of data is validated.");
-    System.out.println("-jsonDir                        "
-        + "directory where json is created.");
-    System.out.println("-mode [online | offline]        "
-        + "specifies the mode in which Freon should run.");
-    System.out.println("-source <url>                   "
-        + "specifies the URL of s3 commoncrawl warc file to " +
-        "be used when the mode is online.");
-    System.out.println("-numOfVolumes <value>           "
-        + "specifies number of Volumes to be created in offline mode");
-    System.out.println("-numOfBuckets <value>           "
-        + "specifies number of Buckets to be created per Volume " +
-        "in offline mode");
-    System.out.println("-numOfKeys <value>              "
-        + "specifies number of Keys to be created per Bucket " +
-        "in offline mode");
-    System.out.println("-keySize <value>                "
-        + "specifies the size of Key in bytes to be created in offline mode");
-    System.out.println("-help                           "
-        + "prints usage.");
-    System.out.println();
-  }
-
-  /**
-   * Adds ShutdownHook to print statistics.
-   */
-  private void addShutdownHook() {
-    Runtime.getRuntime().addShutdownHook(
-        new Thread(() -> printStats(System.out)));
-  }
-
-  private Thread getProgressBarThread() {
-    Supplier<Long> currentValue;
-    long maxValue;
-
-    if (mode.equals("online")) {
-      throw new UnsupportedOperationException("Not yet implemented.");
-    } else {
-      currentValue = () -> numberOfKeysAdded.get();
-      maxValue = Long.parseLong(numOfVolumes) *
-          Long.parseLong(numOfBuckets) *
-          Long.parseLong(numOfKeys);
-    }
-    Thread progressBarThread = new Thread(
-        new ProgressBar(System.out, currentValue, maxValue));
-    progressBarThread.setName("ProgressBar");
-    return progressBarThread;
-  }
-
-  /**
-   * Prints stats of {@link Freon} run to the PrintStream.
-   *
-   * @param out PrintStream
-   */
-  private void printStats(PrintStream out) {
-    long endTime = System.nanoTime() - startTime;
-    String execTime = DurationFormatUtils
-        .formatDuration(TimeUnit.NANOSECONDS.toMillis(endTime),
-            DURATION_FORMAT);
-
-    long volumeTime = TimeUnit.NANOSECONDS.toMillis(volumeCreationTime.get())
-        / threadPoolSize;
-    String prettyAverageVolumeTime =
-        DurationFormatUtils.formatDuration(volumeTime, DURATION_FORMAT);
-
-    long bucketTime = TimeUnit.NANOSECONDS.toMillis(bucketCreationTime.get())
-        / threadPoolSize;
-    String prettyAverageBucketTime =
-        DurationFormatUtils.formatDuration(bucketTime, DURATION_FORMAT);
-
-    long averageKeyCreationTime =
-        TimeUnit.NANOSECONDS.toMillis(keyCreationTime.get())
-            / threadPoolSize;
-    String prettyAverageKeyCreationTime = DurationFormatUtils
-        .formatDuration(averageKeyCreationTime, DURATION_FORMAT);
-
-    long averageKeyWriteTime =
-        TimeUnit.NANOSECONDS.toMillis(keyWriteTime.get()) / threadPoolSize;
-    String prettyAverageKeyWriteTime = DurationFormatUtils
-        .formatDuration(averageKeyWriteTime, DURATION_FORMAT);
-
-    out.println();
-    out.println("***************************************************");
-    out.println("Status: " + (exception ? "Failed" : "Success"));
-    out.println("Git Base Revision: " + VersionInfo.getRevision());
-    out.println("Number of Volumes created: " + numberOfVolumesCreated);
-    out.println("Number of Buckets created: " + numberOfBucketsCreated);
-    out.println("Number of Keys added: " + numberOfKeysAdded);
-    out.println("Ratis replication factor: " + factor.name());
-    out.println("Ratis replication type: " + type.name());
-    out.println(
-        "Average Time spent in volume creation: " + prettyAverageVolumeTime);
-    out.println(
-        "Average Time spent in bucket creation: " + prettyAverageBucketTime);
-    out.println(
-        "Average Time spent in key creation: " + prettyAverageKeyCreationTime);
-    out.println(
-        "Average Time spent in key write: " + prettyAverageKeyWriteTime);
-    out.println("Total bytes written: " + totalBytesWritten);
-    if (validateWrites) {
-      out.println("Total number of writes validated: " +
-          totalWritesValidated);
-      out.println("Writes validated: " +
-          (100.0 * totalWritesValidated / numberOfKeysAdded.get())
-          + " %");
-      out.println("Successful validation: " +
-          writeValidationSuccessCount);
-      out.println("Unsuccessful validation: " +
-          writeValidationFailureCount);
-    }
-    out.println("Total Execution time: " + execTime);
-    out.println("***************************************************");
-
-    if (jsonDir != null) {
-
-      String[][] quantileTime =
-          new String[FreonOps.values().length][QUANTILES + 1];
-      String[] deviations = new String[FreonOps.values().length];
-      String[] means = new String[FreonOps.values().length];
-      for (FreonOps ops : FreonOps.values()) {
-        Snapshot snapshot = histograms.get(ops.ordinal()).getSnapshot();
-        for (int i = 0; i <= QUANTILES; i++) {
-          quantileTime[ops.ordinal()][i] = DurationFormatUtils.formatDuration(
-              TimeUnit.NANOSECONDS
-                  .toMillis((long) snapshot.getValue((1.0 / QUANTILES) * i)),
-              DURATION_FORMAT);
-        }
-        deviations[ops.ordinal()] = DurationFormatUtils.formatDuration(
-            TimeUnit.NANOSECONDS.toMillis((long) snapshot.getStdDev()),
-            DURATION_FORMAT);
-        means[ops.ordinal()] = DurationFormatUtils.formatDuration(
-            TimeUnit.NANOSECONDS.toMillis((long) snapshot.getMean()),
-            DURATION_FORMAT);
-      }
-
-      FreonJobInfo jobInfo = new FreonJobInfo().setExecTime(execTime)
-          .setGitBaseRevision(VersionInfo.getRevision())
-          .setMeanVolumeCreateTime(means[FreonOps.VOLUME_CREATE.ordinal()])
-          .setDeviationVolumeCreateTime(
-              deviations[FreonOps.VOLUME_CREATE.ordinal()])
-          .setTenQuantileVolumeCreateTime(
-              quantileTime[FreonOps.VOLUME_CREATE.ordinal()])
-          .setMeanBucketCreateTime(means[FreonOps.BUCKET_CREATE.ordinal()])
-          .setDeviationBucketCreateTime(
-              deviations[FreonOps.BUCKET_CREATE.ordinal()])
-          .setTenQuantileBucketCreateTime(
-              quantileTime[FreonOps.BUCKET_CREATE.ordinal()])
-          .setMeanKeyCreateTime(means[FreonOps.KEY_CREATE.ordinal()])
-          .setDeviationKeyCreateTime(deviations[FreonOps.KEY_CREATE.ordinal()])
-          .setTenQuantileKeyCreateTime(
-              quantileTime[FreonOps.KEY_CREATE.ordinal()])
-          .setMeanKeyWriteTime(means[FreonOps.KEY_WRITE.ordinal()])
-          .setDeviationKeyWriteTime(deviations[FreonOps.KEY_WRITE.ordinal()])
-          .setTenQuantileKeyWriteTime(
-              quantileTime[FreonOps.KEY_WRITE.ordinal()]);
-      String jsonName =
-          new SimpleDateFormat("yyyyMMddHHmmss").format(Time.now()) + ".json";
-      String jsonPath = jsonDir + "/" + jsonName;
-      FileOutputStream os = null;
-      try {
-        os = new FileOutputStream(jsonPath);
-        ObjectMapper mapper = new ObjectMapper();
-        mapper.setVisibility(PropertyAccessor.FIELD,
-            JsonAutoDetect.Visibility.ANY);
-        ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter();
-        writer.writeValue(os, jobInfo);
-      } catch (FileNotFoundException e) {
-        out.println("Json File could not be created for the path: " + jsonPath);
-        out.println(e);
-      } catch (IOException e) {
-        out.println("Json object could not be created");
-        out.println(e);
-      } finally {
-        try {
-          if (os != null) {
-            os.close();
-          }
-        } catch (IOException e) {
-          LOG.warn("Could not close the output stream for json", e);
-        }
-      }
-    }
-  }
-
-  /**
-   * Returns the number of volumes created.
-   * @return volume count.
-   */
-  @VisibleForTesting
-  int getNumberOfVolumesCreated() {
-    return numberOfVolumesCreated.get();
-  }
-
-  /**
-   * Returns the number of buckets created.
-   * @return bucket count.
-   */
-  @VisibleForTesting
-  int getNumberOfBucketsCreated() {
-    return numberOfBucketsCreated.get();
-  }
-
-  /**
-   * Returns the number of keys added.
-   * @return keys count.
-   */
-  @VisibleForTesting
-  long getNumberOfKeysAdded() {
-    return numberOfKeysAdded.get();
-  }
-
-  /**
-   * Returns true if random validation of write is enabled.
-   * @return validateWrites
-   */
-  @VisibleForTesting
-  boolean getValidateWrites() {
-    return validateWrites;
-  }
-
-  /**
-   * Returns the number of keys validated.
-   * @return validated key count.
-   */
-  @VisibleForTesting
-  long getTotalKeysValidated() {
-    return totalWritesValidated;
-  }
-
-  /**
-   * Returns the number of successful validation.
-   * @return successful validation count.
-   */
-  @VisibleForTesting
-  long getSuccessfulValidationCount() {
-    return writeValidationSuccessCount;
-  }
-
-  /**
-   * Returns the number of unsuccessful validation.
-   * @return unsuccessful validation count.
-   */
-  @VisibleForTesting
-  long getUnsuccessfulValidationCount() {
-    return writeValidationFailureCount;
-  }
-
-  /**
-   * Returns the length of the common key value initialized.
-   * @return key value length initialized.
-   */
-  @VisibleForTesting
-  long getKeyValueLength(){
-    return keyValue.length;
-  }
-
-  /**
-   * Wrapper to hold ozone key-value pair.
-   */
-  private static class KeyValue {
-
-    /**
-     * Bucket name associated with the key-value.
-     */
-    private OzoneBucket bucket;
-    /**
-     * Key name associated with the key-value.
-     */
-    private String key;
-    /**
-     * Value associated with the key-value.
-     */
-    private byte[] value;
-
-    /**
-     * Constructs a new ozone key-value pair.
-     *
-     * @param key   key part
-     * @param value value part
-     */
-    KeyValue(OzoneBucket bucket, String key, byte[] value) {
-      this.bucket = bucket;
-      this.key = key;
-      this.value = value;
-    }
-  }
-
-  private class OfflineProcessor implements Runnable {
-
-    private int totalBuckets;
-    private int totalKeys;
-    private String volumeName;
-
-    OfflineProcessor(String volumeName) {
-      this.totalBuckets = Integer.parseInt(numOfBuckets);
-      this.totalKeys = Integer.parseInt(numOfKeys);
-      this.volumeName = volumeName;
-    }
-
-    @Override
-    public void run() {
-      LOG.trace("Creating volume: {}", volumeName);
-      long start = System.nanoTime();
-      OzoneVolume volume;
-      try {
-        objectStore.createVolume(volumeName);
-        long volumeCreationDuration = System.nanoTime() - start;
-        volumeCreationTime.getAndAdd(volumeCreationDuration);
-        histograms.get(FreonOps.VOLUME_CREATE.ordinal())
-            .update(volumeCreationDuration);
-        numberOfVolumesCreated.getAndIncrement();
-        volume = objectStore.getVolume(volumeName);
-      } catch (IOException e) {
-        exception = true;
-        LOG.error("Could not create volume", e);
-        return;
-      }
-
-      Long threadKeyWriteTime = 0L;
-      for (int j = 0; j < totalBuckets; j++) {
-        String bucketName = "bucket-" + j + "-" +
-            RandomStringUtils.randomNumeric(5);
-        try {
-          LOG.trace("Creating bucket: {} in volume: {}",
-              bucketName, volume.getName());
-          start = System.nanoTime();
-          volume.createBucket(bucketName);
-          long bucketCreationDuration = System.nanoTime() - start;
-          histograms.get(FreonOps.BUCKET_CREATE.ordinal())
-              .update(bucketCreationDuration);
-          bucketCreationTime.getAndAdd(bucketCreationDuration);
-          numberOfBucketsCreated.getAndIncrement();
-          OzoneBucket bucket = volume.getBucket(bucketName);
-          for (int k = 0; k < totalKeys; k++) {
-            String key = "key-" + k + "-" +
-                RandomStringUtils.randomNumeric(5);
-            byte[] randomValue =
-                DFSUtil.string2Bytes(UUID.randomUUID().toString());
-            try {
-              LOG.trace("Adding key: {} in bucket: {} of volume: {}",
-                  key, bucket, volume);
-              long keyCreateStart = System.nanoTime();
-              OzoneOutputStream os =
-                  bucket.createKey(key, keySize, type, factor);
-              long keyCreationDuration = System.nanoTime() - keyCreateStart;
-              histograms.get(FreonOps.KEY_CREATE.ordinal())
-                  .update(keyCreationDuration);
-              keyCreationTime.getAndAdd(keyCreationDuration);
-              long keyWriteStart = System.nanoTime();
-              os.write(keyValue);
-              os.write(randomValue);
-              os.close();
-              long keyWriteDuration = System.nanoTime() - keyWriteStart;
-              threadKeyWriteTime += keyWriteDuration;
-              histograms.get(FreonOps.KEY_WRITE.ordinal())
-                  .update(keyWriteDuration);
-              totalBytesWritten.getAndAdd(keySize);
-              numberOfKeysAdded.getAndIncrement();
-              if (validateWrites) {
-                byte[] value = ArrayUtils.addAll(keyValue, randomValue);
-                boolean validate = validationQueue.offer(
-                    new KeyValue(bucket, key, value));
-                if (validate) {
-                  LOG.trace("Key {}, is queued for validation.", key);
-                }
-              }
-            } catch (Exception e) {
-              exception = true;
-              LOG.error("Exception while adding key: {} in bucket: {}" +
-                  " of volume: {}.", key, bucket, volume, e);
-            }
-          }
-        } catch (Exception e) {
-          exception = true;
-          LOG.error("Exception while creating bucket: {}" +
-              " in volume: {}.", bucketName, volume, e);
-        }
-      }
-
-      keyWriteTime.getAndAdd(threadKeyWriteTime);
-    }
-
-  }
-
-  private final class FreonJobInfo {
-
-    private String status;
-    private String gitBaseRevision;
-    private String jobStartTime;
-    private String numOfVolumes;
-    private String numOfBuckets;
-    private String numOfKeys;
-    private String numOfThreads;
-    private String mode;
-    private String dataWritten;
-    private String execTime;
-    private String replicationFactor;
-    private String replicationType;
-
-    private int keySize;
-
-    private String totalThroughputPerSecond;
-
-    private String meanVolumeCreateTime;
-    private String deviationVolumeCreateTime;
-    private String[] tenQuantileVolumeCreateTime;
-
-    private String meanBucketCreateTime;
-    private String deviationBucketCreateTime;
-    private String[] tenQuantileBucketCreateTime;
-
-    private String meanKeyCreateTime;
-    private String deviationKeyCreateTime;
-    private String[] tenQuantileKeyCreateTime;
-
-    private String meanKeyWriteTime;
-    private String deviationKeyWriteTime;
-    private String[] tenQuantileKeyWriteTime;
-
-    private FreonJobInfo() {
-      this.status = exception ? "Failed" : "Success";
-      this.numOfVolumes = Freon.this.numOfVolumes;
-      this.numOfBuckets = Freon.this.numOfBuckets;
-      this.numOfKeys = Freon.this.numOfKeys;
-      this.numOfThreads = Freon.this.numOfThreads;
-      this.keySize = Freon.this.keySize;
-      this.mode = Freon.this.mode;
-      this.jobStartTime = Time.formatTime(Freon.this.jobStartTime);
-      this.replicationFactor = Freon.this.factor.name();
-      this.replicationType = Freon.this.type.name();
-
-      long totalBytes =
-          Long.parseLong(numOfVolumes) * Long.parseLong(numOfBuckets) * Long
-              .parseLong(numOfKeys) * keySize;
-      this.dataWritten = getInStorageUnits((double) totalBytes);
-      this.totalThroughputPerSecond = getInStorageUnits(
-          (totalBytes * 1.0) / TimeUnit.NANOSECONDS
-              .toSeconds(Freon.this.keyWriteTime.get() / threadPoolSize));
-    }
-
-    private String getInStorageUnits(Double value) {
-      double size;
-      OzoneQuota.Units unit;
-      if ((long) (value / OzoneConsts.TB) != 0) {
-        size = value / OzoneConsts.TB;
-        unit = OzoneQuota.Units.TB;
-      } else if ((long) (value / OzoneConsts.GB) != 0) {
-        size = value / OzoneConsts.GB;
-        unit = OzoneQuota.Units.GB;
-      } else if ((long) (value / OzoneConsts.MB) != 0) {
-        size = value / OzoneConsts.MB;
-        unit = OzoneQuota.Units.MB;
-      } else if ((long) (value / OzoneConsts.KB) != 0) {
-        size = value / OzoneConsts.KB;
-        unit = OzoneQuota.Units.KB;
-      } else {
-        size = value;
-        unit = OzoneQuota.Units.BYTES;
-      }
-      return size + " " + unit;
-    }
-
-    public FreonJobInfo setGitBaseRevision(String gitBaseRevisionVal) {
-      gitBaseRevision = gitBaseRevisionVal;
-      return this;
-    }
-
-    public FreonJobInfo setExecTime(String execTimeVal) {
-      execTime = execTimeVal;
-      return this;
-    }
-
-    public FreonJobInfo setMeanKeyWriteTime(String deviationKeyWriteTimeVal) {
-      this.meanKeyWriteTime = deviationKeyWriteTimeVal;
-      return this;
-    }
-
-    public FreonJobInfo setDeviationKeyWriteTime(
-        String deviationKeyWriteTimeVal) {
-      this.deviationKeyWriteTime = deviationKeyWriteTimeVal;
-      return this;
-    }
-
-    public FreonJobInfo setTenQuantileKeyWriteTime(
-        String[] tenQuantileKeyWriteTimeVal) {
-      this.tenQuantileKeyWriteTime = tenQuantileKeyWriteTimeVal;
-      return this;
-    }
-
-    public FreonJobInfo setMeanKeyCreateTime(String deviationKeyWriteTimeVal) {
-      this.meanKeyCreateTime = deviationKeyWriteTimeVal;
-      return this;
-    }
-
-    public FreonJobInfo setDeviationKeyCreateTime(
-        String deviationKeyCreateTimeVal) {
-      this.deviationKeyCreateTime = deviationKeyCreateTimeVal;
-      return this;
-    }
-
-    public FreonJobInfo setTenQuantileKeyCreateTime(
-        String[] tenQuantileKeyCreateTimeVal) {
-      this.tenQuantileKeyCreateTime = tenQuantileKeyCreateTimeVal;
-      return this;
-    }
-
-    public FreonJobInfo setMeanBucketCreateTime(
-        String deviationKeyWriteTimeVal) {
-      this.meanBucketCreateTime = deviationKeyWriteTimeVal;
-      return this;
-    }
-
-    public FreonJobInfo setDeviationBucketCreateTime(
-        String deviationBucketCreateTimeVal) {
-      this.deviationBucketCreateTime = deviationBucketCreateTimeVal;
-      return this;
-    }
-
-    public FreonJobInfo setTenQuantileBucketCreateTime(
-        String[] tenQuantileBucketCreateTimeVal) {
-      this.tenQuantileBucketCreateTime = tenQuantileBucketCreateTimeVal;
-      return this;
-    }
-
-    public FreonJobInfo setMeanVolumeCreateTime(
-        String deviationKeyWriteTimeVal) {
-      this.meanVolumeCreateTime = deviationKeyWriteTimeVal;
-      return this;
-    }
-
-    public FreonJobInfo setDeviationVolumeCreateTime(
-        String deviationVolumeCreateTimeVal) {
-      this.deviationVolumeCreateTime = deviationVolumeCreateTimeVal;
-      return this;
-    }
-
-    public FreonJobInfo setTenQuantileVolumeCreateTime(
-        String[] tenQuantileVolumeCreateTimeVal) {
-      this.tenQuantileVolumeCreateTime = tenQuantileVolumeCreateTimeVal;
-      return this;
-    }
-
-    public String getJobStartTime() {
-      return jobStartTime;
-    }
-
-    public String getNumOfVolumes() {
-      return numOfVolumes;
-    }
-
-    public String getNumOfBuckets() {
-      return numOfBuckets;
-    }
-
-    public String getNumOfKeys() {
-      return numOfKeys;
-    }
-
-    public String getNumOfThreads() {
-      return numOfThreads;
-    }
-
-    public String getMode() {
-      return mode;
-    }
-
-    public String getExecTime() {
-      return execTime;
-    }
-
-    public String getReplicationFactor() {
-      return replicationFactor;
-    }
-
-    public String getReplicationType() {
-      return replicationType;
-    }
-
-    public String getStatus() {
-      return status;
-    }
-
-    public int getKeySize() {
-      return keySize;
-    }
-
-    public String getGitBaseRevision() {
-      return gitBaseRevision;
-    }
-
-    public String getDataWritten() {
-      return dataWritten;
-    }
-
-    public String getTotalThroughputPerSecond() {
-      return totalThroughputPerSecond;
-    }
-
-    public String getMeanVolumeCreateTime() {
-      return meanVolumeCreateTime;
-    }
-
-    public String getDeviationVolumeCreateTime() {
-      return deviationVolumeCreateTime;
-    }
-
-    public String[] getTenQuantileVolumeCreateTime() {
-      return tenQuantileVolumeCreateTime;
-    }
-
-    public String getMeanBucketCreateTime() {
-      return meanBucketCreateTime;
-    }
-
-    public String getDeviationBucketCreateTime() {
-      return deviationBucketCreateTime;
-    }
-
-    public String[] getTenQuantileBucketCreateTime() {
-      return tenQuantileBucketCreateTime;
-    }
-
-    public String getMeanKeyCreateTime() {
-      return meanKeyCreateTime;
-    }
-
-    public String getDeviationKeyCreateTime() {
-      return deviationKeyCreateTime;
-    }
-
-    public String[] getTenQuantileKeyCreateTime() {
-      return tenQuantileKeyCreateTime;
-    }
-
-    public String getMeanKeyWriteTime() {
-      return meanKeyWriteTime;
-    }
-
-    public String getDeviationKeyWriteTime() {
-      return deviationKeyWriteTime;
-    }
-
-    public String[] getTenQuantileKeyWriteTime() {
-      return tenQuantileKeyWriteTime;
-    }
-  }
-
-  private class ProgressBar implements Runnable {
-
-    private static final long REFRESH_INTERVAL = 1000L;
-
-    private PrintStream stream;
-    private Supplier<Long> currentValue;
-    private long maxValue;
-
-    ProgressBar(PrintStream stream, Supplier<Long> currentValue,
-        long maxValue) {
-      this.stream = stream;
-      this.currentValue = currentValue;
-      this.maxValue = maxValue;
-    }
-
-    @Override
-    public void run() {
-      try {
-        stream.println();
-        long value;
-        while ((value = currentValue.get()) < maxValue) {
-          print(value);
-          if (completed) {
-            break;
-          }
-          Thread.sleep(REFRESH_INTERVAL);
-        }
-        if (exception) {
-          stream.println();
-          stream.println("Incomplete termination, " +
-              "check log for exception.");
-        } else {
-          print(maxValue);
-        }
-        stream.println();
-      } catch (InterruptedException e) {
-      }
-    }
-
-    /**
-     * Given current value prints the progress bar.
-     *
-     * @param value
-     */
-    private void print(long value) {
-      stream.print('\r');
-      double percent = 100.0 * value / maxValue;
-      StringBuilder sb = new StringBuilder();
-      sb.append(" " + String.format("%.2f", percent) + "% |");
-
-      for (int i = 0; i <= percent; i++) {
-        sb.append('█');
-      }
-      for (int j = 0; j < 100 - percent; j++) {
-        sb.append(' ');
-      }
-      sb.append("|  ");
-      sb.append(value + "/" + maxValue);
-      long timeInSec = TimeUnit.SECONDS.convert(
-          System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
-      String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
-          (timeInSec % 3600) / 60, timeInSec % 60);
-      sb.append(" Time: " + timeToPrint);
-      stream.print(sb);
-    }
-  }
-
-  /**
-   * Validates the write done in ozone cluster.
-   */
-  private class Validator implements Runnable {
-
-    @Override
-    public void run() {
-      while (!completed) {
-        try {
-          KeyValue kv = validationQueue.poll(5, TimeUnit.SECONDS);
-          if (kv != null) {
-
-            OzoneInputStream is = kv.bucket.readKey(kv.key);
-            byte[] value = new byte[kv.value.length];
-            int length = is.read(value);
-            totalWritesValidated++;
-            if (length == kv.value.length && Arrays.equals(value, kv.value)) {
-              writeValidationSuccessCount++;
-            } else {
-              writeValidationFailureCount++;
-              LOG.warn("Data validation error for key {}/{}/{}",
-                  kv.bucket.getVolumeName(), kv.bucket, kv.key);
-              LOG.warn("Expected: {}, Actual: {}",
-                  DFSUtil.bytes2String(kv.value),
-                  DFSUtil.bytes2String(value));
-            }
-          }
-        } catch (IOException | InterruptedException ex) {
-          LOG.error("Exception while validating write: " + ex.getMessage());
-        }
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/package-info.java
deleted file mode 100644
index 4d74f48..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.tools;
-
-/**
- This package contains class used for testing and benchmarking ozone cluster.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java
deleted file mode 100644
index 7dd9a33..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web;
-
-import org.apache.hadoop.ozone.client.rest.OzoneExceptionMapper;
-import org.apache.hadoop.ozone.web.handlers.BucketHandler;
-import org.apache.hadoop.ozone.web.handlers.KeyHandler;
-import org.apache.hadoop.ozone.web.handlers.ServiceFilter;
-import org.apache.hadoop.ozone.web.handlers.VolumeHandler;
-import org.apache.hadoop.ozone.web.messages.LengthInputStreamMessageBodyWriter;
-import org.apache.hadoop.ozone.web.messages.StringMessageBodyWriter;
-
-import javax.ws.rs.core.Application;
-import java.util.HashSet;
-import java.util.Set;
-
-/**
- * Ozone Application.
- */
-public class ObjectStoreApplication extends Application {
-  public ObjectStoreApplication() {
-    super();
-  }
-
-  @Override
-  public Set<Class<?>> getClasses() {
-    HashSet<Class<?>> set = new HashSet<>();
-    set.add(BucketHandler.class);
-    set.add(VolumeHandler.class);
-    set.add(KeyHandler.class);
-    set.add(OzoneExceptionMapper.class);
-    set.add(LengthInputStreamMessageBodyWriter.class);
-    set.add(StringMessageBodyWriter.class);
-    return set;
-  }
-
-  @Override
-  public Set<Object> getSingletons() {
-    HashSet<Object> set = new HashSet<>();
-    set.add(ServiceFilter.class);
-    return set;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/OzoneHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/OzoneHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/OzoneHttpServer.java
deleted file mode 100644
index b6a8fda..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/OzoneHttpServer.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.web;
-
-import com.google.common.base.Optional;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.http.HttpServer2;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.eclipse.jetty.webapp.WebAppContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServlet;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-/**
- * Base class for HTTP server of the Ozone related components.
- */
-public abstract class OzoneHttpServer {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(OzoneHttpServer.class);
-
-  private HttpServer2 httpServer;
-  private final Configuration conf;
-
-  private InetSocketAddress httpAddress;
-  private InetSocketAddress httpsAddress;
-
-  private HttpConfig.Policy policy;
-
-  private String name;
-
-  public OzoneHttpServer(Configuration conf, String name) throws IOException {
-    this.name = name;
-    this.conf = conf;
-    if (isEnabled()) {
-      policy = DFSUtil.getHttpPolicy(conf);
-      if (policy.isHttpEnabled()) {
-        this.httpAddress = getHttpBindAddress();
-      }
-      if (policy.isHttpsEnabled()) {
-        this.httpsAddress = getHttpsBindAddress();
-      }
-      HttpServer2.Builder builder = null;
-      builder = DFSUtil.httpServerTemplateForNNAndJN(conf, this.httpAddress,
-          this.httpsAddress, name, getSpnegoPrincipal(), getKeytabFile());
-
-      final boolean xFrameEnabled = conf.getBoolean(
-          DFSConfigKeys.DFS_XFRAME_OPTION_ENABLED,
-          DFSConfigKeys.DFS_XFRAME_OPTION_ENABLED_DEFAULT);
-
-      final String xFrameOptionValue = conf.getTrimmed(
-          DFSConfigKeys.DFS_XFRAME_OPTION_VALUE,
-          DFSConfigKeys.DFS_XFRAME_OPTION_VALUE_DEFAULT);
-
-      builder.configureXFrame(xFrameEnabled).setXFrameOption(xFrameOptionValue);
-
-      httpServer = builder.build();
-
-    }
-
-  }
-
-  /**
-   * Add a servlet to OzoneHttpServer.
-   * @param servletName The name of the servlet
-   * @param pathSpec The path spec for the servlet
-   * @param clazz The servlet class
-   */
-  protected void addServlet(String servletName, String pathSpec,
-                            Class<? extends HttpServlet> clazz) {
-    httpServer.addServlet(servletName, pathSpec, clazz);
-  }
-
-  /**
-   * Returns the WebAppContext associated with this HttpServer.
-   * @return WebAppContext
-   */
-  protected WebAppContext getWebAppContext() {
-    return httpServer.getWebAppContext();
-  }
-
-  protected InetSocketAddress getBindAddress(String bindHostKey,
-      String addressKey, String bindHostDefault, int bindPortdefault) {
-    final Optional<String> bindHost =
-        OzoneClientUtils.getHostNameFromConfigKeys(conf, bindHostKey);
-
-    final Optional<Integer> addressPort =
-        OzoneClientUtils.getPortNumberFromConfigKeys(conf, addressKey);
-
-    final Optional<String> addresHost =
-        OzoneClientUtils.getHostNameFromConfigKeys(conf, addressKey);
-
-    String hostName = bindHost.or(addresHost).or(bindHostDefault);
-
-    return NetUtils.createSocketAddr(
-        hostName + ":" + addressPort.or(bindPortdefault));
-  }
-
-  /**
-   * Retrieve the socket address that should be used by clients to connect
-   * to the  HTTPS web interface.
-   *
-   * @return Target InetSocketAddress for the Ozone HTTPS endpoint.
-   */
-  public InetSocketAddress getHttpsBindAddress() {
-    return getBindAddress(getHttpsBindHostKey(), getHttpsAddressKey(),
-        getBindHostDefault(), getHttpsBindPortDefault());
-  }
-
-  /**
-   * Retrieve the socket address that should be used by clients to connect
-   * to the  HTTP web interface.
-   * <p>
-   * * @return Target InetSocketAddress for the Ozone HTTP endpoint.
-   */
-  public InetSocketAddress getHttpBindAddress() {
-    return getBindAddress(getHttpBindHostKey(), getHttpAddressKey(),
-        getBindHostDefault(), getHttpBindPortDefault());
-
-  }
-
-  public void start() throws IOException {
-    if (httpServer != null && isEnabled()) {
-      httpServer.start();
-      updateConnectorAddress();
-    }
-
-  }
-
-  private boolean isEnabled() {
-    return conf.getBoolean(getEnabledKey(), true);
-  }
-
-  public void stop() throws Exception {
-    if (httpServer != null) {
-      httpServer.stop();
-    }
-  }
-
-  /**
-   * Update the configured listen address based on the real port
-   * <p>
-   * (eg. replace :0 with real port)
-   */
-  public void updateConnectorAddress() {
-    int connIdx = 0;
-    if (policy.isHttpEnabled()) {
-      httpAddress = httpServer.getConnectorAddress(connIdx++);
-      String realAddress = NetUtils.getHostPortString(httpAddress);
-      conf.set(getHttpAddressKey(), realAddress);
-      LOG.info(
-          String.format("HTTP server of %s is listening at http://%s",
-              name.toUpperCase(), realAddress));
-    }
-
-    if (policy.isHttpsEnabled()) {
-      httpsAddress = httpServer.getConnectorAddress(connIdx);
-      String realAddress = NetUtils.getHostPortString(httpsAddress);
-      conf.set(getHttpsAddressKey(), realAddress);
-      LOG.info(
-          String.format("HTTP server of %s is listening at https://%s",
-              name.toUpperCase(), realAddress));
-    }
-  }
-
-  public InetSocketAddress getHttpAddress() {
-    return httpAddress;
-  }
-
-  public InetSocketAddress getHttpsAddress() {
-    return httpsAddress;
-  }
-
-  protected abstract String getHttpAddressKey();
-
-  protected abstract String getHttpsAddressKey();
-
-  protected abstract String getHttpBindHostKey();
-
-  protected abstract String getHttpsBindHostKey();
-
-  protected abstract String getBindHostDefault();
-
-  protected abstract int getHttpBindPortDefault();
-
-  protected abstract int getHttpsBindPortDefault();
-
-  protected abstract String getKeytabFile();
-
-  protected abstract String getSpnegoPrincipal();
-
-  protected abstract String getEnabledKey();
-
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[05/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.js
deleted file mode 100644
index 9cfd702..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.js
+++ /dev/null
@@ -1,11 +0,0 @@
-/* nvd3 version 1.8.5 (https://github.com/novus/nvd3) 2016-12-01 */
-
-!function(){var a={};a.dev=!1,a.tooltip=a.tooltip||{},a.utils=a.utils||{},a.models=a.models||{},a.charts={},a.logs={},a.dom={},"undefined"!=typeof module&&"undefined"!=typeof exports&&"undefined"==typeof d3&&(d3=require("d3")),a.dispatch=d3.dispatch("render_start","render_end"),Function.prototype.bind||(Function.prototype.bind=function(a){if("function"!=typeof this)throw new TypeError("Function.prototype.bind - what is trying to be bound is not callable");var b=Array.prototype.slice.call(arguments,1),c=this,d=function(){},e=function(){return c.apply(this instanceof d&&a?this:a,b.concat(Array.prototype.slice.call(arguments)))};return d.prototype=this.prototype,e.prototype=new d,e}),a.dev&&(a.dispatch.on("render_start",function(b){a.logs.startTime=+new Date}),a.dispatch.on("render_end",function(b){a.logs.endTime=+new Date,a.logs.totalTime=a.logs.endTime-a.logs.startTime,a.log("total",a.logs.totalTime)})),a.log=function(){if(a.dev&&window.console&&console.log&&console.log.apply)console
 .log.apply(console,arguments);else if(a.dev&&window.console&&"function"==typeof console.log&&Function.prototype.bind){var b=Function.prototype.bind.call(console.log,console);b.apply(console,arguments)}return arguments[arguments.length-1]},a.deprecated=function(a,b){console&&console.warn&&console.warn("nvd3 warning: `"+a+"` has been deprecated. ",b||"")},a.render=function(b){b=b||1,a.render.active=!0,a.dispatch.render_start();var c=function(){for(var d,e,f=0;b>f&&(e=a.render.queue[f]);f++)d=e.generate(),typeof e.callback==typeof Function&&e.callback(d);a.render.queue.splice(0,f),a.render.queue.length?setTimeout(c):(a.dispatch.render_end(),a.render.active=!1)};setTimeout(c)},a.render.active=!1,a.render.queue=[],a.addGraph=function(b){typeof arguments[0]==typeof Function&&(b={generate:arguments[0],callback:arguments[1]}),a.render.queue.push(b),a.render.active||a.render()},"undefined"!=typeof module&&"undefined"!=typeof exports&&(module.exports=a),"undefined"!=typeof window&&(window.nv=
 a),a.dom.write=function(a){return void 0!==window.fastdom?fastdom.mutate(a):a()},a.dom.read=function(a){return void 0!==window.fastdom?fastdom.measure(a):a()},a.interactiveGuideline=function(){"use strict";function b(l){l.each(function(l){function m(){var a=d3.mouse(this),d=a[0],e=a[1],h=!0,i=!1;if(k&&(d=d3.event.offsetX,e=d3.event.offsetY,"svg"!==d3.event.target.tagName&&(h=!1),d3.event.target.className.baseVal.match("nv-legend")&&(i=!0)),h&&(d-=c.left,e-=c.top),"mouseout"===d3.event.type||0>d||0>e||d>o||e>p||d3.event.relatedTarget&&void 0===d3.event.relatedTarget.ownerSVGElement||i){if(k&&d3.event.relatedTarget&&void 0===d3.event.relatedTarget.ownerSVGElement&&(void 0===d3.event.relatedTarget.className||d3.event.relatedTarget.className.match(j.nvPointerEventsClass)))return;return g.elementMouseout({mouseX:d,mouseY:e}),b.renderGuideLine(null),void j.hidden(!0)}j.hidden(!1);var l="function"==typeof f.rangeBands,m=void 0;if(l){var n=d3.bisect(f.range(),d)-1;if(!(f.range()[n]+f.rangeB
 and()>=d))return g.elementMouseout({mouseX:d,mouseY:e}),b.renderGuideLine(null),void j.hidden(!0);m=f.domain()[d3.bisect(f.range(),d)-1]}else m=f.invert(d);g.elementMousemove({mouseX:d,mouseY:e,pointXValue:m}),"dblclick"===d3.event.type&&g.elementDblclick({mouseX:d,mouseY:e,pointXValue:m}),"click"===d3.event.type&&g.elementClick({mouseX:d,mouseY:e,pointXValue:m}),"mousedown"===d3.event.type&&g.elementMouseDown({mouseX:d,mouseY:e,pointXValue:m}),"mouseup"===d3.event.type&&g.elementMouseUp({mouseX:d,mouseY:e,pointXValue:m})}var n=d3.select(this),o=d||960,p=e||400,q=n.selectAll("g.nv-wrap.nv-interactiveLineLayer").data([l]),r=q.enter().append("g").attr("class"," nv-wrap nv-interactiveLineLayer");r.append("g").attr("class","nv-interactiveGuideLine"),i&&(i.on("touchmove",m).on("mousemove",m,!0).on("mouseout",m,!0).on("mousedown",m,!0).on("mouseup",m,!0).on("dblclick",m).on("click",m),b.guideLine=null,b.renderGuideLine=function(c){h&&(b.guideLine&&b.guideLine.attr("x1")===c||a.dom.write(f
 unction(){var b=q.select(".nv-interactiveGuideLine").selectAll("line").data(null!=c?[a.utils.NaNtoZero(c)]:[],String);b.enter().append("line").attr("class","nv-guideline").attr("x1",function(a){return a}).attr("x2",function(a){return a}).attr("y1",p).attr("y2",0),b.exit().remove()}))})})}var c={left:0,top:0},d=null,e=null,f=d3.scale.linear(),g=d3.dispatch("elementMousemove","elementMouseout","elementClick","elementDblclick","elementMouseDown","elementMouseUp"),h=!0,i=null,j=a.models.tooltip(),k=window.ActiveXObject;return j.duration(0).hideDelay(0).hidden(!1),b.dispatch=g,b.tooltip=j,b.margin=function(a){return arguments.length?(c.top="undefined"!=typeof a.top?a.top:c.top,c.left="undefined"!=typeof a.left?a.left:c.left,b):c},b.width=function(a){return arguments.length?(d=a,b):d},b.height=function(a){return arguments.length?(e=a,b):e},b.xScale=function(a){return arguments.length?(f=a,b):f},b.showGuideLine=function(a){return arguments.length?(h=a,b):h},b.svgContainer=function(a){retur
 n arguments.length?(i=a,b):i},b},a.interactiveBisect=function(a,b,c){"use strict";if(!(a instanceof Array))return null;var d;d="function"!=typeof c?function(a){return a.x}:c;var e=function(a,b){return d(a)-b},f=d3.bisector(e).left,g=d3.max([0,f(a,b)-1]),h=d(a[g]);if("undefined"==typeof h&&(h=g),h===b)return g;var i=d3.min([g+1,a.length-1]),j=d(a[i]);return"undefined"==typeof j&&(j=i),Math.abs(j-b)>=Math.abs(h-b)?g:i},a.nearestValueIndex=function(a,b,c){"use strict";var d=1/0,e=null;return a.forEach(function(a,f){var g=Math.abs(b-a);null!=a&&d>=g&&c>g&&(d=g,e=f)}),e},a.models.tooltip=function(){"use strict";function b(){if(!l||!l.node()){var a=[1];l=d3.select(document.body).select("#"+d).data(a),l.enter().append("div").attr("class","nvtooltip "+(i?i:"xy-tooltip")).attr("id",d).style("top",0).style("left",0).style("opacity",0).style("position","fixed").selectAll("div, table, td, tr").classed(q,!0).classed(q,!0),l.exit().remove()}}function c(){return n&&w(e)?(a.dom.write(function(){b()
 ;var a=u(e);a&&(l.node().innerHTML=a),y()}),c):void 0}var d="nvtooltip-"+Math.floor(1e5*Math.random()),e=null,f="w",g=25,h=0,i=null,j=!0,k=200,l=null,m={left:null,top:null},n=!0,o=100,p=!0,q="nv-pointer-events-none",r=function(a,b){return a},s=function(a){return a},t=function(a,b){return a},u=function(a){if(null===a)return"";var b=d3.select(document.createElement("table"));if(p){var c=b.selectAll("thead").data([a]).enter().append("thead");c.append("tr").append("td").attr("colspan",3).append("strong").classed("x-value",!0).html(s(a.value))}var d=b.selectAll("tbody").data([a]).enter().append("tbody"),e=d.selectAll("tr").data(function(a){return a.series}).enter().append("tr").classed("highlight",function(a){return a.highlight});e.append("td").classed("legend-color-guide",!0).append("div").style("background-color",function(a){return a.color}),e.append("td").classed("key",!0).classed("total",function(a){return!!a.total}).html(function(a,b){return t(a.key,b)}),e.append("td").classed("valu
 e",!0).html(function(a,b){return r(a.value,b)}),e.filter(function(a,b){return void 0!==a.percent}).append("td").classed("percent",!0).html(function(a,b){return"("+d3.format("%")(a.percent)+")"}),e.selectAll("td").each(function(a){if(a.highlight){var b=d3.scale.linear().domain([0,1]).range(["#fff",a.color]),c=.6;d3.select(this).style("border-bottom-color",b(c)).style("border-top-color",b(c))}});var f=b.node().outerHTML;return void 0!==a.footer&&(f+="<div class='footer'>"+a.footer+"</div>"),f},v=function(){var a={left:null!==d3.event?d3.event.clientX:0,top:null!==d3.event?d3.event.clientY:0};if("none"!=getComputedStyle(document.body).transform){var b=document.body.getBoundingClientRect();a.left-=b.left,a.top-=b.top}return a},w=function(b){if(b&&b.series){if(a.utils.isArray(b.series))return!0;if(a.utils.isObject(b.series))return b.series=[b.series],!0}return!1},x=function(a){var b,c,d,e=l.node().offsetHeight,h=l.node().offsetWidth,i=document.documentElement.clientWidth,j=document.docum
 entElement.clientHeight;switch(f){case"e":b=-h-g,c=-(e/2),a.left+b<0&&(b=g),(d=a.top+c)<0&&(c-=d),(d=a.top+c+e)>j&&(c-=d-j);break;case"w":b=g,c=-(e/2),a.left+b+h>i&&(b=-h-g),(d=a.top+c)<0&&(c-=d),(d=a.top+c+e)>j&&(c-=d-j);break;case"n":b=-(h/2)-5,c=g,a.top+c+e>j&&(c=-e-g),(d=a.left+b)<0&&(b-=d),(d=a.left+b+h)>i&&(b-=d-i);break;case"s":b=-(h/2),c=-e-g,a.top+c<0&&(c=g),(d=a.left+b)<0&&(b-=d),(d=a.left+b+h)>i&&(b-=d-i);break;case"center":b=-(h/2),c=-(e/2);break;default:b=0,c=0}return{left:b,top:c}},y=function(){a.dom.read(function(){var a=v(),b=x(a),c=a.left+b.left,d=a.top+b.top;if(j)l.interrupt().transition().delay(k).duration(0).style("opacity",0);else{var e="translate("+m.left+"px, "+m.top+"px)",f="translate("+Math.round(c)+"px, "+Math.round(d)+"px)",g=d3.interpolateString(e,f),h=l.style("opacity")<.1;l.interrupt().transition().duration(h?0:o).styleTween("transform",function(a){return g},"important").styleTween("-webkit-transform",function(a){return g}).style("-ms-transform",f).styl
 e("opacity",1)}m.left=c,m.top=d})};return c.nvPointerEventsClass=q,c.options=a.utils.optionsFunc.bind(c),c._options=Object.create({},{duration:{get:function(){return o},set:function(a){o=a}},gravity:{get:function(){return f},set:function(a){f=a}},distance:{get:function(){return g},set:function(a){g=a}},snapDistance:{get:function(){return h},set:function(a){h=a}},classes:{get:function(){return i},set:function(a){i=a}},enabled:{get:function(){return n},set:function(a){n=a}},hideDelay:{get:function(){return k},set:function(a){k=a}},contentGenerator:{get:function(){return u},set:function(a){u=a}},valueFormatter:{get:function(){return r},set:function(a){r=a}},headerFormatter:{get:function(){return s},set:function(a){s=a}},keyFormatter:{get:function(){return t},set:function(a){t=a}},headerEnabled:{get:function(){return p},set:function(a){p=a}},position:{get:function(){return v},set:function(a){v=a}},chartContainer:{get:function(){return document.body},set:function(b){a.deprecated("chartCo
 ntainer","feature removed after 1.8.3")}},fixedTop:{get:function(){return null},set:function(b){a.deprecated("fixedTop","feature removed after 1.8.1")}},offset:{get:function(){return{left:0,top:0}},set:function(b){a.deprecated("offset","use chart.tooltip.distance() instead")}},hidden:{get:function(){return j},set:function(a){j!=a&&(j=!!a,c())}},data:{get:function(){return e},set:function(a){a.point&&(a.value=a.point.x,a.series=a.series||{},a.series.value=a.point.y,a.series.color=a.point.color||a.series.color),e=a}},node:{get:function(){return l.node()},set:function(a){}},id:{get:function(){return d},set:function(a){}}}),a.utils.initOptions(c),c},a.utils.windowSize=function(){var a={width:640,height:480};return window.innerWidth&&window.innerHeight?(a.width=window.innerWidth,a.height=window.innerHeight,a):"CSS1Compat"==document.compatMode&&document.documentElement&&document.documentElement.offsetWidth?(a.width=document.documentElement.offsetWidth,a.height=document.documentElement.off
 setHeight,a):document.body&&document.body.offsetWidth?(a.width=document.body.offsetWidth,a.height=document.body.offsetHeight,a):a},a.utils.isArray=Array.isArray,a.utils.isObject=function(a){return null!==a&&"object"==typeof a},a.utils.isFunction=function(a){return"function"==typeof a},a.utils.isDate=function(a){return"[object Date]"===toString.call(a)},a.utils.isNumber=function(a){return!isNaN(a)&&"number"==typeof a},a.utils.windowResize=function(b){return window.addEventListener?window.addEventListener("resize",b):a.log("ERROR: Failed to bind to window.resize with: ",b),{callback:b,clear:function(){window.removeEventListener("resize",b)}}},a.utils.getColor=function(b){if(void 0===b)return a.utils.defaultColor();if(a.utils.isArray(b)){var c=d3.scale.ordinal().range(b);return function(a,b){var d=void 0===b?a:b;return a.color||c(d)}}return b},a.utils.defaultColor=function(){return a.utils.getColor(d3.scale.category20().range())},a.utils.customTheme=function(b,c,d){c=c||function(a){ret
 urn a.key},d=d||d3.scale.category20().range();var e=d.length;return function(f,g){var h=c(f);return a.utils.isFunction(b[h])?b[h]():void 0!==b[h]?b[h]:(e||(e=d.length),e-=1,d[e])}},a.utils.pjax=function(b,c){var d=function(d){d3.html(d,function(d){var e=d3.select(c).node();e.parentNode.replaceChild(d3.select(d).select(c).node(),e),a.utils.pjax(b,c)})};d3.selectAll(b).on("click",function(){history.pushState(this.href,this.textContent,this.href),d(this.href),d3.event.preventDefault()}),d3.select(window).on("popstate",function(){d3.event.state&&d(d3.event.state)})},a.utils.calcApproxTextWidth=function(b){if(a.utils.isFunction(b.style)&&a.utils.isFunction(b.text)){var c=parseInt(b.style("font-size").replace("px",""),10),d=b.text().length;return a.utils.NaNtoZero(d*c*.5)}return 0},a.utils.NaNtoZero=function(b){return!a.utils.isNumber(b)||isNaN(b)||null===b||b===1/0||b===-(1/0)?0:b},d3.selection.prototype.watchTransition=function(a){var b=[this].concat([].slice.call(arguments,1));return a
 .transition.apply(a,b)},a.utils.renderWatch=function(b,c){if(!(this instanceof a.utils.renderWatch))return new a.utils.renderWatch(b,c);var d=void 0!==c?c:250,e=[],f=this;this.models=function(a){return a=[].slice.call(arguments,0),a.forEach(function(a){a.__rendered=!1,function(a){a.dispatch.on("renderEnd",function(b){a.__rendered=!0,f.renderEnd("model")})}(a),e.indexOf(a)<0&&e.push(a)}),this},this.reset=function(a){void 0!==a&&(d=a),e=[]},this.transition=function(a,b,c){if(b=arguments.length>1?[].slice.call(arguments,1):[],c=b.length>1?b.pop():void 0!==d?d:250,a.__rendered=!1,e.indexOf(a)<0&&e.push(a),0===c)return a.__rendered=!0,a.delay=function(){return this},a.duration=function(){return this},a;0===a.length?a.__rendered=!0:a.every(function(a){return!a.length})?a.__rendered=!0:a.__rendered=!1;var g=0;return a.transition().duration(c).each(function(){++g}).each("end",function(c,d){0===--g&&(a.__rendered=!0,f.renderEnd.apply(this,b))})},this.renderEnd=function(){e.every(function(a){
 return a.__rendered})&&(e.forEach(function(a){a.__rendered=!1}),b.renderEnd.apply(this,arguments))}},a.utils.deepExtend=function(b){var c=arguments.length>1?[].slice.call(arguments,1):[];c.forEach(function(c){for(var d in c){var e=a.utils.isArray(b[d]),f=a.utils.isObject(b[d]),g=a.utils.isObject(c[d]);f&&!e&&g?a.utils.deepExtend(b[d],c[d]):b[d]=c[d]}})},a.utils.state=function(){if(!(this instanceof a.utils.state))return new a.utils.state;var b={},c=function(){},d=function(){return{}},e=null,f=null;this.dispatch=d3.dispatch("change","set"),this.dispatch.on("set",function(a){c(a,!0)}),this.getter=function(a){return d=a,this},this.setter=function(a,b){return b||(b=function(){}),c=function(c,d){a(c),d&&b()},this},this.init=function(b){e=e||{},a.utils.deepExtend(e,b)};var g=function(){var a=d();if(JSON.stringify(a)===JSON.stringify(b))return!1;for(var c in a)void 0===b[c]&&(b[c]={}),b[c]=a[c],f=!0;return!0};this.update=function(){e&&(c(e,!1),e=null),g.call(this)&&this.dispatch.change(b)}
 },a.utils.optionsFunc=function(b){return b&&d3.map(b).forEach(function(b,c){a.utils.isFunction(this[b])&&this[b](c)}.bind(this)),this},a.utils.calcTicksX=function(b,c){var d=1,e=0;for(e;e<c.length;e+=1){var f=c[e]&&c[e].values?c[e].values.length:0;d=f>d?f:d}return a.log("Requested number of ticks: ",b),a.log("Calculated max values to be: ",d),b=b>d?b=d-1:b,b=1>b?1:b,b=Math.floor(b),a.log("Calculating tick count as: ",b),b},a.utils.calcTicksY=function(b,c){return a.utils.calcTicksX(b,c)},a.utils.initOption=function(a,b){a._calls&&a._calls[b]?a[b]=a._calls[b]:(a[b]=function(c){return arguments.length?(a._overrides[b]=!0,a._options[b]=c,a):a._options[b]},a["_"+b]=function(c){return arguments.length?(a._overrides[b]||(a._options[b]=c),a):a._options[b]})},a.utils.initOptions=function(b){b._overrides=b._overrides||{};var c=Object.getOwnPropertyNames(b._options||{}),d=Object.getOwnPropertyNames(b._calls||{});c=c.concat(d);for(var e in c)a.utils.initOption(b,c[e])},a.utils.inheritOptionsD3=
 function(a,b,c){a._d3options=c.concat(a._d3options||[]),c.unshift(b),c.unshift(a),d3.rebind.apply(this,c)},a.utils.arrayUnique=function(a){return a.sort().filter(function(b,c){return!c||b!=a[c-1]})},a.utils.symbolMap=d3.map(),a.utils.symbol=function(){function b(b,e){var f=c.call(this,b,e),g=d.call(this,b,e);return-1!==d3.svg.symbolTypes.indexOf(f)?d3.svg.symbol().type(f).size(g)():a.utils.symbolMap.get(f)(g)}var c,d=64;return b.type=function(a){return arguments.length?(c=d3.functor(a),b):c},b.size=function(a){return arguments.length?(d=d3.functor(a),b):d},b},a.utils.inheritOptions=function(b,c){var d=Object.getOwnPropertyNames(c._options||{}),e=Object.getOwnPropertyNames(c._calls||{}),f=c._inherited||[],g=c._d3options||[],h=d.concat(e).concat(f).concat(g);h.unshift(c),h.unshift(b),d3.rebind.apply(this,h),b._inherited=a.utils.arrayUnique(d.concat(e).concat(f).concat(d).concat(b._inherited||[])),b._d3options=a.utils.arrayUnique(g.concat(b._d3options||[]))},a.utils.initSVG=function(a)
 {a.classed({"nvd3-svg":!0})},a.utils.sanitizeHeight=function(a,b){return a||parseInt(b.style("height"),10)||400},a.utils.sanitizeWidth=function(a,b){return a||parseInt(b.style("width"),10)||960},a.utils.availableHeight=function(b,c,d){return Math.max(0,a.utils.sanitizeHeight(b,c)-d.top-d.bottom)},a.utils.availableWidth=function(b,c,d){return Math.max(0,a.utils.sanitizeWidth(b,c)-d.left-d.right)},a.utils.noData=function(b,c){var d=b.options(),e=d.margin(),f=d.noData(),g=null==f?["No Data Available."]:[f],h=a.utils.availableHeight(null,c,e),i=a.utils.availableWidth(null,c,e),j=e.left+i/2,k=e.top+h/2;c.selectAll("g").remove();var l=c.selectAll(".nv-noData").data(g);l.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),l.attr("x",j).attr("y",k).text(function(a){return a})},a.utils.wrapTicks=function(a,b){a.each(function(){for(var a,c=d3.select(this),d=c.text().split(/\s+/).reverse(),e=[],f=0,g=1.1,h=c.attr("y"),i=parseFloat(c.attr("dy")
 ),j=c.text(null).append("tspan").attr("x",0).attr("y",h).attr("dy",i+"em");a=d.pop();)e.push(a),j.text(e.join(" ")),j.node().getComputedTextLength()>b&&(e.pop(),j.text(e.join(" ")),e=[a],j=c.append("tspan").attr("x",0).attr("y",h).attr("dy",++f*g+i+"em").text(a))})},a.utils.arrayEquals=function(b,c){if(b===c)return!0;if(!b||!c)return!1;if(b.length!=c.length)return!1;for(var d=0,e=b.length;e>d;d++)if(b[d]instanceof Array&&c[d]instanceof Array){if(!a.arrayEquals(b[d],c[d]))return!1}else if(b[d]!=c[d])return!1;return!0},a.models.axis=function(){"use strict";function b(g){return t.reset(),g.each(function(b){var g=d3.select(this);a.utils.initSVG(g);var q=g.selectAll("g.nv-wrap.nv-axis").data([b]),r=q.enter().append("g").attr("class","nvd3 nv-wrap nv-axis"),u=(r.append("g"),q.select("g"));null!==n?c.ticks(n):("top"==c.orient()||"bottom"==c.orient())&&c.ticks(Math.abs(d.range()[1]-d.range()[0])/100),u.watchTransition(t,"axis").call(c),s=s||c.scale();var v=c.tickFormat();null==v&&(v=s.tickF
 ormat());var w=u.selectAll("text.nv-axislabel").data([h||null]);w.exit().remove(),void 0!==p&&u.selectAll("g").select("text").style("font-size",p);var x,y,z;switch(c.orient()){case"top":w.enter().append("text").attr("class","nv-axislabel"),z=0,1===d.range().length?z=m?2*d.range()[0]+d.rangeBand():0:2===d.range().length?z=m?d.range()[0]+d.range()[1]+d.rangeBand():d.range()[1]:d.range().length>2&&(z=d.range()[d.range().length-1]+(d.range()[1]-d.range()[0])),w.attr("text-anchor","middle").attr("y",0).attr("x",z/2),i&&(y=q.selectAll("g.nv-axisMaxMin").data(d.domain()),y.enter().append("g").attr("class",function(a,b){return["nv-axisMaxMin","nv-axisMaxMin-x",0==b?"nv-axisMin-x":"nv-axisMax-x"].join(" ")}).append("text"),y.exit().remove(),y.attr("transform",function(b,c){return"translate("+a.utils.NaNtoZero(d(b))+",0)"}).select("text").attr("dy","-0.5em").attr("y",-c.tickPadding()).attr("text-anchor","middle").text(function(a,b){var c=v(a);return(""+c).match("NaN")?"":c}),y.watchTransition
 (t,"min-max top").attr("transform",function(b,c){return"translate("+a.utils.NaNtoZero(d.range()[c])+",0)"}));break;case"bottom":x=o+36;var A=30,B=0,C=u.selectAll("g").select("text"),D="";if(j%360){C.attr("transform",""),C.each(function(a,b){var c=this.getBoundingClientRect(),d=c.width;B=c.height,d>A&&(A=d)}),D="rotate("+j+" 0,"+(B/2+c.tickPadding())+")";var E=Math.abs(Math.sin(j*Math.PI/180));x=(E?E*A:A)+30,C.attr("transform",D).style("text-anchor",j%360>0?"start":"end")}else l?C.attr("transform",function(a,b){return"translate(0,"+(b%2==0?"0":"12")+")"}):C.attr("transform","translate(0,0)");w.enter().append("text").attr("class","nv-axislabel"),z=0,1===d.range().length?z=m?2*d.range()[0]+d.rangeBand():0:2===d.range().length?z=m?d.range()[0]+d.range()[1]+d.rangeBand():d.range()[1]:d.range().length>2&&(z=d.range()[d.range().length-1]+(d.range()[1]-d.range()[0])),w.attr("text-anchor","middle").attr("y",x).attr("x",z/2),i&&(y=q.selectAll("g.nv-axisMaxMin").data([d.domain()[0],d.domain()[
 d.domain().length-1]]),y.enter().append("g").attr("class",function(a,b){return["nv-axisMaxMin","nv-axisMaxMin-x",0==b?"nv-axisMin-x":"nv-axisMax-x"].join(" ")}).append("text"),y.exit().remove(),y.attr("transform",function(b,c){return"translate("+a.utils.NaNtoZero(d(b)+(m?d.rangeBand()/2:0))+",0)"}).select("text").attr("dy",".71em").attr("y",c.tickPadding()).attr("transform",D).style("text-anchor",j?j%360>0?"start":"end":"middle").text(function(a,b){var c=v(a);return(""+c).match("NaN")?"":c}),y.watchTransition(t,"min-max bottom").attr("transform",function(b,c){return"translate("+a.utils.NaNtoZero(d(b)+(m?d.rangeBand()/2:0))+",0)"}));break;case"right":w.enter().append("text").attr("class","nv-axislabel"),w.style("text-anchor",k?"middle":"begin").attr("transform",k?"rotate(90)":"").attr("y",k?-Math.max(e.right,f)+12-(o||0):-10).attr("x",k?d3.max(d.range())/2:c.tickPadding()),i&&(y=q.selectAll("g.nv-axisMaxMin").data(d.domain()),y.enter().append("g").attr("class",function(a,b){return["n
 v-axisMaxMin","nv-axisMaxMin-y",0==b?"nv-axisMin-y":"nv-axisMax-y"].join(" ")}).append("text").style("opacity",0),y.exit().remove(),y.attr("transform",function(b,c){return"translate(0,"+a.utils.NaNtoZero(d(b))+")"}).select("text").attr("dy",".32em").attr("y",0).attr("x",c.tickPadding()).style("text-anchor","start").text(function(a,b){var c=v(a);return(""+c).match("NaN")?"":c}),y.watchTransition(t,"min-max right").attr("transform",function(b,c){return"translate(0,"+a.utils.NaNtoZero(d.range()[c])+")"}).select("text").style("opacity",1));break;case"left":w.enter().append("text").attr("class","nv-axislabel"),w.style("text-anchor",k?"middle":"end").attr("transform",k?"rotate(-90)":"").attr("y",k?-Math.max(e.left,f)+25-(o||0):-10).attr("x",k?-d3.max(d.range())/2:-c.tickPadding()),i&&(y=q.selectAll("g.nv-axisMaxMin").data(d.domain()),y.enter().append("g").attr("class",function(a,b){return["nv-axisMaxMin","nv-axisMaxMin-y",0==b?"nv-axisMin-y":"nv-axisMax-y"].join(" ")}).append("text").styl
 e("opacity",0),y.exit().remove(),y.attr("transform",function(b,c){return"translate(0,"+a.utils.NaNtoZero(s(b))+")"}).select("text").attr("dy",".32em").attr("y",0).attr("x",-c.tickPadding()).attr("text-anchor","end").text(function(a,b){var c=v(a);return(""+c).match("NaN")?"":c}),y.watchTransition(t,"min-max right").attr("transform",function(b,c){return"translate(0,"+a.utils.NaNtoZero(d.range()[c])+")"}).select("text").style("opacity",1))}if(w.text(function(a){return a}),!i||"left"!==c.orient()&&"right"!==c.orient()||(u.selectAll("g").each(function(a,b){d3.select(this).select("text").attr("opacity",1),(d(a)<d.range()[1]+10||d(a)>d.range()[0]-10)&&((a>1e-10||-1e-10>a)&&d3.select(this).attr("opacity",0),d3.select(this).select("text").attr("opacity",0))}),d.domain()[0]==d.domain()[1]&&0==d.domain()[0]&&q.selectAll("g.nv-axisMaxMin").style("opacity",function(a,b){return b?0:1})),i&&("top"===c.orient()||"bottom"===c.orient())){var F=[];q.selectAll("g.nv-axisMaxMin").each(function(a,b){try{
 b?F.push(d(a)-this.getBoundingClientRect().width-4):F.push(d(a)+this.getBoundingClientRect().width+4)}catch(c){b?F.push(d(a)-4):F.push(d(a)+4)}}),u.selectAll("g").each(function(a,b){(d(a)<F[0]||d(a)>F[1])&&(a>1e-10||-1e-10>a?d3.select(this).remove():d3.select(this).select("text").remove())})}u.selectAll(".tick").filter(function(a){return!parseFloat(Math.round(1e5*a)/1e6)&&void 0!==a}).classed("zero",!0),s=d.copy()}),t.renderEnd("axis immediate"),b}var c=d3.svg.axis(),d=d3.scale.linear(),e={top:0,right:0,bottom:0,left:0},f=75,g=60,h=null,i=!0,j=0,k=!0,l=!1,m=!1,n=null,o=0,p=void 0,q=250,r=d3.dispatch("renderEnd");c.scale(d).orient("bottom").tickFormat(function(a){return a});var s,t=a.utils.renderWatch(r,q);return b.axis=c,b.dispatch=r,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{axisLabelDistance:{get:function(){return o},set:function(a){o=a}},staggerLabels:{get:function(){return l},set:function(a){l=a}},rotateLabels:{get:function(){return j},set:function(a){j=a
 }},rotateYLabel:{get:function(){return k},set:function(a){k=a}},showMaxMin:{get:function(){return i},set:function(a){i=a}},axisLabel:{get:function(){return h},set:function(a){h=a}},height:{get:function(){return g},set:function(a){g=a}},ticks:{get:function(){return n},set:function(a){n=a}},width:{get:function(){return f},set:function(a){f=a}},fontSize:{get:function(){return p},set:function(a){p=a}},margin:{get:function(){return e},set:function(a){e.top=void 0!==a.top?a.top:e.top,e.right=void 0!==a.right?a.right:e.right,e.bottom=void 0!==a.bottom?a.bottom:e.bottom,e.left=void 0!==a.left?a.left:e.left}},duration:{get:function(){return q},set:function(a){q=a,t.reset(q)}},scale:{get:function(){return d},set:function(e){d=e,c.scale(d),m="function"==typeof d.rangeBands,a.utils.inheritOptionsD3(b,d,["domain","range","rangeBand","rangeBands"])}}}),a.utils.initOptions(b),a.utils.inheritOptionsD3(b,c,["orient","tickValues","tickSubdivide","tickSize","tickPadding","tickFormat"]),a.utils.inherit
 OptionsD3(b,d,["domain","range","rangeBand","rangeBands"]),b},a.models.boxPlot=function(){"use strict";function b(l){return E.reset(),l.each(function(b){var l=j-i.left-i.right,F=k-i.top-i.bottom;A=d3.select(this),a.utils.initSVG(A),m.domain(c||b.map(function(a,b){return o(a,b)})).rangeBands(d||[0,l],.1);var G=[];if(!e){var H,I,J=[];b.forEach(function(a,b){var c=p(a),d=r(a),e=s(a),f=t(a),g=v(a);g&&g.forEach(function(a,b){J.push(w(a,b,void 0))}),e&&J.push(e),c&&J.push(c),d&&J.push(d),f&&J.push(f)}),H=d3.min(J),I=d3.max(J),G=[H,I]}n.domain(e||G),n.range(f||[F,0]),g=g||m,h=h||n.copy().range([n(0),n(0)]);var K=A.selectAll("g.nv-wrap").data([b]);K.enter().append("g").attr("class","nvd3 nv-wrap");K.attr("transform","translate("+i.left+","+i.top+")");var L=K.selectAll(".nv-boxplot").data(function(a){return a}),M=L.enter().append("g").style("stroke-opacity",1e-6).style("fill-opacity",1e-6);L.attr("class","nv-boxplot").attr("transform",function(a,b,c){return"translate("+(m(o(a,b))+.05*m.range
 Band())+", 0)"}).classed("hover",function(a){return a.hover}),L.watchTransition(E,"nv-boxplot: boxplots").style("stroke-opacity",1).style("fill-opacity",.75).delay(function(a,c){return c*C/b.length}).attr("transform",function(a,b){return"translate("+(m(o(a,b))+.05*m.rangeBand())+", 0)"}),L.exit().remove(),M.each(function(a,b){var c=d3.select(this);[s,t].forEach(function(d){if(void 0!==d(a)&&null!==d(a)){var e=d===s?"low":"high";c.append("line").style("stroke",u(a)||z(a,b)).attr("class","nv-boxplot-whisker nv-boxplot-"+e),c.append("line").style("stroke",u(a)||z(a,b)).attr("class","nv-boxplot-tick nv-boxplot-"+e)}})});var N=function(){return null===D?.9*m.rangeBand():Math.min(75,.9*m.rangeBand())},O=function(){return.45*m.rangeBand()-N()/2},P=function(){return.45*m.rangeBand()+N()/2};[s,t].forEach(function(a){var b=a===s?"low":"high",c=a===s?p:r;L.select("line.nv-boxplot-whisker.nv-boxplot-"+b).watchTransition(E,"nv-boxplot: boxplots").attr("x1",.45*m.rangeBand()).attr("y1",function(b
 ,c){return n(a(b))}).attr("x2",.45*m.rangeBand()).attr("y2",function(a,b){return n(c(a))}),L.select("line.nv-boxplot-tick.nv-boxplot-"+b).watchTransition(E,"nv-boxplot: boxplots").attr("x1",O).attr("y1",function(b,c){return n(a(b))}).attr("x2",P).attr("y2",function(b,c){return n(a(b))})}),[s,t].forEach(function(a){var b=a===s?"low":"high";M.selectAll(".nv-boxplot-"+b).on("mouseover",function(b,c,d){d3.select(this).classed("hover",!0),B.elementMouseover({series:{key:a(b),color:u(b)||z(b,d)},e:d3.event})}).on("mouseout",function(b,c,d){d3.select(this).classed("hover",!1),B.elementMouseout({series:{key:a(b),color:u(b)||z(b,d)},e:d3.event})}).on("mousemove",function(a,b){B.elementMousemove({e:d3.event})})}),M.append("rect").attr("class","nv-boxplot-box").on("mouseover",function(a,b){d3.select(this).classed("hover",!0),B.elementMouseover({key:o(a),value:o(a),series:[{key:"Q3",value:r(a),color:u(a)||z(a,b)},{key:"Q2",value:q(a),color:u(a)||z(a,b)},{key:"Q1",value:p(a),color:u(a)||z(a,b)}]
 ,data:a,index:b,e:d3.event})}).on("mouseout",function(a,b){d3.select(this).classed("hover",!1),B.elementMouseout({key:o(a),value:o(a),series:[{key:"Q3",value:r(a),color:u(a)||z(a,b)},{key:"Q2",value:q(a),color:u(a)||z(a,b)},{key:"Q1",value:p(a),color:u(a)||z(a,b)}],data:a,index:b,e:d3.event})}).on("mousemove",function(a,b){B.elementMousemove({e:d3.event})}),L.select("rect.nv-boxplot-box").watchTransition(E,"nv-boxplot: boxes").attr("y",function(a,b){return n(r(a))}).attr("width",N).attr("x",O).attr("height",function(a,b){return Math.abs(n(r(a))-n(p(a)))||1}).style("fill",function(a,b){return u(a)||z(a,b)}).style("stroke",function(a,b){return u(a)||z(a,b)}),M.append("line").attr("class","nv-boxplot-median"),L.select("line.nv-boxplot-median").watchTransition(E,"nv-boxplot: boxplots line").attr("x1",O).attr("y1",function(a,b){return n(q(a))}).attr("x2",P).attr("y2",function(a,b){return n(q(a))});var Q=L.selectAll(".nv-boxplot-outlier").data(function(a){return v(a)||[]});Q.enter().appen
 d("circle").style("fill",function(a,b,c){return y(a,b,c)||z(a,c)}).style("stroke",function(a,b,c){return y(a,b,c)||z(a,c)}).style("z-index",9e3).on("mouseover",function(a,b,c){d3.select(this).classed("hover",!0),B.elementMouseover({series:{key:x(a,b,c),color:y(a,b,c)||z(a,c)},e:d3.event})}).on("mouseout",function(a,b,c){d3.select(this).classed("hover",!1),B.elementMouseout({series:{key:x(a,b,c),color:y(a,b,c)||z(a,c)},e:d3.event})}).on("mousemove",function(a,b){B.elementMousemove({e:d3.event})}),Q.attr("class","nv-boxplot-outlier"),Q.watchTransition(E,"nv-boxplot: nv-boxplot-outlier").attr("cx",.45*m.rangeBand()).attr("cy",function(a,b,c){return n(w(a,b,c))}).attr("r","3"),Q.exit().remove(),g=m.copy(),h=n.copy()}),E.renderEnd("nv-boxplot immediate"),b}var c,d,e,f,g,h,i={top:0,right:0,bottom:0,left:0},j=960,k=500,l=Math.floor(1e4*Math.random()),m=d3.scale.ordinal(),n=d3.scale.linear(),o=function(a){return a.label},p=function(a){return a.values.Q1},q=function(a){return a.values.Q2},r=
 function(a){return a.values.Q3},s=function(a){return a.values.whisker_low},t=function(a){return a.values.whisker_high},u=function(a){return a.color},v=function(a){return a.values.outliers},w=function(a,b,c){return a},x=function(a,b,c){return a},y=function(a,b,c){return void 0},z=a.utils.defaultColor(),A=null,B=d3.dispatch("elementMouseover","elementMouseout","elementMousemove","renderEnd"),C=250,D=null,E=a.utils.renderWatch(B,C);return b.dispatch=B,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return j},set:function(a){j=a}},height:{get:function(){return k},set:function(a){k=a}},maxBoxWidth:{get:function(){return D},set:function(a){D=a}},x:{get:function(){return o},set:function(a){o=a}},q1:{get:function(){return p},set:function(a){p=a}},q2:{get:function(){return q},set:function(a){q=a}},q3:{get:function(){return r},set:function(a){r=a}},wl:{get:function(){return s},set:function(a){s=a}},wh:{get:function(){return t},set:function(a){t=a}},ite
 mColor:{get:function(){return u},set:function(a){u=a}},outliers:{get:function(){return v},set:function(a){
-v=a}},outlierValue:{get:function(){return w},set:function(a){w=a}},outlierLabel:{get:function(){return x},set:function(a){x=a}},outlierColor:{get:function(){return y},set:function(a){y=a}},xScale:{get:function(){return m},set:function(a){m=a}},yScale:{get:function(){return n},set:function(a){n=a}},xDomain:{get:function(){return c},set:function(a){c=a}},yDomain:{get:function(){return e},set:function(a){e=a}},xRange:{get:function(){return d},set:function(a){d=a}},yRange:{get:function(){return f},set:function(a){f=a}},id:{get:function(){return l},set:function(a){l=a}},y:{get:function(){return console.warn("BoxPlot 'y' chart option is deprecated. Please use model overrides instead."),{}},set:function(a){console.warn("BoxPlot 'y' chart option is deprecated. Please use model overrides instead.")}},margin:{get:function(){return i},set:function(a){i.top=void 0!==a.top?a.top:i.top,i.right=void 0!==a.right?a.right:i.right,i.bottom=void 0!==a.bottom?a.bottom:i.bottom,i.left=void 0!==a.left?a.l
 eft:i.left}},color:{get:function(){return z},set:function(b){z=a.utils.getColor(b)}},duration:{get:function(){return C},set:function(a){C=a,E.reset(C)}}}),a.utils.initOptions(b),b},a.models.boxPlotChart=function(){"use strict";function b(k){return t.reset(),t.models(e),l&&t.models(f),m&&t.models(g),k.each(function(k){var p=d3.select(this);a.utils.initSVG(p);var t=(i||parseInt(p.style("width"))||960)-h.left-h.right,u=(j||parseInt(p.style("height"))||400)-h.top-h.bottom;if(b.update=function(){r.beforeUpdate(),p.transition().duration(s).call(b)},b.container=this,!k||!k.length){var v=p.selectAll(".nv-noData").data([q]);return v.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),v.attr("x",h.left+t/2).attr("y",h.top+u/2).text(function(a){return a}),b}p.selectAll(".nv-noData").remove(),c=e.xScale(),d=e.yScale().clamp(!0);var w=p.selectAll("g.nv-wrap.nv-boxPlotWithAxes").data([k]),x=w.enter().append("g").attr("class","nvd3 nv-wrap nv-boxP
 lotWithAxes").append("g"),y=x.append("defs"),z=w.select("g");x.append("g").attr("class","nv-x nv-axis"),x.append("g").attr("class","nv-y nv-axis").append("g").attr("class","nv-zeroLine").append("line"),x.append("g").attr("class","nv-barsWrap"),z.attr("transform","translate("+h.left+","+h.top+")"),n&&z.select(".nv-y.nv-axis").attr("transform","translate("+t+",0)"),e.width(t).height(u);var A=z.select(".nv-barsWrap").datum(k.filter(function(a){return!a.disabled}));if(A.transition().call(e),y.append("clipPath").attr("id","nv-x-label-clip-"+e.id()).append("rect"),z.select("#nv-x-label-clip-"+e.id()+" rect").attr("width",c.rangeBand()*(o?2:1)).attr("height",16).attr("x",-c.rangeBand()/(o?1:2)),l){f.scale(c).ticks(a.utils.calcTicksX(t/100,k)).tickSize(-u,0),z.select(".nv-x.nv-axis").attr("transform","translate(0,"+d.range()[0]+")"),z.select(".nv-x.nv-axis").call(f);var B=z.select(".nv-x.nv-axis").selectAll("g");o&&B.selectAll("text").attr("transform",function(a,b,c){return"translate(0,"+(c
 %2===0?"5":"17")+")"})}m&&(g.scale(d).ticks(Math.floor(u/36)).tickSize(-t,0),z.select(".nv-y.nv-axis").call(g)),z.select(".nv-zeroLine line").attr("x1",0).attr("x2",t).attr("y1",d(0)).attr("y2",d(0))}),t.renderEnd("nv-boxplot chart immediate"),b}var c,d,e=a.models.boxPlot(),f=a.models.axis(),g=a.models.axis(),h={top:15,right:10,bottom:50,left:60},i=null,j=null,k=a.utils.getColor(),l=!0,m=!0,n=!1,o=!1,p=a.models.tooltip(),q="No Data Available.",r=d3.dispatch("beforeUpdate","renderEnd"),s=250;f.orient("bottom").showMaxMin(!1).tickFormat(function(a){return a}),g.orient(n?"right":"left").tickFormat(d3.format(",.1f")),p.duration(0);var t=a.utils.renderWatch(r,s);return e.dispatch.on("elementMouseover.tooltip",function(a){p.data(a).hidden(!1)}),e.dispatch.on("elementMouseout.tooltip",function(a){p.data(a).hidden(!0)}),e.dispatch.on("elementMousemove.tooltip",function(a){p()}),b.dispatch=r,b.boxplot=e,b.xAxis=f,b.yAxis=g,b.tooltip=p,b.options=a.utils.optionsFunc.bind(b),b._options=Object.c
 reate({},{width:{get:function(){return i},set:function(a){i=a}},height:{get:function(){return j},set:function(a){j=a}},staggerLabels:{get:function(){return o},set:function(a){o=a}},showXAxis:{get:function(){return l},set:function(a){l=a}},showYAxis:{get:function(){return m},set:function(a){m=a}},tooltipContent:{get:function(){return p},set:function(a){p=a}},noData:{get:function(){return q},set:function(a){q=a}},margin:{get:function(){return h},set:function(a){h.top=void 0!==a.top?a.top:h.top,h.right=void 0!==a.right?a.right:h.right,h.bottom=void 0!==a.bottom?a.bottom:h.bottom,h.left=void 0!==a.left?a.left:h.left}},duration:{get:function(){return s},set:function(a){s=a,t.reset(s),e.duration(s),f.duration(s),g.duration(s)}},color:{get:function(){return k},set:function(b){k=a.utils.getColor(b),e.color(k)}},rightAlignYAxis:{get:function(){return n},set:function(a){n=a,g.orient(a?"right":"left")}}}),a.utils.inheritOptions(b,e),a.utils.initOptions(b),b},a.models.bullet=function(){"use str
 ict";function b(a,b){var c=a.slice();a.sort(function(a,d){var e=c.indexOf(a),f=c.indexOf(d);return d3.descending(b[e],b[f])})}function c(e){return e.each(function(c,e){var s=p-d.left-d.right,y=q-d.top-d.bottom;r=d3.select(this),a.utils.initSVG(r);var z=g.call(this,c,e).slice(),A=h.call(this,c,e).slice(),B=i.call(this,c,e).slice(),C=j.call(this,c,e).slice(),D=k.call(this,c,e).slice(),E=l.call(this,c,e).slice(),F=m.call(this,c,e).slice(),G=n.call(this,c,e).slice();b(D,z),b(E,A),b(F,B),b(G,C),z.sort(d3.descending),A.sort(d3.descending),B.sort(d3.descending),C.sort(d3.descending);var H=d3.scale.linear().domain(d3.extent(d3.merge([o,z]))).range(f?[s,0]:[0,s]);this.__chart__||d3.scale.linear().domain([0,1/0]).range(H.range());this.__chart__=H;for(var I=(d3.min(z),d3.max(z),z[1],r.selectAll("g.nv-wrap.nv-bullet").data([c])),J=I.enter().append("g").attr("class","nvd3 nv-wrap nv-bullet"),K=J.append("g"),L=I.select("g"),e=0,M=z.length;M>e;e++){var N="nv-range nv-range"+e;2>=e&&(N=N+" nv-range
 "+w[e]),K.append("rect").attr("class",N)}K.append("rect").attr("class","nv-measure"),I.attr("transform","translate("+d.left+","+d.top+")");for(var O=function(a){return Math.abs(H(a)-H(0))},P=function(a){return H(0>a?a:0)},e=0,M=z.length;M>e;e++){var Q=z[e];L.select("rect.nv-range"+e).datum(Q).attr("height",y).transition().duration(x).attr("width",O(Q)).attr("x",P(Q))}L.select("rect.nv-measure").style("fill",t).attr("height",y/3).attr("y",y/3).on("mouseover",function(){u.elementMouseover({value:C[0],label:G[0]||"Current",color:d3.select(this).style("fill")})}).on("mousemove",function(){u.elementMousemove({value:C[0],label:G[0]||"Current",color:d3.select(this).style("fill")})}).on("mouseout",function(){u.elementMouseout({value:C[0],label:G[0]||"Current",color:d3.select(this).style("fill")})}).transition().duration(x).attr("width",0>C?H(0)-H(C[0]):H(C[0])-H(0)).attr("x",P(C));var R=y/6,S=A.map(function(a,b){return{value:a,label:E[b]}});K.selectAll("path.nv-markerTriangle").data(S).ente
 r().append("path").attr("class","nv-markerTriangle").attr("d","M0,"+R+"L"+R+","+-R+" "+-R+","+-R+"Z").on("mouseover",function(a){u.elementMouseover({value:a.value,label:a.label||"Previous",color:d3.select(this).style("fill"),pos:[H(a.value),y/2]})}).on("mousemove",function(a){u.elementMousemove({value:a.value,label:a.label||"Previous",color:d3.select(this).style("fill")})}).on("mouseout",function(a,b){u.elementMouseout({value:a.value,label:a.label||"Previous",color:d3.select(this).style("fill")})}),L.selectAll("path.nv-markerTriangle").data(S).transition().duration(x).attr("transform",function(a){return"translate("+H(a.value)+","+y/2+")"});var T=B.map(function(a,b){return{value:a,label:F[b]}});K.selectAll("line.nv-markerLine").data(T).enter().append("line").attr("cursor","").attr("class","nv-markerLine").attr("x1",function(a){return H(a.value)}).attr("y1","2").attr("x2",function(a){return H(a.value)}).attr("y2",y-2).on("mouseover",function(a){u.elementMouseover({value:a.value,label:
 a.label||"Previous",color:d3.select(this).style("fill"),pos:[H(a.value),y/2]})}).on("mousemove",function(a){u.elementMousemove({value:a.value,label:a.label||"Previous",color:d3.select(this).style("fill")})}).on("mouseout",function(a,b){u.elementMouseout({value:a.value,label:a.label||"Previous",color:d3.select(this).style("fill")})}),L.selectAll("line.nv-markerLine").data(T).transition().duration(x).attr("x1",function(a){return H(a.value)}).attr("x2",function(a){return H(a.value)}),I.selectAll(".nv-range").on("mouseover",function(a,b){var c=D[b]||v[b];u.elementMouseover({value:a,label:c,color:d3.select(this).style("fill")})}).on("mousemove",function(){u.elementMousemove({value:C[0],label:G[0]||"Previous",color:d3.select(this).style("fill")})}).on("mouseout",function(a,b){var c=D[b]||v[b];u.elementMouseout({value:a,label:c,color:d3.select(this).style("fill")})})}),c}var d={top:0,right:0,bottom:0,left:0},e="left",f=!1,g=function(a){return a.ranges},h=function(a){return a.markers?a.mark
 ers:[]},i=function(a){return a.markerLines?a.markerLines:[0]},j=function(a){return a.measures},k=function(a){return a.rangeLabels?a.rangeLabels:[]},l=function(a){return a.markerLabels?a.markerLabels:[]},m=function(a){return a.markerLineLabels?a.markerLineLabels:[]},n=function(a){return a.measureLabels?a.measureLabels:[]},o=[0],p=380,q=30,r=null,s=null,t=a.utils.getColor(["#1f77b4"]),u=d3.dispatch("elementMouseover","elementMouseout","elementMousemove"),v=["Maximum","Mean","Minimum"],w=["Max","Avg","Min"],x=1e3;return c.dispatch=u,c.options=a.utils.optionsFunc.bind(c),c._options=Object.create({},{ranges:{get:function(){return g},set:function(a){g=a}},markers:{get:function(){return h},set:function(a){h=a}},measures:{get:function(){return j},set:function(a){j=a}},forceX:{get:function(){return o},set:function(a){o=a}},width:{get:function(){return p},set:function(a){p=a}},height:{get:function(){return q},set:function(a){q=a}},tickFormat:{get:function(){return s},set:function(a){s=a}},dur
 ation:{get:function(){return x},set:function(a){x=a}},margin:{get:function(){return d},set:function(a){d.top=void 0!==a.top?a.top:d.top,d.right=void 0!==a.right?a.right:d.right,d.bottom=void 0!==a.bottom?a.bottom:d.bottom,d.left=void 0!==a.left?a.left:d.left}},orient:{get:function(){return e},set:function(a){e=a,f="right"==e||"bottom"==e}},color:{get:function(){return t},set:function(b){t=a.utils.getColor(b)}}}),a.utils.initOptions(c),c},a.models.bulletChart=function(){"use strict";function b(d){return d.each(function(e,o){var p=d3.select(this);a.utils.initSVG(p);var q=a.utils.availableWidth(k,p,g),r=l-g.top-g.bottom;if(b.update=function(){b(d)},b.container=this,!e||!h.call(this,e,o))return a.utils.noData(b,p),b;p.selectAll(".nv-noData").remove();var s=h.call(this,e,o).slice().sort(d3.descending),t=i.call(this,e,o).slice().sort(d3.descending),u=j.call(this,e,o).slice().sort(d3.descending),v=p.selectAll("g.nv-wrap.nv-bulletChart").data([e]),w=v.enter().append("g").attr("class","nvd3 
 nv-wrap nv-bulletChart"),x=w.append("g"),y=v.select("g");x.append("g").attr("class","nv-bulletWrap"),x.append("g").attr("class","nv-titles"),v.attr("transform","translate("+g.left+","+g.top+")");var z=d3.scale.linear().domain([0,Math.max(s[0],t[0]||0,u[0])]).range(f?[q,0]:[0,q]),A=this.__chart__||d3.scale.linear().domain([0,1/0]).range(z.range());this.__chart__=z;var B=x.select(".nv-titles").append("g").attr("text-anchor","end").attr("transform","translate(-6,"+(l-g.top-g.bottom)/2+")");B.append("text").attr("class","nv-title").text(function(a){return a.title}),B.append("text").attr("class","nv-subtitle").attr("dy","1em").text(function(a){return a.subtitle}),c.width(q).height(r);var C=y.select(".nv-bulletWrap");d3.transition(C).call(c);var D=m||z.tickFormat(q/100),E=y.selectAll("g.nv-tick").data(z.ticks(n?n:q/50),function(a){return this.textContent||D(a)}),F=E.enter().append("g").attr("class","nv-tick").attr("transform",function(a){return"translate("+A(a)+",0)"}).style("opacity",1e-
 6);F.append("line").attr("y1",r).attr("y2",7*r/6),F.append("text").attr("text-anchor","middle").attr("dy","1em").attr("y",7*r/6).text(D);var G=d3.transition(E).transition().duration(c.duration()).attr("transform",function(a){return"translate("+z(a)+",0)"}).style("opacity",1);G.select("line").attr("y1",r).attr("y2",7*r/6),G.select("text").attr("y",7*r/6),d3.transition(E.exit()).transition().duration(c.duration()).attr("transform",function(a){return"translate("+z(a)+",0)"}).style("opacity",1e-6).remove()}),d3.timer.flush(),b}var c=a.models.bullet(),d=a.models.tooltip(),e="left",f=!1,g={top:5,right:40,bottom:20,left:120},h=function(a){return a.ranges},i=function(a){return a.markers?a.markers:[]},j=function(a){return a.measures},k=null,l=55,m=null,n=null,o=null,p=d3.dispatch();return d.duration(0).headerEnabled(!1),c.dispatch.on("elementMouseover.tooltip",function(a){a.series={key:a.label,value:a.value,color:a.color},d.data(a).hidden(!1)}),c.dispatch.on("elementMouseout.tooltip",functio
 n(a){d.hidden(!0)}),c.dispatch.on("elementMousemove.tooltip",function(a){d()}),b.bullet=c,b.dispatch=p,b.tooltip=d,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{ranges:{get:function(){return h},set:function(a){h=a}},markers:{get:function(){return i},set:function(a){i=a}},measures:{get:function(){return j},set:function(a){j=a}},width:{get:function(){return k},set:function(a){k=a}},height:{get:function(){return l},set:function(a){l=a}},tickFormat:{get:function(){return m},set:function(a){m=a}},ticks:{get:function(){return n},set:function(a){n=a}},noData:{get:function(){return o},set:function(a){o=a}},margin:{get:function(){return g},set:function(a){g.top=void 0!==a.top?a.top:g.top,g.right=void 0!==a.right?a.right:g.right,g.bottom=void 0!==a.bottom?a.bottom:g.bottom,g.left=void 0!==a.left?a.left:g.left}},orient:{get:function(){return e},set:function(a){e=a,f="right"==e||"bottom"==e}}}),a.utils.inheritOptions(b,c),a.utils.initOptions(b),b},a.models.candlestickBar=fu
 nction(){"use strict";function b(x){return x.each(function(b){c=d3.select(this);var x=a.utils.availableWidth(i,c,h),y=a.utils.availableHeight(j,c,h);a.utils.initSVG(c);var A=x/b[0].values.length*.45;l.domain(d||d3.extent(b[0].values.map(n).concat(t))),v?l.range(f||[.5*x/b[0].values.length,x*(b[0].values.length-.5)/b[0].values.length]):l.range(f||[5+A/2,x-A/2-5]),m.domain(e||[d3.min(b[0].values.map(s).concat(u)),d3.max(b[0].values.map(r).concat(u))]).range(g||[y,0]),l.domain()[0]===l.domain()[1]&&(l.domain()[0]?l.domain([l.domain()[0]-.01*l.domain()[0],l.domain()[1]+.01*l.domain()[1]]):l.domain([-1,1])),m.domain()[0]===m.domain()[1]&&(m.domain()[0]?m.domain([m.domain()[0]+.01*m.domain()[0],m.domain()[1]-.01*m.domain()[1]]):m.domain([-1,1]));var B=d3.select(this).selectAll("g.nv-wrap.nv-candlestickBar").data([b[0].values]),C=B.enter().append("g").attr("class","nvd3 nv-wrap nv-candlestickBar"),D=C.append("defs"),E=C.append("g"),F=B.select("g");E.append("g").attr("class","nv-ticks"),B.a
 ttr("transform","translate("+h.left+","+h.top+")"),c.on("click",function(a,b){z.chartClick({data:a,index:b,pos:d3.event,id:k})}),D.append("clipPath").attr("id","nv-chart-clip-path-"+k).append("rect"),B.select("#nv-chart-clip-path-"+k+" rect").attr("width",x).attr("height",y),F.attr("clip-path",w?"url(#nv-chart-clip-path-"+k+")":"");var G=B.select(".nv-ticks").selectAll(".nv-tick").data(function(a){return a});G.exit().remove();var H=G.enter().append("g");G.attr("class",function(a,b,c){return(p(a,b)>q(a,b)?"nv-tick negative":"nv-tick positive")+" nv-tick-"+c+"-"+b});H.append("line").attr("class","nv-candlestick-lines").attr("transform",function(a,b){return"translate("+l(n(a,b))+",0)"}).attr("x1",0).attr("y1",function(a,b){return m(r(a,b))}).attr("x2",0).attr("y2",function(a,b){return m(s(a,b))}),H.append("rect").attr("class","nv-candlestick-rects nv-bars").attr("transform",function(a,b){return"translate("+(l(n(a,b))-A/2)+","+(m(o(a,b))-(p(a,b)>q(a,b)?m(q(a,b))-m(p(a,b)):0))+")"}).attr
 ("x",0).attr("y",0).attr("width",A).attr("height",function(a,b){var c=p(a,b),d=q(a,b);return c>d?m(d)-m(c):m(c)-m(d)});G.select(".nv-candlestick-lines").transition().attr("transform",function(a,b){return"translate("+l(n(a,b))+",0)"}).attr("x1",0).attr("y1",function(a,b){return m(r(a,b))}).attr("x2",0).attr("y2",function(a,b){return m(s(a,b))}),G.select(".nv-candlestick-rects").transition().attr("transform",function(a,b){return"translate("+(l(n(a,b))-A/2)+","+(m(o(a,b))-(p(a,b)>q(a,b)?m(q(a,b))-m(p(a,b)):0))+")"}).attr("x",0).attr("y",0).attr("width",A).attr("height",function(a,b){var c=p(a,b),d=q(a,b);return c>d?m(d)-m(c):m(c)-m(d)})}),b}var c,d,e,f,g,h={top:0,right:0,bottom:0,left:0},i=null,j=null,k=Math.floor(1e4*Math.random()),l=d3.scale.linear(),m=d3.scale.linear(),n=function(a){return a.x},o=function(a){return a.y},p=function(a){return a.open},q=function(a){return a.close},r=function(a){return a.high},s=function(a){return a.low},t=[],u=[],v=!1,w=!0,x=a.utils.defaultColor(),y=!1
 ,z=d3.dispatch("stateChange","changeState","renderEnd","chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout","elementMousemove");return b.highlightPoint=function(a,d){b.clearHighlights(),c.select(".nv-candlestickBar .nv-tick-0-"+a).classed("hover",d)},b.clearHighlights=function(){c.select(".nv-candlestickBar .nv-tick.hover").classed("hover",!1)},b.dispatch=z,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return i},set:function(a){i=a}},height:{get:function(){return j},set:function(a){j=a}},xScale:{get:function(){return l},set:function(a){l=a}},yScale:{get:function(){return m},set:function(a){m=a}},xDomain:{get:function(){return d},set:function(a){d=a}},yDomain:{get:function(){return e},set:function(a){e=a}},xRange:{get:function(){return f},set:function(a){f=a}},yRange:{get:function(){return g},set:function(a){g=a}},forceX:{get:function(){return t},set:function(a){t=a}},forceY:{get:function(){return u},set:functio
 n(a){u=a}},padData:{get:function(){return v},set:function(a){v=a}},clipEdge:{get:function(){return w},set:function(a){w=a}},id:{get:function(){return k},set:function(a){k=a}},interactive:{get:function(){return y},set:function(a){y=a}},x:{get:function(){return n},set:function(a){n=a}},y:{get:function(){return o},set:function(a){o=a}},open:{get:function(){return p()},set:function(a){p=a}},close:{get:function(){return q()},set:function(a){q=a}},high:{get:function(){return r},set:function(a){r=a}},low:{get:function(){return s},set:function(a){s=a}},margin:{get:function(){return h},set:function(a){h.top=void 0!=a.top?a.top:h.top,h.right=void 0!=a.right?a.right:h.right,h.bottom=void 0!=a.bottom?a.bottom:h.bottom,h.left=void 0!=a.left?a.left:h.left}},color:{get:function(){return x},set:function(b){x=a.utils.getColor(b)}}}),a.utils.initOptions(b),b},a.models.cumulativeLineChart=function(){"use strict";function b(l){return I.reset(),I.models(f),s&&I.models(g),t&&I.models(h),l.each(function(l
 ){function B(a,c){d3.select(b.container).style("cursor","ew-resize")}function F(a,b){H.x=d3.event.x,H.i=Math.round(G.invert(H.x)),L()}function I(a,c){d3.select(b.container).style("cursor","auto"),z.index=H.i,D.stateChange(z)}function L(){ba.data([H]);var a=b.duration();b.duration(0),b.update(),b.duration(a)}var M=d3.select(this);a.utils.initSVG(M),M.classed("nv-chart-"+y,!0);var N=a.utils.availableWidth(p,M,m),O=a.utils.availableHeight(q,M,m);if(b.update=function(){0===E?M.call(b):M.transition().duration(E).call(b)},b.container=this,z.setter(K(l),b.update).getter(J(l)).update(),z.disabled=l.map(function(a){return!!a.disabled}),!A){var P;A={};for(P in z)z[P]instanceof Array?A[P]=z[P].slice(0):A[P]=z[P]}var Q=d3.behavior.drag().on("dragstart",B).on("drag",F).on("dragend",I);if(!(l&&l.length&&l.filter(function(a){return a.values.length}).length))return a.utils.noData(b,M),b;if(M.selectAll(".nv-noData").remove(),d=f.xScale(),e=f.yScale(),x)f.yDomain(null);else{var R=l.filter(function(a)
 {return!a.disabled}).map(function(a,b){var c=d3.extent(a.values,f.y());return c[0]<-.95&&(c[0]=-.95),[(c[0]-c[1])/(1+c[1]),(c[1]-c[0])/(1+c[0])]}),S=[d3.min(R,function(a){return a[0]}),d3.max(R,function(a){return a[1]})];f.yDomain(S)}G.domain([0,l[0].values.length-1]).range([0,N]).clamp(!0);var l=c(H.i,l),T=w?"none":"all",U=M.selectAll("g.nv-wrap.nv-cumulativeLine").data([l]),V=U.enter().append("g").attr("class","nvd3 nv-wrap nv-cumulativeLine").append("g"),W=U.select("g");if(V.append("g").attr("class","nv-interactive"),V.append("g").attr("class","nv-x nv-axis").style("pointer-events","none"),V.append("g").attr("class","nv-y nv-axis"),V.append("g").attr("class","nv-background"),V.append("g").attr("class","nv-linesWrap").style("pointer-events",T),V.append("g").attr("class","nv-avgLinesWrap").style("pointer-events","none"),V.append("g").attr("class","nv-legendWrap"),V.append("g").attr("class","nv-controlsWrap"),r?(i.width(N),W.select(".nv-legendWrap").datum(l).call(i),n||i.height()===
 m.top||(m.top=i.height(),O=a.utils.availableHeight(q,M,m)),W.select(".nv-legendWrap").attr("transform","translate(0,"+-m.top+")")):W.select(".nv-legendWrap").selectAll("*").remove(),v){var X=[{key:"Re-scale y-axis",disabled:!x}];j.width(140).color(["#444","#444","#444"]).rightAlign(!1).margin({top:5,right:0,bottom:5,left:20}),W.select(".nv-controlsWrap").datum(X).attr("transform","translate(0,"+-m.top+")").call(j)}else W.select(".nv-controlsWrap").selectAll("*").remove();U.attr("transform","translate("+m.left+","+m.top+")"),u&&W.select(".nv-y.nv-axis").attr("transform","translate("+N+",0)");var Y=l.filter(function(a){return a.tempDisabled});U.select(".tempDisabled").remove(),Y.length&&U.append("text").attr("class","tempDisabled").attr("x",N/2).attr("y","-.71em").style("text-anchor","end").text(Y.map(function(a){return a.key}).join(", ")+" values cannot be calculated for this time period."),w&&(k.width(N).height(O).margin({left:m.left,top:m.top}).svgContainer(M).xScale(d),U.select(".
 nv-interactive").call(k)),V.select(".nv-background").append("rect"),W.select(".nv-background rect").attr("width",N).attr("height",O),f.y(function(a){return a.display.y}).width(N).height(O).color(l.map(function(a,b){return a.color||o(a,b)}).filter(function(a,b){return!l[b].disabled&&!l[b].tempDisabled}));var Z=W.select(".nv-linesWrap").datum(l.filter(function(a){return!a.disabled&&!a.tempDisabled}));Z.call(f),l.forEach(function(a,b){a.seriesIndex=b});var $=l.filter(function(a){return!a.disabled&&!!C(a)}),_=W.select(".nv-avgLinesWrap").selectAll("line").data($,function(a){return a.key}),aa=function(a){var b=e(C(a));return 0>b?0:b>O?O:b};_.enter().append("line").style("stroke-width",2).style("stroke-dasharray","10,10").style("stroke",function(a,b){return f.color()(a,a.seriesIndex)}).attr("x1",0).attr("x2",N).attr("y1",aa).attr("y2",aa),_.style("stroke-opacity",function(a){var b=e(C(a));return 0>b||b>O?0:1}).attr("x1",0).attr("x2",N).attr("y1",aa).attr("y2",aa),_.exit().remove();var ba=
 Z.selectAll(".nv-indexLine").data([H]);ba.enter().append("rect").attr("class","nv-indexLine").attr("width",3).attr("x",-2).attr("fill","red").attr("fill-opacity",.5).style("pointer-events","all").call(Q),ba.attr("transform",function(a){return"translate("+G(a.i)+",0)"}).attr("height",O),s&&(g.scale(d)._ticks(a.utils.calcTicksX(N/70,l)).tickSize(-O,0),W.select(".nv-x.nv-axis").attr("transform","translate(0,"+e.range()[0]+")"),W.select(".nv-x.nv-axis").call(g)),t&&(h.scale(e)._ticks(a.utils.calcTicksY(O/36,l)).tickSize(-N,0),W.select(".nv-y.nv-axis").call(h)),W.select(".nv-background rect").on("click",function(){H.x=d3.mouse(this)[0],H.i=Math.round(G.invert(H.x)),z.index=H.i,D.stateChange(z),L()}),f.dispatch.on("elementClick",function(a){H.i=a.pointIndex,H.x=G(H.i),z.index=H.i,D.stateChange(z),L()}),j.dispatch.on("legendClick",function(a,c){a.disabled=!a.disabled,x=!a.disabled,z.rescaleY=x,D.stateChange(z),b.update()}),i.dispatch.on("stateChange",function(a){for(var c in a)z[c]=a[c];D.
 stateChange(z),b.update()}),k.dispatch.on("elementMousemove",function(c){f.clearHighlights();var d,e,i,j=[];if(l.filter(function(a,b){return a.seriesIndex=b,!a.disabled}).forEach(function(g,h){e=a.interactiveBisect(g.values,c.pointXValue,b.x()),f.highlightPoint(h,e,!0);var k=g.values[e];"undefined"!=typeof k&&("undefined"==typeof d&&(d=k),"undefined"==typeof i&&(i=b.xScale()(b.x()(k,e))),j.push({key:g.key,value:b.y()(k,e),color:o(g,g.seriesIndex)}))}),j.length>2){var m=b.yScale().invert(c.mouseY),n=Math.abs(b.yScale().domain()[0]-b.yScale().domain()[1]),p=.03*n,q=a.nearestValueIndex(j.map(function(a){return a.value}),m,p);null!==q&&(j[q].highlight=!0)}var r=g.tickFormat()(b.x()(d,e),e);k.tooltip.valueFormatter(function(a,b){return h.tickFormat()(a)}).data({value:r,series:j})(),k.renderGuideLine(i)}),k.dispatch.on("elementMouseout",function(a){f.clearHighlights()}),D.on("changeState",function(a){"undefined"!=typeof a.disabled&&(l.forEach(function(b,c){b.disabled=a.disabled[c]}),z.dis
 abled=a.disabled),"undefined"!=typeof a.index&&(H.i=a.index,H.x=G(H.i),z.index=a.index,ba.data([H])),"undefined"!=typeof a.rescaleY&&(x=a.rescaleY),b.update()})}),I.renderEnd("cumulativeLineChart immediate"),b}function c(a,b){return L||(L=f.y()),b.map(function(b,c){if(!b.values)return b;var d=b.values[a];if(null==d)return b;var e=L(d,a);return-.95>e&&!F?(b.tempDisabled=!0,b):(b.tempDisabled=!1,b.values=b.values.map(function(a,b){return a.display={y:(L(a,b)-e)/(1+e)},a}),b)})}var d,e,f=a.models.line(),g=a.models.axis(),h=a.models.axis(),i=a.models.legend(),j=a.models.legend(),k=a.interactiveGuideline(),l=a.models.tooltip(),m={top:30,right:30,bottom:50,left:60},n=null,o=a.utils.defaultColor(),p=null,q=null,r=!0,s=!0,t=!0,u=!1,v=!0,w=!1,x=!0,y=f.id(),z=a.utils.state(),A=null,B=null,C=function(a){return a.average},D=d3.dispatch("stateChange","changeState","renderEnd"),E=250,F=!1;z.index=0,z.rescaleY=x,g.orient("bottom").tickPadding(7),h.orient(u?"right":"left"),l.valueFormatter(function
 (a,b){return h.tickFormat()(a,b)}).headerFormatter(function(a,b){return g.tickFormat()(a,b)}),j.updateState(!1);var G=d3.scale.linear(),H={i:0,x:0},I=a.utils.renderWatch(D,E),J=function(a){return function(){return{active:a.map(function(a){return!a.disabled}),index:H.i,rescaleY:x}}},K=function(a){return function(b){void 0!==b.index&&(H.i=b.index),void 0!==b.rescaleY&&(x=b.rescaleY),void 0!==b.active&&a.forEach(function(a,c){a.disabled=!b.active[c]})}};f.dispatch.on("elementMouseover.tooltip",function(a){var c={x:b.x()(a.point),y:b.y()(a.point),color:a.point.color};a.point=c,l.data(a).hidden(!1)}),f.dispatch.on("elementMouseout.tooltip",function(a){l.hidden(!0)});var L=null;return b.dispatch=D,b.lines=f,b.legend=i,b.controls=j,b.xAxis=g,b.yAxis=h,b.interactiveLayer=k,b.state=z,b.tooltip=l,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return p},set:function(a){p=a}},height:{get:function(){return q},set:function(a){q=a}},rescaleY:{get:function(
 ){return x},set:function(a){x=a}},showControls:{get:function(){return v},set:function(a){v=a}},showLegend:{get:function(){return r},set:function(a){r=a}},average:{get:function(){return C},set:function(a){C=a}},defaultState:{get:function(){return A},set:function(a){A=a}},noData:{get:function(){return B},set:function(a){B=a}},showXAxis:{get:function(){return s},set:function(a){s=a}},showYAxis:{get:function(){return t},set:function(a){t=a}},noErrorCheck:{get:function(){return F},set:function(a){F=a}},margin:{get:function(){return m},set:function(a){void 0!==a.top&&(m.top=a.top,n=a.top),m.right=void 0!==a.right?a.right:m.right,m.bottom=void 0!==a.bottom?a.bottom:m.bottom,m.left=void 0!==a.left?a.left:m.left}},color:{get:function(){return o},set:function(b){o=a.utils.getColor(b),i.color(o)}},useInteractiveGuideline:{get:function(){return w},set:function(a){w=a,a===!0&&(b.interactive(!1),b.useVoronoi(!1))}},rightAlignYAxis:{get:function(){return u},set:function(a){u=a,h.orient(a?"right":"
 left")}},duration:{get:function(){return E},set:function(a){E=a,f.duration(E),g.duration(E),h.duration(E),I.reset(E)}}}),a.utils.inheritOptions(b,f),a.utils.initOptions(b),b},a.models.discreteBar=function(){"use strict";function b(m){return y.reset(),m.each(function(b){var m=k-j.left-j.right,x=l-j.top-j.bottom;c=d3.select(this),a.utils.initSVG(c),b.forEach(function(a,b){a.values.forEach(function(a){a.series=b})});var z=d&&e?[]:b.map(function(a){return a.values.map(function(a,b){return{x:p(a,b),y:q(a,b),y0:a.y0}})});n.domain(d||d3.merge(z).map(function(a){return a.x})).rangeBands(f||[0,m],.1),o.domain(e||d3.extent(d3.merge(z).map(function(a){return a.y}).concat(r))),t?o.range(g||[x-(o.domain()[0]<0?12:0),o.domain()[1]>0?12:0]):o.range(g||[x,0]),h=h||n,i=i||o.copy().range([o(0),o(0)]);var A=c.selectAll("g.nv-wrap.nv-discretebar").data([b]),B=A.enter().append("g").attr("class","nvd3 nv-wrap nv-discretebar"),C=B.append("g");A.select("g");C.append("g").attr("class","nv-groups"),A.attr("t
 ransform","translate("+j.left+","+j.top+")");var D=A.select(".nv-groups").selectAll(".nv-group").data(function(a){return a},function(a){return a.key});D.enter().append("g").style("stroke-opacity",1e-6).style("fill-opacity",1e-6),D.exit().watchTransition(y,"discreteBar: exit groups").style("stroke-opacity",1e-6).style("fill-opacity",1e-6).remove(),D.attr("class",function(a,b){return"nv-group nv-series-"+b}).classed("hover",function(a){return a.hover}),D.watchTransition(y,"discreteBar: groups").style("stroke-opacity",1).style("fill-opacity",.75);var E=D.selectAll("g.nv-bar").data(function(a){return a.values});E.exit().remove();var F=E.enter().append("g").attr("transform",function(a,b,c){return"translate("+(n(p(a,b))+.05*n.rangeBand())+", "+o(0)+")"}).on("mouseover",function(a,b){d3.select(this).classed("hover",!0),v.elementMouseover({data:a,index:b,color:d3.select(this).style("fill")})}).on("mouseout",function(a,b){d3.select(this).classed("hover",!1),v.elementMouseout({data:a,index:b,
 color:d3.select(this).style("fill")})}).on("mousemove",function(a,b){v.elementMousemove({data:a,index:b,color:d3.select(this).style("fill")})}).on("click",function(a,b){var c=this;v.elementClick({data:a,index:b,color:d3.select(this).style("fill"),event:d3.event,element:c}),d3.event.stopPropagation()}).on("dblclick",function(a,b){v.elementDblClick({data:a,index:b,color:d3.select(this).style("fill")}),d3.event.stopPropagation()});F.append("rect").attr("height",0).attr("width",.9*n.rangeBand()/b.length),t?(F.append("text").attr("text-anchor","middle"),E.select("text").text(function(a,b){return u(q(a,b))}).watchTransition(y,"discreteBar: bars text").attr("x",.9*n.rangeBand()/2).attr("y",function(a,b){return q(a,b)<0?o(q(a,b))-o(0)+12:-4})):E.selectAll("text").remove(),E.attr("class",function(a,b){return q(a,b)<0?"nv-bar negative":"nv-bar positive"}).style("fill",function(a,b){return a.color||s(a,b)}).style("stroke",function(a,b){return a.color||s(a,b)}).select("rect").attr("class",w).wa
 tchTransition(y,"discreteBar: bars rect").attr("width",.9*n.rangeBand()/b.length),E.watchTransition(y,"discreteBar: bars").attr("transform",function(a,b){var c=n(p(a,b))+.05*n.rangeBand(),d=q(a,b)<0?o(0):o(0)-o(q(a,b))<1?o(0)-1:o(q(a,b));return"translate("+c+", "+d+")"}).select("rect").attr("height",function(a,b){return Math.max(Math.abs(o(q(a,b))-o(0)),1)}),h=n.copy(),i=o.copy()}),y.renderEnd("discreteBar immediate"),b}var c,d,e,f,g,h,i,j={top:0,right:0,bottom:0,left:0},k=960,l=500,m=Math.floor(1e4*Math.random()),n=d3.scale.ordinal(),o=d3.scale.linear(),p=function(a){return a.x},q=function(a){return a.y},r=[0],s=a.utils.defaultColor(),t=!1,u=d3.format(",.2f"),v=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout","elementMousemove","renderEnd"),w="discreteBar",x=250,y=a.utils.renderWatch(v,x);return b.dispatch=v,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return k},set:function(a){k=a}},height:{g
 et:function(){return l},set:function(a){l=a}},forceY:{get:function(){return r},set:function(a){r=a;
-}},showValues:{get:function(){return t},set:function(a){t=a}},x:{get:function(){return p},set:function(a){p=a}},y:{get:function(){return q},set:function(a){q=a}},xScale:{get:function(){return n},set:function(a){n=a}},yScale:{get:function(){return o},set:function(a){o=a}},xDomain:{get:function(){return d},set:function(a){d=a}},yDomain:{get:function(){return e},set:function(a){e=a}},xRange:{get:function(){return f},set:function(a){f=a}},yRange:{get:function(){return g},set:function(a){g=a}},valueFormat:{get:function(){return u},set:function(a){u=a}},id:{get:function(){return m},set:function(a){m=a}},rectClass:{get:function(){return w},set:function(a){w=a}},margin:{get:function(){return j},set:function(a){j.top=void 0!==a.top?a.top:j.top,j.right=void 0!==a.right?a.right:j.right,j.bottom=void 0!==a.bottom?a.bottom:j.bottom,j.left=void 0!==a.left?a.left:j.left}},color:{get:function(){return s},set:function(b){s=a.utils.getColor(b)}},duration:{get:function(){return x},set:function(a){x=a,
 y.reset(x)}}}),a.utils.initOptions(b),b},a.models.discreteBarChart=function(){"use strict";function b(i){return y.reset(),y.models(e),p&&y.models(f),q&&y.models(g),i.each(function(i){var n=d3.select(this);a.utils.initSVG(n);var v=a.utils.availableWidth(l,n,j),y=a.utils.availableHeight(m,n,j);if(b.update=function(){w.beforeUpdate(),n.transition().duration(x).call(b)},b.container=this,!(i&&i.length&&i.filter(function(a){return a.values.length}).length))return a.utils.noData(b,n),b;n.selectAll(".nv-noData").remove(),c=e.xScale(),d=e.yScale().clamp(!0);var z=n.selectAll("g.nv-wrap.nv-discreteBarWithAxes").data([i]),A=z.enter().append("g").attr("class","nvd3 nv-wrap nv-discreteBarWithAxes").append("g"),B=A.append("defs"),C=z.select("g");A.append("g").attr("class","nv-x nv-axis"),A.append("g").attr("class","nv-y nv-axis").append("g").attr("class","nv-zeroLine").append("line"),A.append("g").attr("class","nv-barsWrap"),A.append("g").attr("class","nv-legendWrap"),C.attr("transform","translat
 e("+j.left+","+j.top+")"),o?(h.width(v),C.select(".nv-legendWrap").datum(i).call(h),k||h.height()===j.top||(j.top=h.height(),y=a.utils.availableHeight(m,n,j)),z.select(".nv-legendWrap").attr("transform","translate(0,"+-j.top+")")):C.select(".nv-legendWrap").selectAll("*").remove(),r&&C.select(".nv-y.nv-axis").attr("transform","translate("+v+",0)"),e.width(v).height(y);var D=C.select(".nv-barsWrap").datum(i.filter(function(a){return!a.disabled}));if(D.transition().call(e),B.append("clipPath").attr("id","nv-x-label-clip-"+e.id()).append("rect"),C.select("#nv-x-label-clip-"+e.id()+" rect").attr("width",c.rangeBand()*(s?2:1)).attr("height",16).attr("x",-c.rangeBand()/(s?1:2)),p){f.scale(c)._ticks(a.utils.calcTicksX(v/100,i)).tickSize(-y,0),C.select(".nv-x.nv-axis").attr("transform","translate(0,"+(d.range()[0]+(e.showValues()&&d.domain()[0]<0?16:0))+")"),C.select(".nv-x.nv-axis").call(f);var E=C.select(".nv-x.nv-axis").selectAll("g");s&&E.selectAll("text").attr("transform",function(a,b,
 c){return"translate(0,"+(c%2==0?"5":"17")+")"}),u&&E.selectAll(".tick text").attr("transform","rotate("+u+" 0,0)").style("text-anchor",u>0?"start":"end"),t&&C.selectAll(".tick text").call(a.utils.wrapTicks,b.xAxis.rangeBand())}q&&(g.scale(d)._ticks(a.utils.calcTicksY(y/36,i)).tickSize(-v,0),C.select(".nv-y.nv-axis").call(g)),C.select(".nv-zeroLine line").attr("x1",0).attr("x2",r?-v:v).attr("y1",d(0)).attr("y2",d(0))}),y.renderEnd("discreteBar chart immediate"),b}var c,d,e=a.models.discreteBar(),f=a.models.axis(),g=a.models.axis(),h=a.models.legend(),i=a.models.tooltip(),j={top:15,right:10,bottom:50,left:60},k=null,l=null,m=null,n=a.utils.getColor(),o=!1,p=!0,q=!0,r=!1,s=!1,t=!1,u=0,v=null,w=d3.dispatch("beforeUpdate","renderEnd"),x=250;f.orient("bottom").showMaxMin(!1).tickFormat(function(a){return a}),g.orient(r?"right":"left").tickFormat(d3.format(",.1f")),i.duration(0).headerEnabled(!1).valueFormatter(function(a,b){return g.tickFormat()(a,b)}).keyFormatter(function(a,b){return f.
 tickFormat()(a,b)});var y=a.utils.renderWatch(w,x);return e.dispatch.on("elementMouseover.tooltip",function(a){a.series={key:b.x()(a.data),value:b.y()(a.data),color:a.color},i.data(a).hidden(!1)}),e.dispatch.on("elementMouseout.tooltip",function(a){i.hidden(!0)}),e.dispatch.on("elementMousemove.tooltip",function(a){i()}),b.dispatch=w,b.discretebar=e,b.legend=h,b.xAxis=f,b.yAxis=g,b.tooltip=i,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return l},set:function(a){l=a}},height:{get:function(){return m},set:function(a){m=a}},showLegend:{get:function(){return o},set:function(a){o=a}},staggerLabels:{get:function(){return s},set:function(a){s=a}},rotateLabels:{get:function(){return u},set:function(a){u=a}},wrapLabels:{get:function(){return t},set:function(a){t=!!a}},showXAxis:{get:function(){return p},set:function(a){p=a}},showYAxis:{get:function(){return q},set:function(a){q=a}},noData:{get:function(){return v},set:function(a){v=a}},margin:{get:
 function(){return j},set:function(a){void 0!==a.top&&(j.top=a.top,k=a.top),j.right=void 0!==a.right?a.right:j.right,j.bottom=void 0!==a.bottom?a.bottom:j.bottom,j.left=void 0!==a.left?a.left:j.left}},duration:{get:function(){return x},set:function(a){x=a,y.reset(x),e.duration(x),f.duration(x),g.duration(x)}},color:{get:function(){return n},set:function(b){n=a.utils.getColor(b),e.color(n),h.color(n)}},rightAlignYAxis:{get:function(){return r},set:function(a){r=a,g.orient(a?"right":"left")}}}),a.utils.inheritOptions(b,e),a.utils.initOptions(b),b},a.models.distribution=function(){"use strict";function b(k){return m.reset(),k.each(function(b){var k=(e-("x"===g?d.left+d.right:d.top+d.bottom),"x"==g?"y":"x"),l=d3.select(this);a.utils.initSVG(l),c=c||j;var n=l.selectAll("g.nv-distribution").data([b]),o=n.enter().append("g").attr("class","nvd3 nv-distribution"),p=(o.append("g"),n.select("g"));n.attr("transform","translate("+d.left+","+d.top+")");var q=p.selectAll("g.nv-dist").data(function(
 a){return a},function(a){return a.key});q.enter().append("g"),q.attr("class",function(a,b){return"nv-dist nv-series-"+b}).style("stroke",function(a,b){return i(a,b)});var r=q.selectAll("line.nv-dist"+g).data(function(a){return a.values});r.enter().append("line").attr(g+"1",function(a,b){return c(h(a,b))}).attr(g+"2",function(a,b){return c(h(a,b))}),m.transition(q.exit().selectAll("line.nv-dist"+g),"dist exit").attr(g+"1",function(a,b){return j(h(a,b))}).attr(g+"2",function(a,b){return j(h(a,b))}).style("stroke-opacity",0).remove(),r.attr("class",function(a,b){return"nv-dist"+g+" nv-dist"+g+"-"+b}).attr(k+"1",0).attr(k+"2",f),m.transition(r,"dist").attr(g+"1",function(a,b){return j(h(a,b))}).attr(g+"2",function(a,b){return j(h(a,b))}),c=j.copy()}),m.renderEnd("distribution immediate"),b}var c,d={top:0,right:0,bottom:0,left:0},e=400,f=8,g="x",h=function(a){return a[g]},i=a.utils.defaultColor(),j=d3.scale.linear(),k=250,l=d3.dispatch("renderEnd"),m=a.utils.renderWatch(l,k);return b.opt
 ions=a.utils.optionsFunc.bind(b),b.dispatch=l,b.margin=function(a){return arguments.length?(d.top="undefined"!=typeof a.top?a.top:d.top,d.right="undefined"!=typeof a.right?a.right:d.right,d.bottom="undefined"!=typeof a.bottom?a.bottom:d.bottom,d.left="undefined"!=typeof a.left?a.left:d.left,b):d},b.width=function(a){return arguments.length?(e=a,b):e},b.axis=function(a){return arguments.length?(g=a,b):g},b.size=function(a){return arguments.length?(f=a,b):f},b.getData=function(a){return arguments.length?(h=d3.functor(a),b):h},b.scale=function(a){return arguments.length?(j=a,b):j},b.color=function(c){return arguments.length?(i=a.utils.getColor(c),b):i},b.duration=function(a){return arguments.length?(k=a,m.reset(k),b):k},b},a.models.focus=function(b){"use strict";function c(u){return t.reset(),t.models(b),m&&t.models(f),n&&t.models(g),u.each(function(t){function u(a){var b=+("e"==a),c=b?1:-1,d=z/3;return"M"+.5*c+","+d+"A6,6 0 0 "+b+" "+6.5*c+","+(d+6)+"V"+(2*d-6)+"A6,6 0 0 "+b+" "+.5*c+
 ","+2*d+"ZM"+2.5*c+","+(d+8)+"V"+(2*d-8)+"M"+4.5*c+","+(d+8)+"V"+(2*d-8)}function v(){h.empty()||h.extent(p),E.data([h.empty()?d.domain():p]).each(function(a,b){var c=d(a[0])-d.range()[0],e=y-d(a[1]);d3.select(this).select(".left").attr("width",0>c?0:c),d3.select(this).select(".right").attr("x",d(a[1])).attr("width",0>e?0:e)})}function w(a){p=h.empty()?null:h.extent();var b=h.empty()?d.domain():h.extent();r.brush({extent:b,brush:h}),v(),a&&r.onBrush(b)}var x=d3.select(this);a.utils.initSVG(x);var y=a.utils.availableWidth(k,x,i),z=l-i.top-i.bottom;c.update=function(){0===q?x.call(c):x.transition().duration(q).call(c)},c.container=this,d=b.xScale(),e=b.yScale();var A=x.selectAll("g.nv-focus").data([t]),B=A.enter().append("g").attr("class","nvd3 nv-focus").append("g"),C=A.select("g");A.attr("transform","translate("+i.left+","+i.top+")"),B.append("g").attr("class","nv-background").append("rect"),B.append("g").attr("class","nv-x nv-axis"),B.append("g").attr("class","nv-y nv-axis"),B.appe
 nd("g").attr("class","nv-contentWrap"),B.append("g").attr("class","nv-brushBackground"),B.append("g").attr("class","nv-x nv-brush"),o&&C.select(".nv-y.nv-axis").attr("transform","translate("+y+",0)"),C.select(".nv-background rect").attr("width",y).attr("height",z),b.width(y).height(z).color(t.map(function(a,b){return a.color||j(a,b)}).filter(function(a,b){return!t[b].disabled}));var D=C.select(".nv-contentWrap").datum(t.filter(function(a){return!a.disabled}));d3.transition(D).call(b),h.x(d).on("brush",function(){w(s)}),h.on("brushend",function(){s||r.onBrush(h.empty()?d.domain():h.extent())}),p&&h.extent(p);var E=C.select(".nv-brushBackground").selectAll("g").data([p||h.extent()]),F=E.enter().append("g");F.append("rect").attr("class","left").attr("x",0).attr("y",0).attr("height",z),F.append("rect").attr("class","right").attr("x",0).attr("y",0).attr("height",z);var G=C.select(".nv-x.nv-brush").call(h);G.selectAll("rect").attr("height",z),G.selectAll(".resize").append("path").attr("d"
 ,u),w(!0),C.select(".nv-background rect").attr("width",y).attr("height",z),m&&(f.scale(d)._ticks(a.utils.calcTicksX(y/100,t)).tickSize(-z,0),C.select(".nv-x.nv-axis").attr("transform","translate(0,"+e.range()[0]+")"),d3.transition(C.select(".nv-x.nv-axis")).call(f)),n&&(g.scale(e)._ticks(a.utils.calcTicksY(z/36,t)).tickSize(-y,0),d3.transition(C.select(".nv-y.nv-axis")).call(g)),C.select(".nv-x.nv-axis").attr("transform","translate(0,"+e.range()[0]+")")}),t.renderEnd("focus immediate"),c}var d,e,b=b||a.models.line(),f=a.models.axis(),g=a.models.axis(),h=d3.svg.brush(),i={top:10,right:0,bottom:30,left:0},j=a.utils.defaultColor(),k=null,l=70,m=!0,n=!1,o=!1,p=null,q=250,r=d3.dispatch("brush","onBrush","renderEnd"),s=!0;b.interactive(!1),b.pointActive(function(a){return!1});var t=a.utils.renderWatch(r,q);return c.dispatch=r,c.content=b,c.brush=h,c.xAxis=f,c.yAxis=g,c.options=a.utils.optionsFunc.bind(c),c._options=Object.create({},{width:{get:function(){return k},set:function(a){k=a}},he
 ight:{get:function(){return l},set:function(a){l=a}},showXAxis:{get:function(){return m},set:function(a){m=a}},showYAxis:{get:function(){return n},set:function(a){n=a}},brushExtent:{get:function(){return p},set:function(a){p=a}},syncBrushing:{get:function(){return s},set:function(a){s=a}},margin:{get:function(){return i},set:function(a){i.top=void 0!==a.top?a.top:i.top,i.right=void 0!==a.right?a.right:i.right,i.bottom=void 0!==a.bottom?a.bottom:i.bottom,i.left=void 0!==a.left?a.left:i.left}},duration:{get:function(){return q},set:function(a){q=a,t.reset(q),b.duration(q),f.duration(q),g.duration(q)}},color:{get:function(){return j},set:function(c){j=a.utils.getColor(c),b.color(j)}},interpolate:{get:function(){return b.interpolate()},set:function(a){b.interpolate(a)}},xTickFormat:{get:function(){return f.tickFormat()},set:function(a){f.tickFormat(a)}},yTickFormat:{get:function(){return g.tickFormat()},set:function(a){g.tickFormat(a)}},x:{get:function(){return b.x()},set:function(a){b.
 x(a)}},y:{get:function(){return b.y()},set:function(a){b.y(a)}},rightAlignYAxis:{get:function(){return o},set:function(a){o=a,g.orient(o?"right":"left")}}}),a.utils.inheritOptions(c,b),a.utils.initOptions(c),c},a.models.forceDirectedGraph=function(){"use strict";function b(g){return u.reset(),g.each(function(g){f=d3.select(this),a.utils.initSVG(f);var j=a.utils.availableWidth(d,f,c),u=a.utils.availableHeight(e,f,c);if(f.attr("width",j).attr("height",u),!(g&&g.links&&g.nodes))return a.utils.noData(b,f),b;f.selectAll(".nv-noData").remove(),f.selectAll("*").remove();var v=new Set;g.nodes.forEach(function(a){var b=Object.keys(a);b.forEach(function(a){v.add(a)})});var w=d3.layout.force().nodes(g.nodes).links(g.links).size([j,u]).linkStrength(k).friction(l).linkDistance(m).charge(n).gravity(o).theta(p).alpha(q).start(),x=f.selectAll(".link").data(g.links).enter().append("line").attr("class","nv-force-link").style("stroke-width",function(a){return Math.sqrt(a.value)}),y=f.selectAll(".node"
 ).data(g.nodes).enter().append("g").attr("class","nv-force-node").call(w.drag);y.append("circle").attr("r",r).style("fill",function(a){return h(a)}).on("mouseover",function(a){f.select(".nv-series-"+a.seriesIndex+" .nv-distx-"+a.pointIndex).attr("y1",a.py),f.select(".nv-series-"+a.seriesIndex+" .nv-disty-"+a.pointIndex).attr("x2",a.px);var b=h(a);a.series=[],v.forEach(function(c){a.series.push({color:b,key:c,value:a[c]})}),i.data(a).hidden(!1)}).on("mouseout",function(a){i.hidden(!0)}),i.headerFormatter(function(a){return"Node"}),t(x),s(y),w.on("tick",function(){x.attr("x1",function(a){return a.source.x}).attr("y1",function(a){return a.source.y}).attr("x2",function(a){return a.target.x}).attr("y2",function(a){return a.target.y}),y.attr("transform",function(a){return"translate("+a.x+", "+a.y+")"})})}),b}var c={top:2,right:0,bottom:2,left:0},d=400,e=32,f=null,g=d3.dispatch("renderEnd"),h=a.utils.getColor(["#000"]),i=a.models.tooltip(),j=null,k=.1,l=.9,m=30,n=-120,o=.1,p=.8,q=.1,r=5,s=
 function(a){},t=function(a){},u=a.utils.renderWatch(g);return b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return d},set:function(a){d=a}},height:{get:function(){return e},set:function(a){e=a}},linkStrength:{get:function(){return k},set:function(a){k=a}},friction:{get:function(){return l},set:function(a){l=a}},linkDist:{get:function(){return m},set:function(a){m=a}},charge:{get:function(){return n},set:function(a){n=a}},gravity:{get:function(){return o},set:function(a){o=a}},theta:{get:function(){return p},set:function(a){p=a}},alpha:{get:function(){return q},set:function(a){q=a}},radius:{get:function(){return r},set:function(a){r=a}},x:{get:function(){return getX},set:function(a){getX=d3.functor(a)}},y:{get:function(){return getY},set:function(a){getY=d3.functor(a)}},margin:{get:function(){return c},set:function(a){c.top=void 0!==a.top?a.top:c.top,c.right=void 0!==a.right?a.right:c.right,c.bottom=void 0!==a.bottom?a.bottom:c.bottom,c.left
 =void 0!==a.left?a.left:c.left}},color:{get:function(){return h},set:function(b){h=a.utils.getColor(b)}},noData:{get:function(){return j},set:function(a){j=a}},nodeExtras:{get:function(){return s},set:function(a){s=a}},linkExtras:{get:function(){return t},set:function(a){t=a}}}),b.dispatch=g,b.tooltip=i,a.utils.initOptions(b),b},a.models.furiousLegend=function(){"use strict";function b(r){function s(a,b){return"furious"!=q?"#000":o?a.disengaged?h(a,b):"#fff":o?void 0:a.disabled?h(a,b):"#fff"}function t(a,b){return o&&"furious"==q?a.disengaged?"#fff":h(a,b):a.disabled?"#fff":h(a,b)}return r.each(function(b){var r=d-c.left-c.right,u=d3.select(this);a.utils.initSVG(u);var v=u.selectAll("g.nv-legend").data([b]),w=(v.enter().append("g").attr("class","nvd3 nv-legend").append("g"),v.select("g"));v.attr("transform","translate("+c.left+","+c.top+")");var x,y=w.selectAll(".nv-series").data(function(a){return"furious"!=q?a:a.filter(function(a){return o?!0:!a.disengaged})}),z=y.enter().append("
 g").attr("class","nv-series");if("classic"==q)z.append("circle").style("stroke-width",2).attr("class","nv-legend-symbol").attr("r",5),x=y.select("circle");else if("furious"==q){z.append("rect").style("stroke-width",2).attr("class","nv-legend-symbol").attr("rx",3).attr("ry",3),x=y.select("rect"),z.append("g").attr("class","nv-check-box").property("innerHTML",'<path d="M0.5,5 L22.5,5 L22.5,26.5 L0.5,26.5 L0.5,5 Z" class="nv-box"></path><path d="M5.5,12.8618467 L11.9185089,19.2803556 L31,0.198864511" class="nv-check"></path>').attr("transform","translate(-10,-8)scale(0.5)");var A=y.select(".nv-check-box");A.each(function(a,b){d3.select(this).selectAll("path").attr("stroke",s(a,b))})}z.append("text").attr("text-anchor","start").attr("class","nv-legend-text").attr("dy",".32em").attr("dx","8");var B=y.select("text.nv-legend-text");y.on("mouseover",function(a,b){p.legendMouseover(a,b)}).on("mouseout",function(a,b){p.legendMouseout(a,b)}).on("click",function(a,b){p.legendClick(a,b);var c=y.
 data();if(m){if("classic"==q)n?(c.forEach(function(a){a.disabled=!0}),a.disabled=!1):(a.disabled=!a.disabled,c.every(function(a){return a.disabled})&&c.forEach(function(a){a.disabled=!1}));else if("furious"==q)if(o)a.disengaged=!a.disengaged,a.userDisabled=void 0==a.userDisabled?!!a.disabled:a.userDisabled,a.disabled=a.disengaged||a.userDisabled;else if(!o){a.disabled=!a.disabled,a.userDisabled=a.disabled;var d=c.filter(function(a){return!a.disengaged});d.every(function(a){return a.userDisabled})&&c.forEach(function(a){a.disabled=a.userDisabled=!1})}p.stateChange({disabled:c.map(function(a){return!!a.disabled}),disengaged:c.map(function(a){return!!a.disengaged})})}}).on("dblclick",function(a,b){if(("furious"!=q||!o)&&(p.legendDblclick(a,b),m)){var c=y.data();c.forEach(function(a){a.disabled=!0,"furious"==q&&(a.userDisabled=a.disabled)}),a.disabled=!1,"furious"==q&&(a.userDisabled=a.disabled),p.stateChange({disabled:c.map(function(a){return!!a.disabled})})}}),y.classed("nv-disabled",
 function(a){return a.userDisabled}),y.exit().remove(),B.attr("fill",s).text(function(a){return g(f(a))});var C;switch(q){case"furious":C=23;break;case"classic":C=20}if(j){var D=[];y.each(function(b,c){var d;if(g(f(b))&&g(f(b)).length>i){var e=g(f(b)).substring(0,i);d=d3.select(this).select("text").text(e+"..."),d3.select(this).append("svg:title").text(g(f(b)))}else d=d3.select(this).select("text");var h;try{if(h=d.node().getComputedTextLength(),0>=h)throw Error()}catch(j){h=a.utils.calcApproxTextWidth(d)}D.push(h+k)});for(var E=0,F=0,G=[];r>F&&E<D.length;)G[E]=D[E],F+=D[E++];for(0===E&&(E=1);F>r&&E>1;){G=[],E--;for(var H=0;H<D.length;H++)D[H]>(G[H%E]||0)&&(G[H%E]=D[H]);F=G.reduce(function(a,b,c,d){return a+b})}for(var I=[],J=0,K=0;E>J;J++)I[J]=K,K+=G[J];y.attr("transform",function(a,b){return"translate("+I[b%E]+","+(5+Math.floor(b/E)*C)+")"}),l?w.attr("transform","translate("+(d-c.right-F)+","+c.top+")"):w.attr("transform","translate(0,"+c.top+")"),e=c.top+c.bottom+Math.ceil(D.lengt
 h/E)*C}else{var L,M=5,N=5,O=0;y.attr("transform",function(a,b){var e=d3.select(this).select("text").node().getComputedTextLength()+k;return L=N,d<c.left+c.right+L+e&&(N=L=5,M+=C),N+=e,N>O&&(O=N),"translate("+L+","+M+")"}),w.attr("transform","translate("+(d-c.right-O)+","+c.top+")"),e=c.top+c.bottom+M+15}"furious"==q&&x.attr("width",function(a,b){return B[0][b].getComputedTextLength()+27}).attr("height",18).attr("y",-9).attr("x",-15),x.style("fill",t).style("stroke",function(a,b){return a.color||h(a,b)})}),b}var c={top:5,right:0,bottom:5,left:0},d=400,e=20,f=function(a){return a.key},g=function(a){return a},h=a.utils.getColor(),i=20,j=!0,k=28,l=!0,m=!0,n=!1,o=!1,p=d3.dispatch("legendClick","legendDblclick","legendMouseover","legendMouseout","stateChange"),q="classic";return b.dispatch=p,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return d},set:function(a){d=a}},height:{get:function(){return e},set:function(a){e=a}},key:{get:function(){retu
 rn f},set:function(a){f=a}},keyFormatter:{get:function(){return g},set:function(a){g=a}},align:{get:function(){return j},set:function(a){j=a}},rightAlign:{get:function(){return l},set:function(a){l=a}},maxKeyLength:{get:function(){return i},set:function(a){i=a}},padding:{get:function(){return k},set:function(a){k=a}},updateState:{get:function(){return m},set:function(a){m=a}},radioButtonMode:{get:function(){return n},set:function(a){n=a}},expanded:{get:function(){return o},set:function(a){o=a}},vers:{get:function(){return q},set:function(a){q=a}},margin:{get:function(){return c},set:function(a){c.top=void 0!==a.top?a.top:c.top,c.right=void 0!==a.right?a.right:c.right,c.bottom=void 0!==a.bottom?a.bottom:c.bottom,c.left=void 0!==a.left?a.left:c.left}},color:{get:function(){return h},set:function(b){h=a.utils.getColor(b)}}}),a.utils.initOptions(b),b},a.models.historicalBar=function(){"use strict";function b(x){return x.each(function(b){w.reset(),k=d3.select(this);var x=a.utils.availabl
 eWidth(h,k,g),y=a.utils.availableHeight(i,k,g);a.utils.initSVG(k),l.domain(c||d3.extent(b[0].values.map(n).concat(p))),r?l.range(e||[.5*x/b[0].values.length,x*(b[0].values.length-.5)/b[0].values.length]):l.range(e||[0,x]),m.domain(d||d3.extent(b[0].values.map(o).concat(q))).range(f||[y,0]),l.domain()[0]===l.domain()[1]&&(l.domain()[0]?l.domain([l.domain()[0]-.01*l.domain()[0],l.domain()[1]+.01*l.domain()[1]]):l.domain([-1,1])),m.domain()[0]===m.domain()[1]&&(m.domain()[0]?m.domain([m.domain()[0]+.01*m.domain()[0],m.domain()[1]-.01*m.domain()[1]]):m.domain([-1,1]));var z=k.selectAll("g.nv-wrap.nv-historicalBar-"+j).data([b[0].values]),A=z.enter().append("g").attr("class","nvd3 nv-wrap nv-historicalBar-"+j),B=A.append("defs"),C=A.append("g"),D=z.select("g");C.append("g").attr("class","nv-bars"),z.attr("transform","translate("+g.left+","+g.top+")"),k.on("click",function(a,b){u.chartClick({data:a,index:b,pos:d3.event,id:j})}),B.append("clipPath").attr("id","nv-chart-clip-path-"+j).appen
 d("rect"),z.select("#nv-chart-clip-path-"+j+" rect").attr("width",x).attr("height",y),D.attr("clip-path",s?"url(#nv-chart-clip-path-"+j+")":"");var E=z.select(".nv-bars").selectAll(".nv-bar").data(function(a){return a},function(a,b){return n(a,b)});E.exit().remove(),E.enter().append("rect").attr("x",0).attr("y",function(b,c){return a.utils.NaNtoZero(m(Math.max(0,o(b,c))))}).attr("height",function(b,c){return a.utils.NaNtoZero(Math.abs(m(o(b,c))-m(0)))}).attr("transform",function(a,c){return"translate("+(l(n(a,c))-x/b[0].values.length*.45)+",0)"}).on("mouseover",function(a,b){v&&(d3.select(this).classed("hover",!0),u.elementMouseover({data:a,index:b,color:d3.select(this).style("fill")}))}).on("mouseout",function(a,b){v&&(d3.select(this).classed("hover",!1),u.elementMouseout({data:a,index:b,color:d3.select(this).style("fill")}))}).on("mousemove",function(a,b){v&&u.elementMousemove({data:a,index:b,color:d3.select(this).style("fill")})}).on("click",function(a,b){if(v){var c=this;u.eleme
 ntClick({data:a,index:b,color:d3.select(this).style("fill"),event:d3.event,element:c}),d3.event.stopPropagation()}}).on("dblclick",function(a,b){v&&(u.elementDblClick({data:a,index:b,color:d3.select(this).style("fill")}),d3.event.stopPropagation())}),E.attr("fill",function(a,b){return t(a,b)}).attr("class",function(a,b,c){return(o(a,b)<0?"nv-bar negative":"nv-bar positive")+" nv-bar-"+c+"-"+b}).watchTransition(w,"bars").attr("transform",function(a,c){return"translate("+(l(n(a,c))-x/b[0].values.length*.45)+",0)"}).attr("width",x/b[0].values.length*.9),E.watchTransition(w,"bars").attr("y",function(b,c){var d=o(b,c)<0?m(0):m(0)-m(o(b,c))<1?m(0)-1:m(o(b,c));return a.utils.NaNtoZero(d)}).attr("height",function(b,c){return a.utils.NaNtoZero(Math.max(Math.abs(m(o(b,c))-m(0)),1))})}),w.renderEnd("historicalBar immediate"),b}var c,d,e,f,g={top:0,right:0,bottom:0,left:0},h=null,i=null,j=Math.floor(1e4*Math.random()),k=null,l=d3.scale.linear(),m=d3.scale.linear(),n=function(a){return a.x},o=fu
 nction(a){return a.y},p=[],q=[0],r=!1,s=!0,t=a.utils.defaultColor(),u=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout","elementMousemove","renderEnd"),v=!0,w=a.utils.renderWatch(u,0);return b.highlightPoint=function(a,b){k.select(".nv-bars .nv-bar-0-"+a).classed("hover",b)},b.clearHighlights=function(){k.select(".nv-bars .nv-bar.hover").classed("hover",!1)},b.dispatch=u,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return h},set:function(a){h=a}},height:{get:function(){return i},set:function(a){i=a}},forceX:{get:function(){return p},set:function(a){p=a}},forceY:{get:function(){return q},set:function(a){q=a}},padData:{get:function(){return r},set:function(a){r=a}},x:{get:function(){return n},set:function(a){n=a}},y:{get:function(){return o},set:function(a){o=a}},xScale:{get:function(){return l},set:function(a){l=a}},yScale:{get:function(){retur

<TRUNCATED>

---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[36/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java
deleted file mode 100644
index af0c1db..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java
+++ /dev/null
@@ -1,289 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.cli;
-
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.cblock.client.CBlockVolumeClient;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.PrintStream;
-import java.net.InetSocketAddress;
-import java.util.Arrays;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * The command line tool class.
- */
-public class CBlockCli extends Configured implements Tool {
-
-  private static final String CREATE_VOLUME = "createVolume";
-
-  private static final String DELETE_VOLUME = "deleteVolume";
-
-  private static final String INFO_VOLUME = "infoVolume";
-
-  private static final String LIST_VOLUME = "listVolume";
-
-  private static final String SERVER_ADDR = "serverAddr";
-
-  private static final String HELP = "help";
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(CBlockCli.class);
-  private OzoneConfiguration conf;
-
-  private PrintStream printStream;
-
-  private Options options;
-
-  private BasicParser parser;
-
-  private CBlockVolumeClient localProxy;
-
-  public CBlockCli(OzoneConfiguration conf, PrintStream printStream)
-      throws IOException {
-    this.printStream = printStream;
-    this.conf = conf;
-    this.options = getOptions();
-    this.parser = new BasicParser();
-  }
-
-  public CBlockCli(OzoneConfiguration conf) throws IOException{
-    this(conf, System.out);
-  }
-
-  private CommandLine parseArgs(String[] argv)
-      throws ParseException {
-    return parser.parse(options, argv);
-  }
-
-  private static Options getOptions() {
-    Options options = new Options();
-    Option serverAddress = OptionBuilder
-        .withArgName("serverAddress>:<serverPort")
-        .withLongOpt(SERVER_ADDR)
-        .withValueSeparator(':')
-        .hasArgs(2)
-        .withDescription("specify server address:port")
-        .create("s");
-    options.addOption(serverAddress);
-
-    // taking 4 args: userName, volumeName, volumeSize, blockSize
-    Option createVolume = OptionBuilder
-        .withArgName("user> <volume> <volumeSize in [GB/TB]> <blockSize")
-        .withLongOpt(CREATE_VOLUME)
-        .withValueSeparator(' ')
-        .hasArgs(4)
-        .withDescription("create a fresh new volume")
-        .create("c");
-    options.addOption(createVolume);
-
-    // taking 2 args: userName, volumeName
-    Option deleteVolume = OptionBuilder
-        .withArgName("user> <volume")
-        .withLongOpt(DELETE_VOLUME)
-        .hasArgs(2)
-        .withDescription("delete a volume")
-        .create("d");
-    options.addOption(deleteVolume);
-
-    // taking 2 args: userName, volumeName
-    Option infoVolume = OptionBuilder
-        .withArgName("user> <volume")
-        .withLongOpt(INFO_VOLUME)
-        .hasArgs(2)
-        .withDescription("info a volume")
-        .create("i");
-    options.addOption(infoVolume);
-
-    // taking 1 arg: userName
-    Option listVolume = OptionBuilder
-        .withArgName("user")
-        .withLongOpt(LIST_VOLUME)
-        .hasOptionalArgs(1)
-        .withDescription("list all volumes")
-        .create("l");
-    options.addOption(listVolume);
-
-    Option help = OptionBuilder
-        .withLongOpt(HELP)
-        .withDescription("help")
-        .create("h");
-    options.addOption(help);
-
-    return options;
-  }
-
-  @Override
-  public int run(String[] args) throws ParseException, IOException {
-    CommandLine commandLine = parseArgs(args);
-    if (commandLine.hasOption("s")) {
-      String[] serverAddrArgs = commandLine.getOptionValues("s");
-      LOG.info("server address" + Arrays.toString(serverAddrArgs));
-      String serverHost = serverAddrArgs[0];
-      int serverPort = Integer.parseInt(serverAddrArgs[1]);
-      InetSocketAddress serverAddress =
-          new InetSocketAddress(serverHost, serverPort);
-      this.localProxy = new CBlockVolumeClient(conf, serverAddress);
-    } else {
-      this.localProxy = new CBlockVolumeClient(conf);
-    }
-
-    if (commandLine.hasOption("h")) {
-      LOG.info("help");
-      help();
-    }
-
-    if (commandLine.hasOption("c")) {
-      String[] createArgs = commandLine.getOptionValues("c");
-      LOG.info("create volume:" + Arrays.toString(createArgs));
-      createVolume(createArgs);
-    }
-
-    if (commandLine.hasOption("d")) {
-      String[] deleteArgs = commandLine.getOptionValues("d");
-      LOG.info("delete args:" + Arrays.toString(deleteArgs));
-      deleteVolume(deleteArgs);
-    }
-
-    if (commandLine.hasOption("l")) {
-      String[] listArg = commandLine.getOptionValues("l");
-      LOG.info("list args:" + Arrays.toString(listArg));
-      listVolume(listArg);
-    }
-
-    if (commandLine.hasOption("i")) {
-      String[] infoArgs = commandLine.getOptionValues("i");
-      LOG.info("info args:" + Arrays.toString(infoArgs));
-      infoVolume(infoArgs);
-    }
-    return 0;
-  }
-
-  public static void main(String[] argv) throws Exception {
-    OzoneConfiguration cblockConf = new OzoneConfiguration();
-    RPC.setProtocolEngine(cblockConf, CBlockServiceProtocolPB.class,
-        ProtobufRpcEngine.class);
-    int res = 0;
-    Tool shell = new CBlockCli(cblockConf, System.out);
-    try {
-      ToolRunner.run(shell, argv);
-    } catch (Exception ex) {
-      LOG.error(ex.toString());
-      res = 1;
-    }
-    System.exit(res);
-  }
-
-  public static long parseSize(String volumeSizeArgs) throws IOException {
-    long multiplier = 1;
-
-    Pattern p = Pattern.compile("([0-9]+)([a-zA-Z]+)");
-    Matcher m = p.matcher(volumeSizeArgs);
-
-    if (!m.find()) {
-      throw new IOException("Invalid volume size args " + volumeSizeArgs);
-    }
-
-    int size = Integer.parseInt(m.group(1));
-    String s = m.group(2);
-
-    if (s.equalsIgnoreCase("MB") ||
-        s.equalsIgnoreCase("Mi")) {
-      multiplier = 1024L * 1024;
-    } else if (s.equalsIgnoreCase("GB") ||
-        s.equalsIgnoreCase("Gi")) {
-      multiplier = 1024L * 1024 * 1024;
-    } else if (s.equalsIgnoreCase("TB") ||
-        s.equalsIgnoreCase("Ti")) {
-      multiplier = 1024L * 1024 * 1024 * 1024;
-    } else {
-      throw new IOException("Invalid volume size args " + volumeSizeArgs);
-    }
-    return size * multiplier;
-  }
-
-  private void createVolume(String[] createArgs) throws IOException {
-    String userName = createArgs[0];
-    String volumeName = createArgs[1];
-    long volumeSize = parseSize(createArgs[2]);
-    int blockSize = Integer.parseInt(createArgs[3])*1024;
-    localProxy.createVolume(userName, volumeName, volumeSize, blockSize);
-  }
-
-  private void deleteVolume(String[] deleteArgs) throws IOException {
-    String userName = deleteArgs[0];
-    String volumeName = deleteArgs[1];
-    boolean force = false;
-    if (deleteArgs.length > 2) {
-      force = Boolean.parseBoolean(deleteArgs[2]);
-    }
-    localProxy.deleteVolume(userName, volumeName, force);
-  }
-
-  private void infoVolume(String[] infoArgs) throws IOException {
-    String userName = infoArgs[0];
-    String volumeName = infoArgs[1];
-    VolumeInfo volumeInfo = localProxy.infoVolume(userName, volumeName);
-    printStream.println(volumeInfo.toString());
-  }
-
-  private void listVolume(String[] listArgs) throws IOException {
-    StringBuilder stringBuilder = new StringBuilder();
-    List<VolumeInfo> volumeResponse;
-    if (listArgs == null) {
-      volumeResponse = localProxy.listVolume(null);
-    } else {
-      volumeResponse = localProxy.listVolume(listArgs[0]);
-    }
-    for (int i = 0; i<volumeResponse.size(); i++) {
-      stringBuilder.append(
-          String.format("%s:%s\t%d\t%d", volumeResponse.get(i).getUserName(),
-              volumeResponse.get(i).getVolumeName(),
-          volumeResponse.get(i).getVolumeSize(),
-          volumeResponse.get(i).getBlockSize()));
-      if (i < volumeResponse.size() - 1) {
-        stringBuilder.append("\n");
-      }
-    }
-    printStream.println(stringBuilder);
-  }
-
-  private void help() {
-    HelpFormatter formatter = new HelpFormatter();
-    formatter.printHelp(100, "cblock", "", options, "");
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/cli/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/cli/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/cli/package-info.java
deleted file mode 100644
index b8b1889..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/cli/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.cli;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java
deleted file mode 100644
index ec38c9c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.client;
-
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.proto.CBlockServiceProtocol;
-import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
-import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolTranslator;
-import org.apache.hadoop.ipc.RPC;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * The client side implement of CBlockServiceProtocol.
- */
-@InterfaceAudience.Private
-public final class CBlockServiceProtocolClientSideTranslatorPB
-    implements CBlockServiceProtocol, ProtocolTranslator, Closeable {
-
-  private final CBlockServiceProtocolPB rpcProxy;
-
-  public CBlockServiceProtocolClientSideTranslatorPB(
-      CBlockServiceProtocolPB rpcProxy) {
-    this.rpcProxy = rpcProxy;
-  }
-
-  @Override
-  public void close() throws IOException {
-    RPC.stopProxy(rpcProxy);
-  }
-
-  @Override
-  public void createVolume(String userName, String volumeName,
-      long volumeSize, int blockSize) throws IOException {
-    CBlockServiceProtocolProtos.CreateVolumeRequestProto.Builder req =
-        CBlockServiceProtocolProtos.CreateVolumeRequestProto.newBuilder();
-    req.setUserName(userName);
-    req.setVolumeName(volumeName);
-    req.setVolumeSize(volumeSize);
-    req.setBlockSize(blockSize);
-    try {
-      rpcProxy.createVolume(null, req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-  }
-
-  @Override
-  public void deleteVolume(String userName, String volumeName, boolean force)
-      throws IOException {
-    CBlockServiceProtocolProtos.DeleteVolumeRequestProto.Builder req =
-        CBlockServiceProtocolProtos.DeleteVolumeRequestProto.newBuilder();
-    req.setUserName(userName);
-    req.setVolumeName(volumeName);
-    req.setForce(force);
-    try {
-      rpcProxy.deleteVolume(null, req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-  }
-
-  @Override
-  public Object getUnderlyingProxyObject() {
-    return rpcProxy;
-  }
-
-  @Override
-  public VolumeInfo infoVolume(String userName, String volumeName)
-      throws IOException {
-    CBlockServiceProtocolProtos.InfoVolumeRequestProto.Builder req =
-        CBlockServiceProtocolProtos.InfoVolumeRequestProto.newBuilder();
-    req.setUserName(userName);
-    req.setVolumeName(volumeName);
-    try {
-      CBlockServiceProtocolProtos.InfoVolumeResponseProto resp =
-          rpcProxy.infoVolume(null, req.build());
-      return new VolumeInfo(resp.getVolumeInfo().getUserName(),
-          resp.getVolumeInfo().getVolumeName(),
-          resp.getVolumeInfo().getVolumeSize(),
-          resp.getVolumeInfo().getBlockSize(),
-          resp.getVolumeInfo().getUsage());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-  }
-
-  @Override
-  public List<VolumeInfo> listVolume(String userName) throws IOException {
-    CBlockServiceProtocolProtos.ListVolumeRequestProto.Builder req =
-        CBlockServiceProtocolProtos.ListVolumeRequestProto.newBuilder();
-    if (userName != null) {
-      req.setUserName(userName);
-    }
-    try {
-      CBlockServiceProtocolProtos.ListVolumeResponseProto resp =
-          rpcProxy.listVolume(null, req.build());
-      List<VolumeInfo> respList = new ArrayList<>();
-      for (CBlockServiceProtocolProtos.VolumeInfoProto entry :
-          resp.getVolumeEntryList()) {
-        VolumeInfo volumeInfo = new VolumeInfo(
-            entry.getUserName(), entry.getVolumeName(), entry.getVolumeSize(),
-            entry.getBlockSize());
-        respList.add(volumeInfo);
-      }
-      return respList;
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    } catch (Exception e) {
-      throw new IOException("got" + e.getCause() + " " + e.getMessage());
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java
deleted file mode 100644
index 9ed54e3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.client;
-
-import org.apache.hadoop.cblock.CBlockConfigKeys;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.security.UserGroupInformation;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Implementation of client used by CBlock command line tool.
- */
-public class CBlockVolumeClient {
-  private final CBlockServiceProtocolClientSideTranslatorPB cblockClient;
-
-  public CBlockVolumeClient(OzoneConfiguration conf) throws IOException {
-    this(conf, null);
-  }
-
-  public CBlockVolumeClient(OzoneConfiguration conf,
-      InetSocketAddress serverAddress) throws IOException {
-    InetSocketAddress address = serverAddress != null ? serverAddress :
-        OzoneClientUtils.getCblockServiceRpcAddr(conf);
-    long version = RPC.getProtocolVersion(CBlockServiceProtocolPB.class);
-    int rpcTimeout = Math.toIntExact(
-        conf.getTimeDuration(CBlockConfigKeys.DFS_CBLOCK_RPC_TIMEOUT,
-            CBlockConfigKeys.DFS_CBLOCK_RPC_TIMEOUT_DEFAULT,
-            TimeUnit.MILLISECONDS));
-    cblockClient = new CBlockServiceProtocolClientSideTranslatorPB(
-        RPC.getProtocolProxy(CBlockServiceProtocolPB.class, version,
-            address, UserGroupInformation.getCurrentUser(), conf,
-            NetUtils.getDefaultSocketFactory(conf), rpcTimeout, RetryPolicies
-                .retryUpToMaximumCountWithFixedSleep(
-                    300, 1, TimeUnit.SECONDS)).getProxy());
-  }
-
-  public void createVolume(String userName, String volumeName,
-      long volumeSize, int blockSize) throws IOException {
-    cblockClient.createVolume(userName, volumeName,
-        volumeSize, blockSize);
-  }
-
-  public void deleteVolume(String userName, String volumeName, boolean force)
-      throws IOException {
-    cblockClient.deleteVolume(userName, volumeName, force);
-  }
-
-  public VolumeInfo infoVolume(String userName, String volumeName)
-      throws IOException {
-    return cblockClient.infoVolume(userName, volumeName);
-  }
-
-  public List<VolumeInfo> listVolume(String userName)
-      throws IOException {
-    return cblockClient.listVolume(userName);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/package-info.java
deleted file mode 100644
index 761b71e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.client;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java
deleted file mode 100644
index 8f6b82b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.exception;
-
-import java.io.IOException;
-
-/**
- * The exception class used in CBlock.
- */
-public class CBlockException extends IOException {
-  public CBlockException(String message) {
-    super(message);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/exception/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/exception/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/exception/package-info.java
deleted file mode 100644
index 268b8cb..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/exception/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.exception;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java
deleted file mode 100644
index 04fe3a4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper;
-
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.Longs;
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import org.apache.hadoop.cblock.jscsiHelper.cache.impl.AsyncBlockWriter;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.LevelDBStore;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.Paths;
-
-/**
- * The blockWriter task.
- */
-public class BlockWriterTask implements Runnable {
-  private final LogicalBlock block;
-  private int tryCount;
-  private final ContainerCacheFlusher flusher;
-  private final String dbPath;
-  private final String fileName;
-  private final int maxRetryCount;
-
-  /**
-   * Constructs a BlockWriterTask.
-   *
-   * @param block - Block Information.
-   * @param flusher - ContainerCacheFlusher.
-   */
-  public BlockWriterTask(LogicalBlock block, ContainerCacheFlusher flusher,
-      String dbPath, int tryCount, String fileName, int maxRetryCount) {
-    this.block = block;
-    this.flusher = flusher;
-    this.dbPath = dbPath;
-    this.tryCount = tryCount;
-    this.fileName = fileName;
-    this.maxRetryCount = maxRetryCount;
-  }
-
-  /**
-   * When an object implementing interface <code>Runnable</code> is used
-   * to create a thread, starting the thread causes the object's
-   * <code>run</code> method to be called in that separately executing
-   * thread.
-   * <p>
-   * The general contract of the method <code>run</code> is that it may
-   * take any action whatsoever.
-   *
-   * @see Thread#run()
-   */
-  @Override
-  public void run() {
-    String containerName = null;
-    XceiverClientSpi client = null;
-    LevelDBStore levelDBStore = null;
-    String traceID = flusher.getTraceID(new File(dbPath), block.getBlockID());
-    flusher.getLOG().debug(
-        "Writing block to remote. block ID: {}", block.getBlockID());
-    try {
-      incTryCount();
-      Pipeline pipeline = flusher.getPipeline(this.dbPath, block.getBlockID());
-      client = flusher.getXceiverClientManager().acquireClient(pipeline);
-      containerName = pipeline.getContainerName();
-      byte[] keybuf = Longs.toByteArray(block.getBlockID());
-      byte[] data;
-      long startTime = Time.monotonicNow();
-      levelDBStore = flusher.getCacheDB(this.dbPath);
-      data = levelDBStore.get(keybuf);
-      Preconditions.checkNotNull(data);
-      long endTime = Time.monotonicNow();
-      Preconditions.checkState(data.length > 0, "Block data is zero length");
-      startTime = Time.monotonicNow();
-      ContainerProtocolCalls.writeSmallFile(client, containerName,
-          Long.toString(block.getBlockID()), data, traceID);
-      endTime = Time.monotonicNow();
-      flusher.getTargetMetrics().updateContainerWriteLatency(
-          endTime - startTime);
-      flusher.getLOG().debug("Time taken for Write Small File : {} ms",
-          endTime - startTime);
-
-      flusher.incrementRemoteIO();
-
-    } catch (Exception ex) {
-      flusher.getLOG().error("Writing of block:{} failed, We have attempted " +
-              "to write this block {} times to the container {}.Trace ID:{}",
-          block.getBlockID(), this.getTryCount(), containerName, traceID, ex);
-      writeRetryBlock(block);
-      if (ex instanceof IOException) {
-        flusher.getTargetMetrics().incNumWriteIOExceptionRetryBlocks();
-      } else {
-        flusher.getTargetMetrics().incNumWriteGenericExceptionRetryBlocks();
-      }
-      if (this.getTryCount() >= maxRetryCount) {
-        flusher.getTargetMetrics().incNumWriteMaxRetryBlocks();
-      }
-    } finally {
-      flusher.incFinishCount(fileName);
-      if (levelDBStore != null) {
-        flusher.releaseCacheDB(dbPath);
-      }
-      if(client != null) {
-        flusher.getXceiverClientManager().releaseClient(client);
-      }
-    }
-  }
-
-
-  private void writeRetryBlock(LogicalBlock currentBlock) {
-    boolean append = false;
-    String retryFileName =
-        String.format("%s.%d.%s.%s", AsyncBlockWriter.RETRY_LOG_PREFIX,
-            currentBlock.getBlockID(), Time.monotonicNow(), tryCount);
-    File logDir = new File(this.dbPath);
-    if (!logDir.exists() && !logDir.mkdirs()) {
-      flusher.getLOG().error(
-          "Unable to create the log directory, Critical error cannot continue");
-      return;
-    }
-    String log = Paths.get(this.dbPath, retryFileName).toString();
-    ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE / Byte.SIZE);
-    buffer.putLong(currentBlock.getBlockID());
-    buffer.flip();
-    try {
-      FileChannel channel = new FileOutputStream(log, append).getChannel();
-      channel.write(buffer);
-      channel.close();
-      flusher.processDirtyBlocks(this.dbPath, retryFileName);
-    } catch (IOException e) {
-      flusher.getTargetMetrics().incNumFailedRetryLogFileWrites();
-      flusher.getLOG().error("Unable to write the retry block. Block ID: {}",
-          currentBlock.getBlockID(), e);
-    }
-  }
-
-  /**
-   * Increments the try count. This is done each time we try this block
-   * write to the container.
-   */
-  private void incTryCount() {
-    tryCount++;
-  }
-
-  /**
-   * Get the retry count.
-   *
-   * @return int
-   */
-  public int getTryCount() {
-    return tryCount;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java
deleted file mode 100644
index 84b68e3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper;
-
-import com.google.common.primitives.Longs;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.cblock.exception.CBlockException;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.proto.CBlockClientProtocol;
-import org.apache.hadoop.cblock.proto.MountVolumeResponse;
-import org.apache.hadoop.cblock.protocol.proto
-    .CBlockClientServerProtocolProtos.ContainerIDProto;
-import org.apache.hadoop.cblock.protocol.proto
-    .CBlockClientServerProtocolProtos.ListVolumesRequestProto;
-import org.apache.hadoop.cblock.protocol.proto
-    .CBlockClientServerProtocolProtos.ListVolumesResponseProto;
-import org.apache.hadoop.cblock.protocol.proto
-    .CBlockClientServerProtocolProtos.MountVolumeRequestProto;
-import org.apache.hadoop.cblock.protocol.proto
-    .CBlockClientServerProtocolProtos.MountVolumeResponseProto;
-import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos
-    .VolumeInfoProto;
-import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolTranslator;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-/**
- * The client side of CBlockClientProtocol.
- *
- * CBlockClientProtocol is the protocol used between cblock client side
- * and cblock manager (cblock client side is just the node where jscsi daemon
- * process runs. a machines talks to jscsi daemon for mounting a volume).
- *
- * Right now, the only communication carried by this protocol is for client side
- * to request mounting a volume.
- */
-public class CBlockClientProtocolClientSideTranslatorPB
-    implements CBlockClientProtocol, ProtocolTranslator, Closeable {
-
-  private final CBlockClientServerProtocolPB rpcProxy;
-
-  public CBlockClientProtocolClientSideTranslatorPB(
-      CBlockClientServerProtocolPB rpcProxy) {
-    this.rpcProxy = rpcProxy;
-  }
-
-
-  @Override
-  public void close() throws IOException {
-    RPC.stopProxy(rpcProxy);
-  }
-
-  @Override
-  public Object getUnderlyingProxyObject() {
-    return rpcProxy;
-  }
-
-  @Override
-  public MountVolumeResponse mountVolume(
-      String userName, String volumeName) throws IOException {
-    MountVolumeRequestProto.Builder
-        request
-        = MountVolumeRequestProto
-        .newBuilder();
-    request.setUserName(userName);
-    request.setVolumeName(volumeName);
-    try {
-      MountVolumeResponseProto resp
-          = rpcProxy.mountVolume(null, request.build());
-      if (!resp.getIsValid()) {
-        throw new CBlockException(
-            "Not a valid volume:" + userName + ":" + volumeName);
-      }
-      List<Pipeline> containerIDs = new ArrayList<>();
-      HashMap<String, Pipeline> containerPipelines = new HashMap<>();
-      if (resp.getAllContainerIDsList().size() == 0) {
-        throw new CBlockException("Mount volume request returned no container");
-      }
-      for (ContainerIDProto containerID :
-          resp.getAllContainerIDsList()) {
-        if (containerID.hasPipeline()) {
-          // it should always have a pipeline only except for tests.
-          Pipeline p = Pipeline.getFromProtoBuf(containerID.getPipeline());
-          p.setData(Longs.toByteArray(containerID.getIndex()));
-          containerIDs.add(p);
-          containerPipelines.put(containerID.getContainerID(), p);
-        } else {
-          throw new CBlockException("ContainerID does not have pipeline!");
-        }
-      }
-      return new MountVolumeResponse(
-          resp.getIsValid(),
-          resp.getUserName(),
-          resp.getVolumeName(),
-          resp.getVolumeSize(),
-          resp.getBlockSize(),
-          containerIDs,
-          containerPipelines);
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-  }
-
-  @Override
-  public List<VolumeInfo> listVolumes() throws IOException {
-    try {
-      List<VolumeInfo> result = new ArrayList<>();
-      ListVolumesResponseProto
-          listVolumesResponseProto = this.rpcProxy.listVolumes(null,
-          ListVolumesRequestProto.newBuilder()
-              .build());
-      for (VolumeInfoProto volumeInfoProto :
-          listVolumesResponseProto
-          .getVolumeEntryList()) {
-        result.add(new VolumeInfo(volumeInfoProto.getUserName(),
-            volumeInfoProto.getVolumeName(), volumeInfoProto.getVolumeSize(),
-            volumeInfoProto.getBlockSize()));
-      }
-      return result;
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java
deleted file mode 100644
index 2f35668..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java
+++ /dev/null
@@ -1,440 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.cblock.jscsiHelper.cache.CacheModule;
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.jscsi.target.storage.IStorageModule;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_TRACE_IO;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
-    .DFS_CBLOCK_TRACE_IO_DEFAULT;
-
-/**
- * The SCSI Target class for CBlockSCSIServer.
- */
-final public class CBlockIStorageImpl implements IStorageModule {
-  private static final Logger LOGGER =
-      LoggerFactory.getLogger(CBlockIStorageImpl.class);
-  private static final Logger TRACER =
-      LoggerFactory.getLogger("TraceIO");
-
-  private CacheModule cache;
-  private final long volumeSize;
-  private final int blockSize;
-  private final String userName;
-  private final String volumeName;
-  private final boolean traceEnabled;
-  private final Configuration conf;
-  private final ContainerCacheFlusher flusher;
-  private List<Pipeline> fullContainerList;
-
-  /**
-   * private: constructs a SCSI Target.
-   *
-   * @param config - config
-   * @param userName - Username
-   * @param volumeName - Name of the volume
-   * @param volumeSize - Size of the volume
-   * @param blockSize - Size of the block
-   * @param fullContainerList - Ordered list of containers that make up this
-   * volume.
-   * @param flusher - flusher which is used to flush data from
-   *                  level db cache to containers
-   * @throws IOException - Throws IOException.
-   */
-  private CBlockIStorageImpl(Configuration config, String userName,
-      String volumeName, long volumeSize, int blockSize,
-      List<Pipeline> fullContainerList, ContainerCacheFlusher flusher) {
-    this.conf = config;
-    this.userName = userName;
-    this.volumeName = volumeName;
-    this.volumeSize = volumeSize;
-    this.blockSize = blockSize;
-    this.fullContainerList = new ArrayList<>(fullContainerList);
-    this.flusher = flusher;
-    this.traceEnabled = conf.getBoolean(DFS_CBLOCK_TRACE_IO,
-        DFS_CBLOCK_TRACE_IO_DEFAULT);
-  }
-
-  /**
-   * private: initialize the cache.
-   *
-   * @param xceiverClientManager - client manager that is used for creating new
-   * connections to containers.
-   * @param metrics  - target metrics to maintain metrics for target server
-   * @throws IOException - Throws IOException.
-   */
-  private void initCache(XceiverClientManager xceiverClientManager,
-      CBlockTargetMetrics metrics) throws IOException {
-    this.cache = CBlockLocalCache.newBuilder()
-        .setConfiguration(conf)
-        .setVolumeName(this.volumeName)
-        .setUserName(this.userName)
-        .setPipelines(this.fullContainerList)
-        .setClientManager(xceiverClientManager)
-        .setBlockSize(blockSize)
-        .setVolumeSize(volumeSize)
-        .setFlusher(flusher)
-        .setCBlockTargetMetrics(metrics)
-        .build();
-    this.cache.start();
-  }
-
-  /**
-   * Gets a new builder for CBlockStorageImpl.
-   *
-   * @return builder
-   */
-  public static Builder newBuilder() {
-    return new Builder();
-  }
-
-  /**
-   * Get Cache.
-   *
-   * @return - Cache
-   */
-  public CacheModule getCache() {
-    return cache;
-  }
-
-  /**
-   * Returns block size of this volume.
-   *
-   * @return int size of block for this volume.
-   */
-  @Override
-  public int getBlockSize() {
-    return blockSize;
-  }
-
-  /**
-   * Checks the index boundary of a block address.
-   *
-   * @param logicalBlockAddress the index of the first block of data to be read
-   * or written
-   * @param transferLengthInBlocks the total number of consecutive blocks about
-   * to be read or written
-   * @return 0 == Success, 1 indicates the LBA address is out of bounds and 2
-   * indicates that LBA + transfer size is out of bounds.
-   */
-  @Override
-  public int checkBounds(long logicalBlockAddress, int transferLengthInBlocks) {
-    long sizeInBlocks = volumeSize / blockSize;
-    int res = 0;
-    if (logicalBlockAddress < 0 || logicalBlockAddress >= sizeInBlocks) {
-      res = 1;
-    }
-
-    if (transferLengthInBlocks < 0 ||
-        logicalBlockAddress + transferLengthInBlocks > sizeInBlocks) {
-      if (res == 0) {
-        res = 2;
-      }
-    }
-    return res;
-  }
-
-  /**
-   * Number of blocks that make up this volume.
-   *
-   * @return long - count of blocks.
-   */
-  @Override
-  public long getSizeInBlocks() {
-    return volumeSize / blockSize;
-  }
-
-  /**
-   * Reads the number of bytes that can be read into the bytes buffer from the
-   * location indicated.
-   *
-   * @param bytes the array into which the data will be copied will be filled
-   * with data from storage
-   * @param storageIndex the position of the first byte to be copied
-   * @throws IOException
-   */
-  @Override
-  public void read(byte[] bytes, long storageIndex) throws IOException {
-    int startingIdxInBlock = (int) storageIndex % blockSize;
-    int idxInBytes = 0;
-    if (this.traceEnabled) {
-      TRACER.info("Task=ReadStart,length={},location={}",
-          bytes.length, storageIndex);
-    }
-    while (idxInBytes < bytes.length - 1) {
-      long blockId = (storageIndex + idxInBytes) / blockSize;
-      byte[] dataBytes;
-
-      try {
-        LogicalBlock block = this.cache.get(blockId);
-        dataBytes = block.getData().array();
-
-        if (this.traceEnabled) {
-          TRACER.info("Task=ReadBlock,BlockID={},length={},SHA={}",
-              blockId,
-              dataBytes.length,
-              dataBytes.length > 0 ? DigestUtils.sha256Hex(dataBytes) : null);
-        }
-      } catch (IOException e) {
-        // For an non-existing block cache.get will return a block with zero
-        // bytes filled. So any error here is a real error.
-        LOGGER.error("getting errors when reading data:" + e);
-        throw e;
-      }
-
-      int length = blockSize - startingIdxInBlock;
-      if (length > bytes.length - idxInBytes) {
-        length = bytes.length - idxInBytes;
-      }
-      if (dataBytes.length >= length) {
-        System.arraycopy(dataBytes, startingIdxInBlock, bytes, idxInBytes,
-            length);
-      }
-      startingIdxInBlock = 0;
-      idxInBytes += length;
-    }
-    if (this.traceEnabled) {
-      TRACER.info("Task=ReadEnd,length={},location={},SHA={}",
-          bytes.length, storageIndex, DigestUtils.sha256Hex(bytes));
-    }
-  }
-
-  @Override
-  public void write(byte[] bytes, long storageIndex) throws IOException {
-    int startingIdxInBlock = (int) storageIndex % blockSize;
-    int idxInBytes = 0;
-    if (this.traceEnabled) {
-      TRACER.info("Task=WriteStart,length={},location={},SHA={}",
-          bytes.length, storageIndex,
-          bytes.length > 0 ? DigestUtils.sha256Hex(bytes) : null);
-    }
-
-    ByteBuffer dataByte = ByteBuffer.allocate(blockSize);
-    while (idxInBytes < bytes.length - 1) {
-      long blockId = (storageIndex + idxInBytes) / blockSize;
-      int length = blockSize - startingIdxInBlock;
-      if (length > bytes.length - idxInBytes) {
-        length = bytes.length - idxInBytes;
-      }
-      System.arraycopy(bytes, idxInBytes, dataByte.array(), startingIdxInBlock,
-          length);
-      this.cache.put(blockId, dataByte.array());
-
-      if (this.traceEnabled) {
-        TRACER.info("Task=WriteBlock,BlockID={},length={},SHA={}",
-            blockId, dataByte.array().length,
-            dataByte.array().length > 0 ?
-                DigestUtils.sha256Hex(dataByte.array()) : null);
-      }
-      dataByte.clear();
-      startingIdxInBlock = 0;
-      idxInBytes += length;
-    }
-
-    if (this.traceEnabled) {
-      TRACER.info("Task=WriteEnd,length={},location={} ",
-          bytes.length, storageIndex);
-    }
-  }
-
-  @Override
-  public void close() throws IOException {
-    try {
-      cache.close();
-    } catch (IllegalStateException ise) {
-      LOGGER.error("Can not close the storage {}", ise);
-      throw ise;
-    }
-  }
-
-  /**
-   * Builder class for CBlocklocalCache.
-   */
-  public static class Builder {
-    private String userName;
-    private String volumeName;
-    private long volumeSize;
-    private int blockSize;
-    private List<Pipeline> containerList;
-    private Configuration conf;
-    private XceiverClientManager clientManager;
-    private ContainerCacheFlusher flusher;
-    private CBlockTargetMetrics metrics;
-
-    /**
-     * Constructs a builder.
-     */
-    Builder() {
-
-    }
-
-    public Builder setFlusher(ContainerCacheFlusher cacheFlusher) {
-      this.flusher = cacheFlusher;
-      return this;
-    }
-
-    /**
-     * set config.
-     *
-     * @param config - config
-     * @return Builder
-     */
-    public Builder setConf(Configuration config) {
-      this.conf = config;
-      return this;
-    }
-
-    /**
-     * set user name.
-     *
-     * @param cblockUserName - user name
-     * @return Builder
-     */
-    public Builder setUserName(String cblockUserName) {
-      this.userName = cblockUserName;
-      return this;
-    }
-
-    /**
-     * set volume name.
-     *
-     * @param cblockVolumeName -- volume name
-     * @return Builder
-     */
-    public Builder setVolumeName(String cblockVolumeName) {
-      this.volumeName = cblockVolumeName;
-      return this;
-    }
-
-    /**
-     * set volume size.
-     *
-     * @param cblockVolumeSize -- set volume size.
-     * @return Builder
-     */
-    public Builder setVolumeSize(long cblockVolumeSize) {
-      this.volumeSize = cblockVolumeSize;
-      return this;
-    }
-
-    /**
-     * set block size.
-     *
-     * @param cblockBlockSize -- block size
-     * @return Builder
-     */
-    public Builder setBlockSize(int cblockBlockSize) {
-      this.blockSize = cblockBlockSize;
-      return this;
-    }
-
-    /**
-     * Set contianer list.
-     *
-     * @param cblockContainerList - set the pipeline list
-     * @return Builder
-     */
-    public Builder setContainerList(List<Pipeline> cblockContainerList) {
-      this.containerList = cblockContainerList;
-      return this;
-    }
-
-    /**
-     * Set client manager.
-     *
-     * @param xceiverClientManager -- sets the client manager.
-     * @return Builder
-     */
-    public Builder setClientManager(XceiverClientManager xceiverClientManager) {
-      this.clientManager = xceiverClientManager;
-      return this;
-    }
-
-    /**
-     * Set Cblock Target Metrics.
-     *
-     * @param targetMetrics -- sets the cblock target metrics
-     * @return Builder
-     */
-    public Builder setCBlockTargetMetrics(CBlockTargetMetrics targetMetrics) {
-      this.metrics = targetMetrics;
-      return this;
-    }
-
-    /**
-     * Builds the CBlockStorageImpl.
-     *
-     * @return builds the CBlock Scsi Target.
-     */
-    public CBlockIStorageImpl build() throws IOException {
-      if (StringUtils.isBlank(userName)) {
-        throw new IllegalArgumentException("User name cannot be null or empty" +
-            ".");
-      }
-      if (StringUtils.isBlank(volumeName)) {
-        throw new IllegalArgumentException("Volume name cannot be null or " +
-            "empty");
-      }
-
-      if (volumeSize < 1) {
-        throw new IllegalArgumentException("Volume size cannot be negative or" +
-            " zero.");
-      }
-
-      if (blockSize < 1) {
-        throw new IllegalArgumentException("Block size cannot be negative or " +
-            "zero.");
-      }
-
-      if (containerList == null || containerList.size() == 0) {
-        throw new IllegalArgumentException("Container list cannot be null or " +
-            "empty");
-      }
-      if (clientManager == null) {
-        throw new IllegalArgumentException("Client manager cannot be null");
-      }
-      if (conf == null) {
-        throw new IllegalArgumentException("Configuration cannot be null");
-      }
-
-      if (flusher == null) {
-        throw new IllegalArgumentException("Flusher Cannot be null.");
-      }
-      CBlockIStorageImpl impl = new CBlockIStorageImpl(this.conf, this.userName,
-          this.volumeName, this.volumeSize, this.blockSize, this.containerList,
-          this.flusher);
-      impl.initCache(this.clientManager, this.metrics);
-      return impl;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java
deleted file mode 100644
index 6367c61..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper;
-
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.proto.MountVolumeResponse;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * This class is the handler of CBlockManager used by target server
- * to communicate with CBlockManager.
- *
- * More specifically, this class will expose local methods to target
- * server, and make RPC calls to CBlockManager accordingly
- */
-public class CBlockManagerHandler {
-
-  private final CBlockClientProtocolClientSideTranslatorPB handler;
-
-  public CBlockManagerHandler(
-      CBlockClientProtocolClientSideTranslatorPB handler) {
-    this.handler = handler;
-  }
-
-  public MountVolumeResponse mountVolume(
-      String userName, String volumeName) throws IOException {
-    return handler.mountVolume(userName, volumeName);
-  }
-
-  public List<VolumeInfo> listVolumes() throws IOException {
-    return handler.listVolumes();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java
deleted file mode 100644
index e7df0cf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java
+++ /dev/null
@@ -1,334 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.cblock.jscsiHelper;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-import org.apache.hadoop.metrics2.lib.MutableRate;
-
-/**
- * This class is for maintaining  the various Cblock Target statistics
- * and publishing them through the metrics interfaces.
- * This also registers the JMX MBean for RPC.
- *
- * This class maintains stats like cache hit and miss ratio
- * as well as the latency time of read and write ops.
- */
-public class CBlockTargetMetrics {
-  // IOPS based Metrics
-  @Metric private MutableCounterLong numReadOps;
-  @Metric private MutableCounterLong numWriteOps;
-  @Metric private MutableCounterLong numReadCacheHits;
-  @Metric private MutableCounterLong numReadCacheMiss;
-  @Metric private MutableCounterLong numDirectBlockWrites;
-
-  // Cblock internal Metrics
-  @Metric private MutableCounterLong numDirtyLogBlockRead;
-  @Metric private MutableCounterLong numBytesDirtyLogRead;
-  @Metric private MutableCounterLong numBytesDirtyLogWritten;
-  @Metric private MutableCounterLong numBlockBufferFlushCompleted;
-  @Metric private MutableCounterLong numBlockBufferFlushTriggered;
-  @Metric private MutableCounterLong numBlockBufferUpdates;
-  @Metric private MutableCounterLong numRetryLogBlockRead;
-  @Metric private MutableCounterLong numBytesRetryLogRead;
-
-  // Failure Metrics
-  @Metric private MutableCounterLong numReadLostBlocks;
-  @Metric private MutableCounterLong numFailedReadBlocks;
-  @Metric private MutableCounterLong numWriteIOExceptionRetryBlocks;
-  @Metric private MutableCounterLong numWriteGenericExceptionRetryBlocks;
-  @Metric private MutableCounterLong numFailedDirectBlockWrites;
-  @Metric private MutableCounterLong numIllegalDirtyLogFiles;
-  @Metric private MutableCounterLong numFailedDirtyLogFileDeletes;
-  @Metric private MutableCounterLong numFailedBlockBufferFlushes;
-  @Metric private MutableCounterLong numInterruptedBufferWaits;
-  @Metric private MutableCounterLong numFailedRetryLogFileWrites;
-  @Metric private MutableCounterLong numWriteMaxRetryBlocks;
-  @Metric private MutableCounterLong numFailedReleaseLevelDB;
-
-  // Latency based Metrics
-  @Metric private MutableRate dbReadLatency;
-  @Metric private MutableRate containerReadLatency;
-  @Metric private MutableRate dbWriteLatency;
-  @Metric private MutableRate containerWriteLatency;
-  @Metric private MutableRate blockBufferFlushLatency;
-  @Metric private MutableRate directBlockWriteLatency;
-
-  public CBlockTargetMetrics() {
-  }
-
-  public static CBlockTargetMetrics create() {
-    MetricsSystem ms = DefaultMetricsSystem.instance();
-    return ms.register("CBlockTargetMetrics",
-        "CBlock Target Metrics",
-        new CBlockTargetMetrics());
-  }
-
-  public void incNumReadOps() {
-    numReadOps.incr();
-  }
-
-  public void incNumWriteOps() {
-    numWriteOps.incr();
-  }
-
-  public void incNumReadCacheHits() {
-    numReadCacheHits.incr();
-  }
-
-  public void incNumReadCacheMiss() {
-    numReadCacheMiss.incr();
-  }
-
-  public void incNumReadLostBlocks() {
-    numReadLostBlocks.incr();
-  }
-
-  public void incNumDirectBlockWrites() {
-    numDirectBlockWrites.incr();
-  }
-
-  public void incNumWriteIOExceptionRetryBlocks() {
-    numWriteIOExceptionRetryBlocks.incr();
-  }
-
-  public void incNumWriteGenericExceptionRetryBlocks() {
-    numWriteGenericExceptionRetryBlocks.incr();
-  }
-
-  public void incNumFailedDirectBlockWrites() {
-    numFailedDirectBlockWrites.incr();
-  }
-
-  public void incNumFailedReadBlocks() {
-    numFailedReadBlocks.incr();
-  }
-
-  public void incNumBlockBufferFlushCompleted() {
-    numBlockBufferFlushCompleted.incr();
-  }
-
-  public void incNumBlockBufferFlushTriggered() {
-    numBlockBufferFlushTriggered.incr();
-  }
-
-  public void incNumDirtyLogBlockRead() {
-    numDirtyLogBlockRead.incr();
-  }
-
-  public void incNumBytesDirtyLogRead(int bytes) {
-    numBytesDirtyLogRead.incr(bytes);
-  }
-
-  public void incNumBlockBufferUpdates() {
-    numBlockBufferUpdates.incr();
-  }
-
-  public void incNumRetryLogBlockRead() {
-    numRetryLogBlockRead.incr();
-  }
-
-  public void incNumBytesRetryLogRead(int bytes) {
-    numBytesRetryLogRead.incr(bytes);
-  }
-
-  public void incNumBytesDirtyLogWritten(int bytes) {
-    numBytesDirtyLogWritten.incr(bytes);
-  }
-
-  public void incNumFailedBlockBufferFlushes() {
-    numFailedBlockBufferFlushes.incr();
-  }
-
-  public void incNumInterruptedBufferWaits() {
-    numInterruptedBufferWaits.incr();
-  }
-
-  public void incNumIllegalDirtyLogFiles() {
-    numIllegalDirtyLogFiles.incr();
-  }
-
-  public void incNumFailedDirtyLogFileDeletes() {
-    numFailedDirtyLogFileDeletes.incr();
-  }
-
-  public void incNumFailedRetryLogFileWrites() {
-    numFailedRetryLogFileWrites.incr();
-  }
-
-  public void incNumWriteMaxRetryBlocks() {
-    numWriteMaxRetryBlocks.incr();
-  }
-
-  public void incNumFailedReleaseLevelDB() {
-    numFailedReleaseLevelDB.incr();
-  }
-
-  public void updateDBReadLatency(long latency) {
-    dbReadLatency.add(latency);
-  }
-
-  public void updateContainerReadLatency(long latency) {
-    containerReadLatency.add(latency);
-  }
-
-  public void updateDBWriteLatency(long latency) {
-    dbWriteLatency.add(latency);
-  }
-
-  public void updateContainerWriteLatency(long latency) {
-    containerWriteLatency.add(latency);
-  }
-
-  public void updateDirectBlockWriteLatency(long latency) {
-    directBlockWriteLatency.add(latency);
-  }
-
-  public void updateBlockBufferFlushLatency(long latency) {
-    blockBufferFlushLatency.add(latency);
-  }
-
-  @VisibleForTesting
-  public long getNumReadOps() {
-    return numReadOps.value();
-  }
-
-  @VisibleForTesting
-  public long getNumWriteOps() {
-    return numWriteOps.value();
-  }
-
-  @VisibleForTesting
-  public long getNumReadCacheHits() {
-    return numReadCacheHits.value();
-  }
-
-  @VisibleForTesting
-  public long getNumReadCacheMiss() {
-    return numReadCacheMiss.value();
-  }
-
-  @VisibleForTesting
-  public long getNumReadLostBlocks() {
-    return numReadLostBlocks.value();
-  }
-
-  @VisibleForTesting
-  public long getNumDirectBlockWrites() {
-    return numDirectBlockWrites.value();
-  }
-
-  @VisibleForTesting
-  public long getNumFailedDirectBlockWrites() {
-    return numFailedDirectBlockWrites.value();
-  }
-
-  @VisibleForTesting
-  public long getNumFailedReadBlocks() {
-    return numFailedReadBlocks.value();
-  }
-
-  @VisibleForTesting
-  public long getNumWriteIOExceptionRetryBlocks() {
-    return numWriteIOExceptionRetryBlocks.value();
-  }
-
-  @VisibleForTesting
-  public long getNumWriteGenericExceptionRetryBlocks() {
-    return numWriteGenericExceptionRetryBlocks.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBlockBufferFlushCompleted() {
-    return numBlockBufferFlushCompleted.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBlockBufferFlushTriggered() {
-    return numBlockBufferFlushTriggered.value();
-  }
-
-  @VisibleForTesting
-  public long getNumDirtyLogBlockRead() {
-    return numDirtyLogBlockRead.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBytesDirtyLogReads() {
-    return numBytesDirtyLogRead.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBlockBufferUpdates() {
-    return numBlockBufferUpdates.value();
-  }
-
-  @VisibleForTesting
-  public long getNumRetryLogBlockRead() {
-    return numRetryLogBlockRead.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBytesRetryLogReads() {
-    return numBytesRetryLogRead.value();
-  }
-
-  @VisibleForTesting
-  public long getNumBytesDirtyLogWritten() {
-    return numBytesDirtyLogWritten.value();
-  }
-
-  @VisibleForTesting
-  public long getNumFailedBlockBufferFlushes() {
-    return numFailedBlockBufferFlushes.value();
-  }
-
-  @VisibleForTesting
-  public long getNumInterruptedBufferWaits() {
-    return numInterruptedBufferWaits.value();
-  }
-
-  @VisibleForTesting
-  public long getNumIllegalDirtyLogFiles() {
-    return numIllegalDirtyLogFiles.value();
-  }
-
-  @VisibleForTesting
-  public long getNumFailedDirtyLogFileDeletes() {
-    return numFailedDirtyLogFileDeletes.value();
-  }
-
-  @VisibleForTesting
-  public long getNumFailedRetryLogFileWrites() {
-    return numFailedRetryLogFileWrites.value();
-  }
-
-  @VisibleForTesting
-  public long getNumWriteMaxRetryBlocks() {
-    return numWriteMaxRetryBlocks.value();
-  }
-
-  @VisibleForTesting
-  public long getNumFailedReleaseLevelDB() {
-    return numFailedReleaseLevelDB.value();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java
deleted file mode 100644
index 6c5c564..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.cblock.proto.MountVolumeResponse;
-import org.apache.hadoop.cblock.util.KeyUtil;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.jscsi.target.Configuration;
-import org.jscsi.target.Target;
-import org.jscsi.target.TargetServer;
-
-import java.io.IOException;
-import java.util.HashMap;
-
-/**
- * This class extends JSCSI target server, which is a ISCSI target that can be
- * recognized by a remote machine with ISCSI installed.
- */
-public final class CBlockTargetServer extends TargetServer {
-  private final OzoneConfiguration conf;
-  private final CBlockManagerHandler cBlockManagerHandler;
-  private final XceiverClientManager xceiverClientManager;
-  private final ContainerCacheFlusher containerCacheFlusher;
-  private final CBlockTargetMetrics metrics;
-
-  public CBlockTargetServer(OzoneConfiguration ozoneConfig,
-      Configuration jscsiConf,
-      CBlockManagerHandler cBlockManagerHandler,
-      CBlockTargetMetrics metrics)
-      throws IOException {
-    super(jscsiConf);
-    this.cBlockManagerHandler = cBlockManagerHandler;
-    this.xceiverClientManager = new XceiverClientManager(ozoneConfig);
-    this.conf = ozoneConfig;
-    this.containerCacheFlusher = new ContainerCacheFlusher(this.conf,
-        xceiverClientManager, metrics);
-    this.metrics = metrics;
-    LOGGER.info("Starting flusher thread.");
-    Thread flushListenerThread = new Thread(containerCacheFlusher);
-    flushListenerThread.setDaemon(true);
-    flushListenerThread.start();
-  }
-
-  public static void main(String[] args) throws Exception {
-  }
-
-  @Override
-  public boolean isValidTargetName(String checkTargetName) {
-    if (!KeyUtil.isValidVolumeKey(checkTargetName)) {
-      return false;
-    }
-    String userName = KeyUtil.getUserNameFromVolumeKey(checkTargetName);
-    String volumeName = KeyUtil.getVolumeFromVolumeKey(checkTargetName);
-    if (userName == null || volumeName == null) {
-      return false;
-    }
-    try {
-      MountVolumeResponse result =
-          cBlockManagerHandler.mountVolume(userName, volumeName);
-      if (!result.getIsValid()) {
-        LOGGER.error("Not a valid volume:" + checkTargetName);
-        return false;
-      }
-      String volumeKey = KeyUtil.getVolumeKey(result.getUserName(),
-          result.getVolumeName());
-      if (!targets.containsKey(volumeKey)) {
-        LOGGER.info("Mounting Volume. username: {} volume:{}",
-            userName, volumeName);
-        CBlockIStorageImpl ozoneStore = CBlockIStorageImpl.newBuilder()
-            .setUserName(userName)
-            .setVolumeName(volumeName)
-            .setVolumeSize(result.getVolumeSize())
-            .setBlockSize(result.getBlockSize())
-            .setContainerList(result.getContainerList())
-            .setClientManager(xceiverClientManager)
-            .setConf(this.conf)
-            .setFlusher(containerCacheFlusher)
-            .setCBlockTargetMetrics(metrics)
-            .build();
-        Target target = new Target(volumeKey, volumeKey, ozoneStore);
-        targets.put(volumeKey, target);
-      }
-    } catch (IOException e) {
-      LOGGER.error("Can not connect to server when validating target!"
-          + e.getMessage());
-    }
-    return targets.containsKey(checkTargetName);
-  }
-
-  @Override
-  public String[] getTargetNames() {
-    try {
-      if (cBlockManagerHandler != null) {
-        return cBlockManagerHandler.listVolumes().
-            stream().map(
-              volumeInfo -> volumeInfo.getUserName() + ":" + volumeInfo
-                .getVolumeName()).toArray(String[]::new);
-      } else {
-        return new String[0];
-      }
-    } catch (IOException e) {
-      LOGGER.error("Can't list existing volumes", e);
-      return new String[0];
-    }
-  }
-
-  @VisibleForTesting
-  public HashMap<String, Target> getTargets() {
-    return targets;
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[24/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
deleted file mode 100644
index 6321923..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
+++ /dev/null
@@ -1,539 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.protocolPB;
-
-import com.google.common.collect.Lists;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
-import org.apache.hadoop.ozone.ksm.protocol.KeySpaceManagerProtocol;
-import org.apache.hadoop.ozone.ksm.protocolPB.KeySpaceManagerProtocolPB;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.AllocateBlockRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.AllocateBlockResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.CommitKeyRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.CommitKeyResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.CreateBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.CreateBucketResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.InfoBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.InfoBucketResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.SetBucketPropertyRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.SetBucketPropertyResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.DeleteBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.DeleteBucketResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.CreateVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.CreateVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.LocateKeyRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.LocateKeyResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.KeyArgs;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.SetVolumePropertyRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.SetVolumePropertyResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.CheckVolumeAccessRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.CheckVolumeAccessResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.InfoVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.InfoVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.DeleteVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.DeleteVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ListVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ListVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ListBucketsRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ListBucketsResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ListKeysRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ListKeysResponse;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.Status;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ServiceListRequest;
-import org.apache.hadoop.ozone.protocol.proto
-    .KeySpaceManagerProtocolProtos.ServiceListResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * This class is the server-side translator that forwards requests received on
- * {@link org.apache.hadoop.ozone.ksm.protocolPB.KeySpaceManagerProtocolPB}
- * to the KeySpaceManagerService server implementation.
- */
-public class KeySpaceManagerProtocolServerSideTranslatorPB implements
-    KeySpaceManagerProtocolPB {
-  private static final Logger LOG = LoggerFactory
-      .getLogger(KeySpaceManagerProtocolServerSideTranslatorPB.class);
-  private final KeySpaceManagerProtocol impl;
-
-  /**
-   * Constructs an instance of the server handler.
-   *
-   * @param impl KeySpaceManagerProtocolPB
-   */
-  public KeySpaceManagerProtocolServerSideTranslatorPB(
-      KeySpaceManagerProtocol impl) {
-    this.impl = impl;
-  }
-
-  // Convert and exception to corresponding status code
-  private Status exceptionToResponseStatus(IOException ex) {
-    if (ex instanceof KSMException) {
-      KSMException ksmException = (KSMException)ex;
-      switch (ksmException.getResult()) {
-      case FAILED_VOLUME_ALREADY_EXISTS:
-        return Status.VOLUME_ALREADY_EXISTS;
-      case FAILED_TOO_MANY_USER_VOLUMES:
-        return Status.USER_TOO_MANY_VOLUMES;
-      case FAILED_VOLUME_NOT_FOUND:
-        return Status.VOLUME_NOT_FOUND;
-      case FAILED_VOLUME_NOT_EMPTY:
-        return Status.VOLUME_NOT_EMPTY;
-      case FAILED_USER_NOT_FOUND:
-        return Status.USER_NOT_FOUND;
-      case FAILED_BUCKET_ALREADY_EXISTS:
-        return Status.BUCKET_ALREADY_EXISTS;
-      case FAILED_BUCKET_NOT_FOUND:
-        return Status.BUCKET_NOT_FOUND;
-      case FAILED_BUCKET_NOT_EMPTY:
-        return Status.BUCKET_NOT_EMPTY;
-      case FAILED_KEY_ALREADY_EXISTS:
-        return Status.KEY_ALREADY_EXISTS;
-      case FAILED_KEY_NOT_FOUND:
-        return Status.KEY_NOT_FOUND;
-      default:
-        return Status.INTERNAL_ERROR;
-      }
-    } else {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Unknown error occurs", ex);
-      }
-      return Status.INTERNAL_ERROR;
-    }
-  }
-
-  @Override
-  public CreateVolumeResponse createVolume(
-      RpcController controller, CreateVolumeRequest request)
-      throws ServiceException {
-    CreateVolumeResponse.Builder resp = CreateVolumeResponse.newBuilder();
-    resp.setStatus(Status.OK);
-    try {
-      impl.createVolume(KsmVolumeArgs.getFromProtobuf(request.getVolumeInfo()));
-    } catch (IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-    return resp.build();
-  }
-
-  @Override
-  public SetVolumePropertyResponse setVolumeProperty(
-      RpcController controller, SetVolumePropertyRequest request)
-      throws ServiceException {
-    SetVolumePropertyResponse.Builder resp =
-        SetVolumePropertyResponse.newBuilder();
-    resp.setStatus(Status.OK);
-    String volume = request.getVolumeName();
-
-    try {
-      if (request.hasQuotaInBytes()) {
-        long quota = request.getQuotaInBytes();
-        impl.setQuota(volume, quota);
-      } else {
-        String owner = request.getOwnerName();
-        impl.setOwner(volume, owner);
-      }
-    } catch (IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-    return resp.build();
-  }
-
-  @Override
-  public CheckVolumeAccessResponse checkVolumeAccess(
-      RpcController controller, CheckVolumeAccessRequest request)
-      throws ServiceException {
-    CheckVolumeAccessResponse.Builder resp =
-        CheckVolumeAccessResponse.newBuilder();
-    resp.setStatus(Status.OK);
-    try {
-      boolean access = impl.checkVolumeAccess(request.getVolumeName(),
-          request.getUserAcl());
-      // if no access, set the response status as access denied
-      if (!access) {
-        resp.setStatus(Status.ACCESS_DENIED);
-      }
-    } catch (IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-
-    return resp.build();
-  }
-
-  @Override
-  public InfoVolumeResponse infoVolume(
-      RpcController controller, InfoVolumeRequest request)
-      throws ServiceException {
-    InfoVolumeResponse.Builder resp = InfoVolumeResponse.newBuilder();
-    resp.setStatus(Status.OK);
-    String volume = request.getVolumeName();
-    try {
-      KsmVolumeArgs ret = impl.getVolumeInfo(volume);
-      resp.setVolumeInfo(ret.getProtobuf());
-    } catch (IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-    return resp.build();
-  }
-
-  @Override
-  public DeleteVolumeResponse deleteVolume(
-      RpcController controller, DeleteVolumeRequest request)
-      throws ServiceException {
-    DeleteVolumeResponse.Builder resp = DeleteVolumeResponse.newBuilder();
-    resp.setStatus(Status.OK);
-    try {
-      impl.deleteVolume(request.getVolumeName());
-    } catch (IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-    return resp.build();
-  }
-
-  @Override
-  public ListVolumeResponse listVolumes(
-      RpcController controller, ListVolumeRequest request)
-      throws ServiceException {
-    ListVolumeResponse.Builder resp = ListVolumeResponse.newBuilder();
-    List<KsmVolumeArgs> result = Lists.newArrayList();
-    try {
-      if (request.getScope()
-          == ListVolumeRequest.Scope.VOLUMES_BY_USER) {
-        result = impl.listVolumeByUser(request.getUserName(),
-            request.getPrefix(), request.getPrevKey(), request.getMaxKeys());
-      } else if (request.getScope()
-          == ListVolumeRequest.Scope.VOLUMES_BY_CLUSTER) {
-        result = impl.listAllVolumes(request.getPrefix(), request.getPrevKey(),
-            request.getMaxKeys());
-      }
-
-      if (result == null) {
-        throw new ServiceException("Failed to get volumes for given scope "
-            + request.getScope());
-      }
-
-      result.forEach(item -> resp.addVolumeInfo(item.getProtobuf()));
-      resp.setStatus(Status.OK);
-    } catch (IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-    return resp.build();
-  }
-
-  @Override
-  public CreateBucketResponse createBucket(
-      RpcController controller, CreateBucketRequest
-      request) throws ServiceException {
-    CreateBucketResponse.Builder resp =
-        CreateBucketResponse.newBuilder();
-    try {
-      impl.createBucket(KsmBucketInfo.getFromProtobuf(
-          request.getBucketInfo()));
-      resp.setStatus(Status.OK);
-    } catch (IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-    return resp.build();
-  }
-
-  @Override
-  public InfoBucketResponse infoBucket(
-      RpcController controller, InfoBucketRequest request)
-      throws ServiceException {
-    InfoBucketResponse.Builder resp =
-        InfoBucketResponse.newBuilder();
-    try {
-      KsmBucketInfo ksmBucketInfo = impl.getBucketInfo(
-          request.getVolumeName(), request.getBucketName());
-      resp.setStatus(Status.OK);
-      resp.setBucketInfo(ksmBucketInfo.getProtobuf());
-    } catch(IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-    return resp.build();
-  }
-
-  @Override
-  public LocateKeyResponse createKey(
-      RpcController controller, LocateKeyRequest request
-  ) throws ServiceException {
-    LocateKeyResponse.Builder resp =
-        LocateKeyResponse.newBuilder();
-    try {
-      KeyArgs keyArgs = request.getKeyArgs();
-      OzoneProtos.ReplicationType type =
-          keyArgs.hasType()? keyArgs.getType() : null;
-      OzoneProtos.ReplicationFactor factor =
-          keyArgs.hasFactor()? keyArgs.getFactor() : null;
-      KsmKeyArgs ksmKeyArgs = new KsmKeyArgs.Builder()
-          .setVolumeName(keyArgs.getVolumeName())
-          .setBucketName(keyArgs.getBucketName())
-          .setKeyName(keyArgs.getKeyName())
-          .setDataSize(keyArgs.getDataSize())
-          .setType(type)
-          .setFactor(factor)
-          .build();
-      if (keyArgs.hasDataSize()) {
-        ksmKeyArgs.setDataSize(keyArgs.getDataSize());
-      } else {
-        ksmKeyArgs.setDataSize(0);
-      }
-      OpenKeySession openKey = impl.openKey(ksmKeyArgs);
-      resp.setKeyInfo(openKey.getKeyInfo().getProtobuf());
-      resp.setID(openKey.getId());
-      resp.setOpenVersion(openKey.getOpenVersion());
-      resp.setStatus(Status.OK);
-    } catch (IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-    return resp.build();
-  }
-
-  @Override
-  public LocateKeyResponse lookupKey(
-      RpcController controller, LocateKeyRequest request
-  ) throws ServiceException {
-    LocateKeyResponse.Builder resp =
-        LocateKeyResponse.newBuilder();
-    try {
-      KeyArgs keyArgs = request.getKeyArgs();
-      KsmKeyArgs ksmKeyArgs = new KsmKeyArgs.Builder()
-          .setVolumeName(keyArgs.getVolumeName())
-          .setBucketName(keyArgs.getBucketName())
-          .setKeyName(keyArgs.getKeyName())
-          .build();
-      KsmKeyInfo keyInfo = impl.lookupKey(ksmKeyArgs);
-      resp.setKeyInfo(keyInfo.getProtobuf());
-      resp.setStatus(Status.OK);
-    } catch (IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-    return resp.build();
-  }
-
-  @Override
-  public SetBucketPropertyResponse setBucketProperty(
-      RpcController controller, SetBucketPropertyRequest request)
-      throws ServiceException {
-    SetBucketPropertyResponse.Builder resp =
-        SetBucketPropertyResponse.newBuilder();
-    try {
-      impl.setBucketProperty(KsmBucketArgs.getFromProtobuf(
-          request.getBucketArgs()));
-      resp.setStatus(Status.OK);
-    } catch(IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-    return resp.build();
-  }
-
-  @Override
-  public LocateKeyResponse deleteKey(RpcController controller,
-      LocateKeyRequest request) throws ServiceException {
-    LocateKeyResponse.Builder resp =
-        LocateKeyResponse.newBuilder();
-    try {
-      KeyArgs keyArgs = request.getKeyArgs();
-      KsmKeyArgs ksmKeyArgs = new KsmKeyArgs.Builder()
-          .setVolumeName(keyArgs.getVolumeName())
-          .setBucketName(keyArgs.getBucketName())
-          .setKeyName(keyArgs.getKeyName())
-          .build();
-      impl.deleteKey(ksmKeyArgs);
-      resp.setStatus(Status.OK);
-    } catch (IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-    return resp.build();
-  }
-
-  @Override
-  public DeleteBucketResponse deleteBucket(
-      RpcController controller, DeleteBucketRequest request)
-      throws ServiceException {
-    DeleteBucketResponse.Builder resp = DeleteBucketResponse.newBuilder();
-    resp.setStatus(Status.OK);
-    try {
-      impl.deleteBucket(request.getVolumeName(), request.getBucketName());
-    } catch (IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-    return resp.build();
-  }
-
-  @Override
-  public ListBucketsResponse listBuckets(
-      RpcController controller, ListBucketsRequest request)
-      throws ServiceException {
-    ListBucketsResponse.Builder resp =
-        ListBucketsResponse.newBuilder();
-    try {
-      List<KsmBucketInfo> buckets = impl.listBuckets(
-          request.getVolumeName(),
-          request.getStartKey(),
-          request.getPrefix(),
-          request.getCount());
-      for(KsmBucketInfo bucket : buckets) {
-        resp.addBucketInfo(bucket.getProtobuf());
-      }
-      resp.setStatus(Status.OK);
-    } catch (IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-    return resp.build();
-  }
-
-  @Override
-  public ListKeysResponse listKeys(RpcController controller,
-      ListKeysRequest request) throws ServiceException {
-    ListKeysResponse.Builder resp =
-        ListKeysResponse.newBuilder();
-    try {
-      List<KsmKeyInfo> keys = impl.listKeys(
-          request.getVolumeName(),
-          request.getBucketName(),
-          request.getStartKey(),
-          request.getPrefix(),
-          request.getCount());
-      for(KsmKeyInfo key : keys) {
-        resp.addKeyInfo(key.getProtobuf());
-      }
-      resp.setStatus(Status.OK);
-    } catch (IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-    return resp.build();
-  }
-
-  @Override
-  public CommitKeyResponse commitKey(RpcController controller,
-      CommitKeyRequest request) throws ServiceException {
-    CommitKeyResponse.Builder resp =
-        CommitKeyResponse.newBuilder();
-    try {
-      KeyArgs keyArgs = request.getKeyArgs();
-      OzoneProtos.ReplicationType type =
-          keyArgs.hasType()? keyArgs.getType() : null;
-      OzoneProtos.ReplicationFactor factor =
-          keyArgs.hasFactor()? keyArgs.getFactor() : null;
-      KsmKeyArgs ksmKeyArgs = new KsmKeyArgs.Builder()
-          .setVolumeName(keyArgs.getVolumeName())
-          .setBucketName(keyArgs.getBucketName())
-          .setKeyName(keyArgs.getKeyName())
-          .setDataSize(keyArgs.getDataSize())
-          .setType(type)
-          .setFactor(factor)
-          .build();
-      int id = request.getClientID();
-      impl.commitKey(ksmKeyArgs, id);
-      resp.setStatus(Status.OK);
-    } catch (IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-    return resp.build();
-  }
-
-  @Override
-  public AllocateBlockResponse allocateBlock(RpcController controller,
-      AllocateBlockRequest request) throws ServiceException {
-    AllocateBlockResponse.Builder resp =
-        AllocateBlockResponse.newBuilder();
-    try {
-      KeyArgs keyArgs = request.getKeyArgs();
-      OzoneProtos.ReplicationType type =
-          keyArgs.hasType()? keyArgs.getType() : null;
-      OzoneProtos.ReplicationFactor factor =
-          keyArgs.hasFactor()? keyArgs.getFactor() : null;
-      KsmKeyArgs ksmKeyArgs = new KsmKeyArgs.Builder()
-          .setVolumeName(keyArgs.getVolumeName())
-          .setBucketName(keyArgs.getBucketName())
-          .setKeyName(keyArgs.getKeyName())
-          .setType(type)
-          .setFactor(factor)
-          .build();
-      int id = request.getClientID();
-      KsmKeyLocationInfo newLocation = impl.allocateBlock(ksmKeyArgs, id);
-      resp.setKeyLocation(newLocation.getProtobuf());
-      resp.setStatus(Status.OK);
-    } catch (IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-    return resp.build();
-  }
-
-  @Override
-  public ServiceListResponse getServiceList(RpcController controller,
-      ServiceListRequest request) throws ServiceException {
-    ServiceListResponse.Builder resp = ServiceListResponse.newBuilder();
-    try {
-      resp.addAllServiceInfo(impl.getServiceList().stream()
-          .map(ServiceInfo::getProtobuf)
-          .collect(Collectors.toList()));
-      resp.setStatus(Status.OK);
-    } catch (IOException e) {
-      resp.setStatus(exceptionToResponseStatus(e));
-    }
-    return resp.build();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/ScmBlockLocationProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/ScmBlockLocationProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/ScmBlockLocationProtocolServerSideTranslatorPB.java
deleted file mode 100644
index 2ab11ba..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/ScmBlockLocationProtocolServerSideTranslatorPB.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocolPB;
-
-import com.google.common.collect.Sets;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.DeleteKeyBlocksResultProto;
-import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
-import org.apache.hadoop.scm.ScmInfo;
-import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.scm.protocol.StorageContainerLocationProtocol;
-import org.apache.hadoop.scm.protocolPB.ScmBlockLocationProtocolPB;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
-import org.apache.hadoop.ozone.protocol.proto
-    .ScmBlockLocationProtocolProtos.AllocateScmBlockRequestProto;
-import org.apache.hadoop.ozone.protocol.proto
-    .ScmBlockLocationProtocolProtos.AllocateScmBlockResponseProto;
-import org.apache.hadoop.ozone.protocol.proto
-    .ScmBlockLocationProtocolProtos.DeleteScmKeyBlocksRequestProto;
-import org.apache.hadoop.ozone.protocol.proto
-    .ScmBlockLocationProtocolProtos.DeleteScmKeyBlocksResponseProto;
-import org.apache.hadoop.ozone.protocol.proto
-    .ScmBlockLocationProtocolProtos.GetScmBlockLocationsRequestProto;
-import org.apache.hadoop.ozone.protocol.proto
-    .ScmBlockLocationProtocolProtos.GetScmBlockLocationsResponseProto;
-import org.apache.hadoop.ozone.protocol.proto
-    .ScmBlockLocationProtocolProtos.ScmLocatedBlockProto;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/**
- * This class is the server-side translator that forwards requests received on
- * {@link StorageContainerLocationProtocolPB} to the
- * {@link StorageContainerLocationProtocol} server implementation.
- */
-@InterfaceAudience.Private
-public final class ScmBlockLocationProtocolServerSideTranslatorPB
-    implements ScmBlockLocationProtocolPB {
-
-  private final ScmBlockLocationProtocol impl;
-
-  /**
-   * Creates a new ScmBlockLocationProtocolServerSideTranslatorPB.
-   *
-   * @param impl {@link ScmBlockLocationProtocol} server implementation
-   */
-  public ScmBlockLocationProtocolServerSideTranslatorPB(
-      ScmBlockLocationProtocol impl) throws IOException {
-    this.impl = impl;
-  }
-
-
-  @Override
-  public GetScmBlockLocationsResponseProto getScmBlockLocations(
-      RpcController controller, GetScmBlockLocationsRequestProto req)
-      throws ServiceException {
-    Set<String> keys = Sets.newLinkedHashSetWithExpectedSize(
-        req.getKeysCount());
-    for (String key : req.getKeysList()) {
-      keys.add(key);
-    }
-    final Set<AllocatedBlock> blocks;
-    try {
-      blocks = impl.getBlockLocations(keys);
-    } catch (IOException ex) {
-      throw new ServiceException(ex);
-    }
-    GetScmBlockLocationsResponseProto.Builder resp =
-        GetScmBlockLocationsResponseProto.newBuilder();
-    for (AllocatedBlock block: blocks) {
-      ScmLocatedBlockProto.Builder locatedBlock =
-          ScmLocatedBlockProto.newBuilder()
-              .setKey(block.getKey())
-              .setPipeline(block.getPipeline().getProtobufMessage());
-      resp.addLocatedBlocks(locatedBlock.build());
-    }
-    return resp.build();
-  }
-
-  @Override
-  public AllocateScmBlockResponseProto allocateScmBlock(
-      RpcController controller, AllocateScmBlockRequestProto request)
-      throws ServiceException {
-    try {
-      AllocatedBlock allocatedBlock =
-          impl.allocateBlock(request.getSize(), request.getType(),
-              request.getFactor(), request.getOwner());
-      if (allocatedBlock != null) {
-        return
-            AllocateScmBlockResponseProto.newBuilder()
-                .setKey(allocatedBlock.getKey())
-                .setPipeline(allocatedBlock.getPipeline().getProtobufMessage())
-                .setCreateContainer(allocatedBlock.getCreateContainer())
-                .setErrorCode(AllocateScmBlockResponseProto.Error.success)
-                .build();
-      } else {
-        return AllocateScmBlockResponseProto.newBuilder()
-            .setErrorCode(AllocateScmBlockResponseProto.Error.unknownFailure)
-            .build();
-      }
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-
-  @Override
-  public DeleteScmKeyBlocksResponseProto deleteScmKeyBlocks(
-      RpcController controller, DeleteScmKeyBlocksRequestProto req)
-      throws ServiceException {
-    DeleteScmKeyBlocksResponseProto.Builder resp =
-        DeleteScmKeyBlocksResponseProto.newBuilder();
-    try {
-      List<BlockGroup> infoList = req.getKeyBlocksList().stream()
-          .map(BlockGroup::getFromProto).collect(Collectors.toList());
-      final List<DeleteBlockGroupResult> results =
-          impl.deleteKeyBlocks(infoList);
-      for (DeleteBlockGroupResult result: results) {
-        DeleteKeyBlocksResultProto.Builder deleteResult =
-            DeleteKeyBlocksResultProto
-            .newBuilder()
-            .setObjectKey(result.getObjectKey())
-            .addAllBlockResults(result.getBlockResultProtoList());
-        resp.addResults(deleteResult.build());
-      }
-    } catch (IOException ex) {
-      throw new ServiceException(ex);
-    }
-    return resp.build();
-  }
-
-  @Override
-  public OzoneProtos.GetScmInfoRespsonseProto getScmInfo(
-      RpcController controller, OzoneProtos.GetScmInfoRequestProto req)
-      throws ServiceException {
-    ScmInfo scmInfo;
-    try {
-      scmInfo = impl.getScmInfo();
-    } catch (IOException ex) {
-      throw new ServiceException(ex);
-    }
-    return OzoneProtos.GetScmInfoRespsonseProto.newBuilder()
-        .setClusterId(scmInfo.getClusterId())
-        .setScmId(scmInfo.getScmId())
-        .build();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
deleted file mode 100644
index 50d4fbb..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.protocolPB;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolTranslator;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMRegisterRequestProto;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMRegisteredCmdResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKResponseProto;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-/**
- * This class is the client-side translator to translate the requests made on
- * the {@link StorageContainerDatanodeProtocol} interface to the RPC server
- * implementing {@link StorageContainerDatanodeProtocolPB}.
- */
-public class StorageContainerDatanodeProtocolClientSideTranslatorPB
-    implements StorageContainerDatanodeProtocol, ProtocolTranslator, Closeable {
-
-  /**
-   * RpcController is not used and hence is set to null.
-   */
-  private static final RpcController NULL_RPC_CONTROLLER = null;
-  private final StorageContainerDatanodeProtocolPB rpcProxy;
-
-  /**
-   * Constructs a Client side interface that calls into SCM datanode protocol.
-   *
-   * @param rpcProxy - Proxy for RPC.
-   */
-  public StorageContainerDatanodeProtocolClientSideTranslatorPB(
-      StorageContainerDatanodeProtocolPB rpcProxy) {
-    this.rpcProxy = rpcProxy;
-  }
-
-  /**
-   * Closes this stream and releases any system resources associated with it. If
-   * the stream is already closed then invoking this method has no effect.
-   * <p>
-   * <p> As noted in {@link AutoCloseable#close()}, cases where the close may
-   * fail require careful attention. It is strongly advised to relinquish the
-   * underlying resources and to internally <em>mark</em> the {@code Closeable}
-   * as closed, prior to throwing the {@code IOException}.
-   *
-   * @throws IOException if an I/O error occurs
-   */
-  @Override
-  public void close() throws IOException {
-    RPC.stopProxy(rpcProxy);
-  }
-
-  /**
-   * Return the proxy object underlying this protocol translator.
-   *
-   * @return the proxy object underlying this protocol translator.
-   */
-  @Override
-  public Object getUnderlyingProxyObject() {
-    return rpcProxy;
-  }
-
-  /**
-   * Returns SCM version.
-   *
-   * @param unused - set to null and unused.
-   * @return Version info.
-   */
-  @Override
-  public SCMVersionResponseProto getVersion(SCMVersionRequestProto
-      unused) throws IOException {
-    SCMVersionRequestProto request =
-        SCMVersionRequestProto.newBuilder().build();
-    final SCMVersionResponseProto response;
-    try {
-      response = rpcProxy.getVersion(NULL_RPC_CONTROLLER, request);
-    } catch (ServiceException ex) {
-      throw ProtobufHelper.getRemoteException(ex);
-    }
-    return response;
-  }
-
-  /**
-   * Send by datanode to SCM.
-   *
-   * @param datanodeID - DatanodeID
-   * @param nodeReport - node report
-   * @throws IOException
-   */
-
-  @Override
-  public SCMHeartbeatResponseProto sendHeartbeat(DatanodeID datanodeID,
-      SCMNodeReport nodeReport, ReportState reportState) throws IOException {
-    SCMHeartbeatRequestProto.Builder req = SCMHeartbeatRequestProto
-        .newBuilder();
-    req.setDatanodeID(datanodeID.getProtoBufMessage());
-    req.setNodeReport(nodeReport);
-    req.setContainerReportState(reportState);
-    final SCMHeartbeatResponseProto resp;
-    try {
-      resp = rpcProxy.sendHeartbeat(NULL_RPC_CONTROLLER, req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    return resp;
-  }
-
-  /**
-   * Register Datanode.
-   *
-   * @param datanodeID - DatanodID.
-   * @return SCM Command.
-   */
-  @Override
-  public SCMRegisteredCmdResponseProto register(DatanodeID datanodeID,
-      String[] scmAddresses) throws IOException {
-    SCMRegisterRequestProto.Builder req =
-        SCMRegisterRequestProto.newBuilder();
-    req.setDatanodeID(datanodeID.getProtoBufMessage());
-    final SCMRegisteredCmdResponseProto response;
-    try {
-      response = rpcProxy.register(NULL_RPC_CONTROLLER, req.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    return response;
-  }
-
-  /**
-   * Send a container report.
-   *
-   * @param reports -- Container report
-   * @return HeartbeatRespose.nullcommand.
-   * @throws IOException
-   */
-  @Override
-  public ContainerReportsResponseProto sendContainerReport(
-      ContainerReportsRequestProto reports) throws IOException {
-    final ContainerReportsResponseProto resp;
-    try {
-      resp = rpcProxy.sendContainerReport(NULL_RPC_CONTROLLER, reports);
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    return resp;
-  }
-
-  @Override
-  public ContainerBlocksDeletionACKResponseProto sendContainerBlocksDeletionACK(
-      ContainerBlocksDeletionACKProto deletedBlocks) throws IOException {
-    final ContainerBlocksDeletionACKResponseProto resp;
-    try {
-      resp = rpcProxy.sendContainerBlocksDeletionACK(NULL_RPC_CONTROLLER,
-          deletedBlocks);
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
-    return resp;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
deleted file mode 100644
index 3e6f392..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.protocolPB;
-
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageContainerDatanodeProtocolService;
-
-/**
- * Protocol used from a datanode to StorageContainerManager.  This extends
- * the Protocol Buffers service interface to add Hadoop-specific annotations.
- */
-
-@ProtocolInfo(protocolName =
-    "org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol",
-    protocolVersion = 1)
-public interface StorageContainerDatanodeProtocolPB extends
-    StorageContainerDatanodeProtocolService.BlockingInterface {
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
deleted file mode 100644
index 2fd7038..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.protocolPB;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
-
-import java.io.IOException;
-
-/**
- * This class is the server-side translator that forwards requests received on
- * {@link StorageContainerDatanodeProtocolPB} to the {@link
- * StorageContainerDatanodeProtocol} server implementation.
- */
-public class StorageContainerDatanodeProtocolServerSideTranslatorPB
-    implements StorageContainerDatanodeProtocolPB {
-
-  private final StorageContainerDatanodeProtocol impl;
-
-  public StorageContainerDatanodeProtocolServerSideTranslatorPB(
-      StorageContainerDatanodeProtocol impl) {
-    this.impl = impl;
-  }
-
-  @Override
-  public StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto
-      getVersion(RpcController controller,
-      StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto request)
-      throws ServiceException {
-    try {
-      return impl.getVersion(request);
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-
-  @Override
-  public StorageContainerDatanodeProtocolProtos.SCMRegisteredCmdResponseProto
-      register(RpcController controller, StorageContainerDatanodeProtocolProtos
-      .SCMRegisterRequestProto request) throws ServiceException {
-    String[] addressArray = null;
-
-    if (request.hasAddressList()) {
-      addressArray = request.getAddressList().getAddressListList()
-          .toArray(new String[0]);
-    }
-
-    try {
-      return impl.register(DatanodeID.getFromProtoBuf(request
-          .getDatanodeID()), addressArray);
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-
-  @Override
-  public SCMHeartbeatResponseProto
-      sendHeartbeat(RpcController controller,
-      SCMHeartbeatRequestProto request) throws ServiceException {
-    try {
-      return impl.sendHeartbeat(DatanodeID.getFromProtoBuf(request
-          .getDatanodeID()), request.getNodeReport(),
-          request.getContainerReportState());
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-
-  @Override
-  public ContainerReportsResponseProto sendContainerReport(
-      RpcController controller, ContainerReportsRequestProto request)
-      throws ServiceException {
-    try {
-      return impl.sendContainerReport(request);
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-
-  @Override
-  public ContainerBlocksDeletionACKResponseProto sendContainerBlocksDeletionACK(
-      RpcController controller, ContainerBlocksDeletionACKProto request)
-      throws ServiceException {
-    try {
-      return impl.sendContainerBlocksDeletionACK(request);
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
deleted file mode 100644
index c8258b9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ /dev/null
@@ -1,202 +0,0 @@
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocolPB;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.List;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerLocationProtocolProtos;
-import org.apache.hadoop.scm.ScmInfo;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.protocol.StorageContainerLocationProtocol;
-
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ContainerResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.DeleteContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.DeleteContainerResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ListContainerResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ListContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.PipelineResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.PipelineRequestProto;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
-
-/**
- * This class is the server-side translator that forwards requests received on
- * {@link StorageContainerLocationProtocolPB} to the
- * {@link StorageContainerLocationProtocol} server implementation.
- */
-@InterfaceAudience.Private
-public final class StorageContainerLocationProtocolServerSideTranslatorPB
-    implements StorageContainerLocationProtocolPB {
-
-  private final StorageContainerLocationProtocol impl;
-
-  /**
-   * Creates a new StorageContainerLocationProtocolServerSideTranslatorPB.
-   *
-   * @param impl {@link StorageContainerLocationProtocol} server implementation
-   */
-  public StorageContainerLocationProtocolServerSideTranslatorPB(
-      StorageContainerLocationProtocol impl) throws IOException {
-    this.impl = impl;
-  }
-
-  @Override
-  public ContainerResponseProto allocateContainer(RpcController unused,
-      ContainerRequestProto request) throws ServiceException {
-    try {
-      Pipeline pipeline = impl.allocateContainer(request.getReplicationType(),
-          request.getReplicationFactor(), request.getContainerName(),
-          request.getOwner());
-      return ContainerResponseProto.newBuilder()
-          .setPipeline(pipeline.getProtobufMessage())
-          .setErrorCode(ContainerResponseProto.Error.success)
-          .build();
-
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-
-  @Override
-  public GetContainerResponseProto getContainer(
-      RpcController controller, GetContainerRequestProto request)
-      throws ServiceException {
-    try {
-      Pipeline pipeline = impl.getContainer(request.getContainerName());
-      return GetContainerResponseProto.newBuilder()
-          .setPipeline(pipeline.getProtobufMessage())
-          .build();
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-
-  @Override
-  public ListContainerResponseProto listContainer(RpcController controller,
-      ListContainerRequestProto request) throws ServiceException {
-    try {
-      String startName = null;
-      String prefixName = null;
-      int count = -1;
-
-      // Arguments check.
-      if (request.hasPrefixName()) {
-        // End container name is given.
-        prefixName = request.getPrefixName();
-      }
-      if (request.hasStartName()) {
-        // End container name is given.
-        startName = request.getStartName();
-      }
-
-      count = request.getCount();
-      List<ContainerInfo> containerList =
-          impl.listContainer(startName, prefixName, count);
-      ListContainerResponseProto.Builder builder =
-          ListContainerResponseProto.newBuilder();
-      for (ContainerInfo container : containerList) {
-        builder.addContainers(container.getProtobuf());
-      }
-      return builder.build();
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-
-  @Override
-  public DeleteContainerResponseProto deleteContainer(
-      RpcController controller, DeleteContainerRequestProto request)
-      throws ServiceException {
-    try {
-      impl.deleteContainer(request.getContainerName());
-      return DeleteContainerResponseProto.newBuilder().build();
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-
-  @Override
-  public StorageContainerLocationProtocolProtos.NodeQueryResponseProto
-      queryNode(RpcController controller,
-      StorageContainerLocationProtocolProtos.NodeQueryRequestProto request)
-      throws ServiceException {
-    try {
-      EnumSet<OzoneProtos.NodeState> nodeStateEnumSet = EnumSet.copyOf(request
-          .getQueryList());
-      OzoneProtos.NodePool datanodes = impl.queryNode(nodeStateEnumSet,
-          request.getScope(), request.getPoolName());
-      return StorageContainerLocationProtocolProtos
-          .NodeQueryResponseProto.newBuilder()
-          .setDatanodes(datanodes)
-          .build();
-    } catch (Exception e) {
-      throw new ServiceException(e);
-    }
-  }
-
-  @Override
-  public ObjectStageChangeResponseProto notifyObjectStageChange(
-      RpcController controller, ObjectStageChangeRequestProto request)
-      throws ServiceException {
-    try {
-      impl.notifyObjectStageChange(request.getType(), request.getName(),
-          request.getOp(), request.getStage());
-      return ObjectStageChangeResponseProto.newBuilder().build();
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-
-  @Override
-  public PipelineResponseProto allocatePipeline(
-      RpcController controller, PipelineRequestProto request)
-      throws ServiceException {
-    // TODO : Wiring this up requires one more patch.
-    return null;
-  }
-
-  @Override
-  public OzoneProtos.GetScmInfoRespsonseProto getScmInfo(
-      RpcController controller, OzoneProtos.GetScmInfoRequestProto req)
-      throws ServiceException {
-    try {
-      ScmInfo scmInfo = impl.getScmInfo();
-      return OzoneProtos.GetScmInfoRespsonseProto.newBuilder()
-          .setClusterId(scmInfo.getClusterId())
-          .setScmId(scmInfo.getScmId())
-          .build();
-    } catch (IOException ex) {
-      throw new ServiceException(ex);
-    }
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java
deleted file mode 100644
index 860386d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.protocolPB;
-
-/**
- * This package contains classes for the Protocol Buffers binding of Ozone
- * protocols.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/SCMMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/SCMMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/SCMMXBean.java
deleted file mode 100644
index 336ed44..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/SCMMXBean.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm;
-
-import java.util.Map;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.jmx.ServiceRuntimeInfo;
-
-/**
- *
- * This is the JMX management interface for scm information.
- */
-@InterfaceAudience.Private
-public interface SCMMXBean extends ServiceRuntimeInfo {
-
-  /**
-   * Get the SCM RPC server port that used to listen to datanode requests.
-   * @return SCM datanode RPC server port
-   */
-  String getDatanodeRpcPort();
-
-  /**
-   * Get the SCM RPC server port that used to listen to client requests.
-   * @return SCM client RPC server port
-   */
-  String getClientRpcPort();
-
-  /**
-   * Get container report info that includes container IO stats of nodes.
-   * @return The datanodeUUid to report json string mapping
-   */
-  Map<String, String> getContainerReport();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/SCMStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/SCMStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/SCMStorage.java
deleted file mode 100644
index e5437c7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/SCMStorage.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm;
-
-
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.common.Storage;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeType;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-
-import java.io.IOException;
-import java.util.Properties;
-import java.util.UUID;
-
-/**
- * SCMStorage is responsible for management of the StorageDirectories used by
- * the SCM.
- */
-public class SCMStorage extends Storage {
-
-  public static final String STORAGE_DIR = "scm";
-  public static final String SCM_ID = "scmUuid";
-
-  /**
-   * Construct SCMStorage.
-   * @throws IOException if any directories are inaccessible.
-   */
-  public SCMStorage(OzoneConfiguration conf) throws IOException {
-    super(NodeType.SCM, OzoneUtils.getOzoneMetaDirPath(conf), STORAGE_DIR);
-  }
-
-  public void setScmId(String scmId) throws IOException {
-    if (getState() == StorageState.INITIALIZED) {
-      throw new IOException("SCM is already initialized.");
-    } else {
-      getStorageInfo().setProperty(SCM_ID, scmId);
-    }
-  }
-
-  /**
-   * Retrieves the SCM ID from the version file.
-   * @return SCM_ID
-   */
-  public String getScmId() {
-    return getStorageInfo().getProperty(SCM_ID);
-  }
-
-  @Override
-  protected Properties getNodeProperties() {
-    String scmId = getScmId();
-    if (scmId == null) {
-      scmId = UUID.randomUUID().toString();
-    }
-    Properties scmProperties = new Properties();
-    scmProperties.setProperty(SCM_ID, scmId);
-    return scmProperties;
-  }
-
-}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[15/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeProcessTemplate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeProcessTemplate.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeProcessTemplate.java
deleted file mode 100644
index 0cd652a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeProcessTemplate.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos;
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.interfaces.UserAuth;
-import org.apache.hadoop.ozone.web.response.ListBuckets;
-import org.apache.hadoop.ozone.web.response.ListVolumes;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-import java.nio.file.DirectoryNotEmptyException;
-import java.nio.file.FileAlreadyExistsException;
-import java.nio.file.NoSuchFileException;
-
-import static java.net.HttpURLConnection.HTTP_OK;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_COMPONENT;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RESOURCE;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_REQUEST;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_USER;
-
-
-/**
- * This class abstracts way the repetitive tasks in
- * handling volume related code.
- */
-@InterfaceAudience.Private
-public abstract class VolumeProcessTemplate {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(VolumeProcessTemplate.class);
-
-
-  /**
-   * The handle call is the common functionality for Volume
-   * handling code.
-   *
-   * @param volume - Name of the Volume
-   * @param request - request
-   * @param info - UriInfo
-   * @param headers - Http Headers
-   *
-   * @return Response
-   *
-   * @throws OzoneException
-   */
-  public Response handleCall(String volume, Request request, UriInfo info,
-                             HttpHeaders headers) throws OzoneException {
-    String reqID = OzoneUtils.getRequestID();
-    String hostName = OzoneUtils.getHostName();
-    MDC.put(OZONE_COMPONENT, "ozone");
-    MDC.put(OZONE_REQUEST, reqID);
-    UserArgs userArgs  = null;
-    try {
-      userArgs = new UserArgs(reqID, hostName, request, info, headers);
-      OzoneUtils.validate(request, headers, reqID, volume, hostName);
-
-      // we use the same logic for both bucket and volume names
-      OzoneUtils.verifyResourceName(volume);
-      UserAuth auth = UserHandlerBuilder.getAuthHandler();
-
-      userArgs.setUserName(auth.getUser(userArgs));
-      MDC.put(OZONE_USER, userArgs.getUserName());
-      VolumeArgs args = new VolumeArgs(volume, userArgs);
-
-      MDC.put(OZONE_RESOURCE, args.getResourceName());
-      Response response =  doProcess(args);
-      LOG.info("Success");
-      MDC.clear();
-      return response;
-
-    } catch (IllegalArgumentException ex) {
-      LOG.error("Illegal argument.", ex);
-      throw ErrorTable.newError(ErrorTable.INVALID_VOLUME_NAME, userArgs, ex);
-    } catch (IOException ex) {
-      handleIOException(volume, reqID, hostName, ex);
-    }
-    return null;
-  }
-
-  /**
-   * Specific handler for each call.
-   *
-   * @param args - Volume Args
-   *
-   * @return - Response
-   *
-   * @throws IOException
-   * @throws OzoneException
-   */
-  public abstract Response doProcess(VolumeArgs args)
-      throws IOException, OzoneException;
-
-  /**
-   * Maps Java File System Exceptions to Ozone Exceptions in the Volume path.
-   *
-   * @param volume - Name of the Volume
-   * @param reqID - Request ID
-   * @param hostName - HostName
-   * @param fsExp - Exception
-   *
-   * @throws OzoneException
-   */
-  private void handleIOException(String volume, String reqID, String hostName,
-                                 IOException fsExp) throws OzoneException {
-    LOG.error("IOException:", fsExp);
-    OzoneException exp = null;
-
-    if ((fsExp != null && fsExp.getMessage().endsWith(
-        KeySpaceManagerProtocolProtos.Status.VOLUME_ALREADY_EXISTS.name()))
-        || fsExp instanceof FileAlreadyExistsException) {
-      exp = ErrorTable
-          .newError(ErrorTable.VOLUME_ALREADY_EXISTS, reqID, volume, hostName);
-    }
-
-    if (fsExp instanceof DirectoryNotEmptyException) {
-      exp = ErrorTable
-          .newError(ErrorTable.VOLUME_NOT_EMPTY, reqID, volume, hostName);
-    }
-
-    if (fsExp instanceof NoSuchFileException) {
-      exp = ErrorTable
-          .newError(ErrorTable.INVALID_VOLUME_NAME, reqID, volume, hostName);
-    }
-
-    if ((fsExp != null) && (exp != null)) {
-      exp.setMessage(fsExp.getMessage());
-    }
-
-    // We don't handle that FS error yet, report a Server Internal Error
-    if (exp == null) {
-      exp =
-          ErrorTable.newError(ErrorTable.SERVER_ERROR, reqID, volume, hostName);
-      if (fsExp != null) {
-        exp.setMessage(fsExp.getMessage());
-      }
-    }
-    throw exp;
-  }
-
-  /**
-   * Set the user provided string into args and throw ozone exception
-   * if needed.
-   *
-   * @param args - volume args
-   * @param quota - quota sting
-   *
-   * @throws OzoneException
-   */
-  void setQuotaArgs(VolumeArgs args, String quota) throws OzoneException {
-    try {
-      args.setQuota(quota);
-    } catch (IllegalArgumentException ex) {
-      LOG.debug("Malformed Quota.", ex);
-      throw ErrorTable.newError(ErrorTable.MALFORMED_QUOTA, args, ex);
-    }
-  }
-
-  /**
-   * Wraps calls into volumeInfo data.
-   *
-   * @param args - volumeArgs
-   *
-   * @return - VolumeInfo
-   *
-   * @throws IOException
-   * @throws OzoneException
-   */
-  Response getVolumeInfoResponse(VolumeArgs args)
-      throws IOException, OzoneException {
-    StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
-    VolumeInfo info = fs.getVolumeInfo(args);
-    return OzoneUtils.getResponse(args, HTTP_OK, info.toJsonString());
-  }
-
-  /**
-   * Returns all the volumes belonging to a user.
-   *
-   * @param user - userArgs
-   * @return - Response
-   * @throws OzoneException
-   * @throws IOException
-   */
-  Response getVolumesByUser(UserArgs user, String prefix, int maxKeys,
-      String prevKey, boolean rootScan) throws OzoneException, IOException {
-
-    String validatedUser = user.getUserName();
-    try {
-      UserAuth auth = UserHandlerBuilder.getAuthHandler();
-      if(rootScan && !auth.isAdmin(user)) {
-        throw ErrorTable.newError(ErrorTable.UNAUTHORIZED, user);
-      }
-      if (auth.isAdmin(user)) {
-        validatedUser = auth.getOzoneUser(user);
-        if (validatedUser == null) {
-          validatedUser = auth.getUser(user);
-        }
-      }
-
-      UserArgs onBehalfOf =
-          new UserArgs(validatedUser, user.getRequestID(), user.getHostName(),
-              user.getRequest(), user.getUri(), user.getHeaders());
-
-      StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
-      ListArgs<UserArgs> listArgs = new ListArgs<>(onBehalfOf, prefix,
-          maxKeys, prevKey);
-      listArgs.setRootScan(rootScan);
-      ListVolumes volumes = fs.listVolumes(listArgs);
-      return OzoneUtils.getResponse(user, HTTP_OK, volumes.toJsonString());
-    } catch (IOException ex) {
-      LOG.debug("unable to get the volume list for the user.", ex);
-      OzoneException exp = ErrorTable.newError(ErrorTable.SERVER_ERROR,
-          user, ex);
-      exp.setMessage("unable to get the volume list for the user");
-      throw exp;
-    }
-  }
-
-  /**
-   * Returns a list of Buckets in a Volume.
-   *
-   * @param args    - VolumeArgs
-   * @param prefix  - Prefix to Match
-   * @param maxKeys - Max results to return.
-   * @param prevKey - PrevKey
-   * @return List of Buckets
-   * @throws OzoneException
-   */
-  Response getBucketsInVolume(VolumeArgs args, String prefix, int maxKeys,
-                              String prevKey) throws OzoneException {
-    try {
-      // UserAuth auth = UserHandlerBuilder.getAuthHandler();
-      // TODO : Check ACLS.
-      StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
-      ListArgs<VolumeArgs> listArgs = new ListArgs<>(args, prefix,
-          maxKeys, prevKey);
-      ListBuckets bucketList = fs.listBuckets(listArgs);
-      return OzoneUtils.getResponse(args, HTTP_OK, bucketList.toJsonString());
-    } catch (IOException ex) {
-      LOG.debug("unable to get the bucket list for the specified volume.", ex);
-      OzoneException exp =
-          ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
-      exp.setMessage("unable to get the bucket list for the specified volume.");
-      throw exp;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/package-info.java
deleted file mode 100644
index 4d34c2d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.handlers;
-
-/**
- This package contains ozone client side libraries.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Accounting.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Accounting.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Accounting.java
deleted file mode 100644
index f03276c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Accounting.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.interfaces;
-
-/**
- * This in the accounting interface, Ozone Rest interface will call into this
- * interface whenever a put or delete key happens.
- * <p>
- * TODO : Technically we need to report bucket creation and deletion too
- * since the bucket names and metadata consume storage.
- * <p>
- * TODO : We should separate out reporting metadata & data --
- * <p>
- * In some cases end users will only want to account for the data they are
- * storing since metadata is mostly a cost of business.
- */
-public interface Accounting {
-  /**
-   * This call is made when ever a put key call is made.
-   * <p>
-   * In case of a Put which causes a over write of a key accounting system will
-   * see two calls, a removeByte call followed by an addByte call.
-   *
-   * @param owner  - Volume Owner
-   * @param volume - Name of the Volume
-   * @param bucket - Name of the bucket
-   * @param bytes  - How many bytes are put
-   */
-  void addBytes(String owner, String volume, String bucket, int bytes);
-
-  /**
-   * This call is made whenever a delete call is made.
-   *
-   * @param owner  - Volume Owner
-   * @param volume - Name of the Volume
-   * @param bucket - Name of the bucket
-   * @param bytes  - How many bytes are deleted
-   */
-  void removeBytes(String owner, String volume, String bucket, int bytes);
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java
deleted file mode 100644
index 4c8b85b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.interfaces;
-
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiImplicitParam;
-import io.swagger.annotations.ApiImplicitParams;
-import io.swagger.annotations.ApiOperation;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-
-import javax.ws.rs.DELETE;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-
-/**
- * Bucket Interface acts as the HTTP entry point for
- * bucket related functionality.
- */
-@Path("/{volume}/{bucket}")
-@Api(tags = "bucket")
-public interface Bucket {
-  /**
-   * createBucket call handles the POST request for Creating a Bucket.
-   *
-   * @param volume - Volume name
-   * @param bucket - Bucket Name
-   * @param req - Http request
-   * @param info - Uri Info
-   * @param headers - Http headers
-   *
-   * @return Response
-   *
-   * @throws OzoneException
-   */
-  @POST
-  @ApiOperation("Create new bucket to a volume")
-  @ApiImplicitParams({
-      @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
-          + "04:23:30 GMT", required = true, paramType = "header"),
-      @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
-          true, paramType = "header")})
-  Response createBucket(@PathParam("volume") String volume,
-                        @PathParam("bucket") String bucket,
-                        @Context Request req, @Context UriInfo info,
-                        @Context HttpHeaders headers) throws OzoneException;
-
-  /**
-   * updateBucket call handles the PUT request for updating a Bucket.
-   *
-   * @param volume - Volume name
-   * @param bucket - Bucket name
-   * @param req - Http request
-   * @param info - Uri Info
-   * @param headers - Http headers
-   *
-   * @return Response
-   *
-   * @throws OzoneException
-   */
-  @PUT
-  @ApiOperation("Update bucket")
-  @ApiImplicitParams({
-      @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
-          + "04:23:30 GMT", required = true, paramType = "header"),
-      @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
-          true, paramType = "header")})
-  Response updateBucket(@PathParam("volume") String volume,
-                        @PathParam("bucket") String bucket,
-                        @Context Request req, @Context UriInfo info,
-                        @Context HttpHeaders headers) throws OzoneException;
-
-  /**
-   * Deletes an empty bucket.
-   *
-   * @param volume Volume name
-   * @param bucket Bucket Name
-   * @param req - Http request
-   * @param info - Uri Info
-   * @param headers - Http headers
-   *
-   * @return Response
-   *
-   * @throws OzoneException
-   */
-  @DELETE
-  @ApiOperation("Deletes an empty bucket.")
-  @ApiImplicitParams({
-      @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
-          + "04:23:30 GMT", required = true, paramType = "header"),
-      @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
-          true, paramType = "header")})
-  Response deleteBucket(@PathParam("volume") String volume,
-                        @PathParam("bucket") String bucket,
-                        @Context Request req, @Context UriInfo info,
-                        @Context HttpHeaders headers) throws OzoneException;
-
-  /**
-   * List Buckets lists the contents of a bucket.
-   *
-   * @param volume - Storage Volume Name
-   * @param bucket - Bucket Name
-   * @param info - Information type needed
-   * @param prefix - Prefix for the keys to be fetched
-   * @param maxKeys - MaxNumber of Keys to Return
-   * @param prevKey - Continuation Token
-   * @param req - Http request
-   * @param headers - Http headers
-   *
-   * @return - Json Body
-   *
-   * @throws OzoneException
-   */
-
-  @GET
-  @Produces(MediaType.APPLICATION_JSON)
-  @ApiOperation("List contents of a bucket")
-  @ApiImplicitParams({
-      @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
-          + "04:23:30 GMT", required = true, paramType = "header"),
-      @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
-          true, paramType = "header")})
-  Response listBucket(@PathParam("volume") String volume,
-                      @PathParam("bucket") String bucket,
-                      @DefaultValue(Header.OZONE_INFO_QUERY_KEY)
-                      @QueryParam(Header.OZONE_INFO_QUERY_TAG)
-                      String info,
-                      @QueryParam(Header.OZONE_LIST_QUERY_PREFIX)
-                      String prefix,
-                      @DefaultValue(Header.OZONE_DEFAULT_LIST_SIZE)
-                      @QueryParam(Header.OZONE_LIST_QUERY_MAXKEYS)
-                      int maxKeys,
-                      @QueryParam(Header.OZONE_LIST_QUERY_PREVKEY)
-                      String prevKey,
-                      @Context Request req, @Context UriInfo uriInfo,
-                      @Context HttpHeaders headers) throws OzoneException;
-
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java
deleted file mode 100644
index f9255f2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.interfaces;
-
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiImplicitParam;
-import io.swagger.annotations.ApiImplicitParams;
-import io.swagger.annotations.ApiOperation;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-import java.io.InputStream;
-
-/**
- * This interface defines operations permitted on a key.
- */
-@Path("/{volume}/{bucket}/{keys:.*}")
-@Api(tags = "key")
-public interface Keys {
-
-  /**
-   * Adds a key to an existing bucket. If the object already exists
-   * this call will overwrite or add with new version number if the bucket
-   * versioning is turned on.
-   *
-   * @param volume Storage Volume Name
-   * @param bucket Name of the bucket
-   * @param keys Name of the Object
-   * @param is InputStream or File Data
-   * @param req Request
-   * @param headers http headers
-   *
-   * @return Response
-   *
-   * @throws OzoneException
-   */
-  @PUT
-  @Consumes(MediaType.WILDCARD)
-  @ApiOperation(value = "Adds a key to an existing bucket.", notes = "If the "
-      + "object already exists this call will overwrite or add with new version"
-      + " number if the bucket versioning is turned on.")
-  @ApiImplicitParams({
-      @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
-          + "04:23:30 GMT", required = true, paramType = "header"),
-      @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
-          true, paramType = "header")})
-  Response putKey(@PathParam("volume") String volume,
-      @PathParam("bucket") String bucket, @PathParam("keys") String keys,
-      InputStream is, @Context Request req, @Context UriInfo info,
-      @Context HttpHeaders headers) throws OzoneException;
-
-  /**
-   * Gets the Key if it exists.
-   *
-   * @param volume Storage Volume
-   * @param bucket Name of the bucket
-   * @param keys Object Name
-   * @param info Tag info
-   * @param req Request
-   * @param uriInfo Uri info
-   * @param headers Http Header
-   *
-   * @return Response
-   *
-   * @throws OzoneException
-   */
-  @GET
-  @ApiOperation("Gets the Key if it exists.")
-  @ApiImplicitParams({
-      @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
-          + "04:23:30 GMT", required = true, paramType = "header"),
-      @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
-          true, paramType = "header")})
-  Response getKey(@PathParam("volume") String volume,
-      @PathParam("bucket") String bucket, @PathParam("keys") String keys,
-      @QueryParam(Header.OZONE_INFO_QUERY_TAG) String info,
-      @Context Request req, @Context UriInfo uriInfo,
-      @Context HttpHeaders headers) throws OzoneException;
-
-  /**
-   * Deletes an existing key.
-   *
-   * @param volume Storage Volume Name
-   * @param bucket Name of the bucket
-   * @param keys Name of the Object
-   * @param req http Request
-   * @param headers HttpHeaders
-   *
-   * @return Response
-   *
-   * @throws OzoneException
-   */
-  @DELETE
-  @ApiOperation("Deletes an existing key")
-  @ApiImplicitParams({
-      @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
-          + "04:23:30 GMT", required = true, paramType = "header"),
-      @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
-          true, paramType = "header")})
-  Response deleteKey(@PathParam("volume") String volume,
-      @PathParam("bucket") String bucket, @PathParam("keys") String keys,
-      @Context Request req, @Context UriInfo info, @Context HttpHeaders headers)
-      throws OzoneException;
-}
-

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
deleted file mode 100644
index 6336c90..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
+++ /dev/null
@@ -1,295 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.interfaces;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.client.io.LengthInputStream;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.KeyArgs;
-import org.apache.hadoop.ozone.web.handlers.ListArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-import org.apache.hadoop.ozone.web.response.ListBuckets;
-import org.apache.hadoop.ozone.web.response.ListKeys;
-import org.apache.hadoop.ozone.web.response.ListVolumes;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * Storage handler Interface is the Interface between
- * REST protocol and file system.
- *
- * We will have two default implementations of this interface.
- * One for the local file system that is handy while testing
- * and another which will point to the HDFS backend.
- */
-@InterfaceAudience.Private
-public interface StorageHandler extends Closeable{
-
-  /**
-   * Creates a Storage Volume.
-   *
-   * @param args - Volume Name
-   *
-   * @throws IOException
-   * @throws OzoneException
-   */
-  void createVolume(VolumeArgs args) throws IOException, OzoneException;
-
-
-  /**
-   * setVolumeOwner - sets the owner of the volume.
-   *
-   * @param args owner info is present in the args
-   *
-   * @throws IOException
-   * @throws OzoneException
-   */
-  void setVolumeOwner(VolumeArgs args) throws IOException, OzoneException;
-
-
-  /**
-   * Set Volume Quota.
-   *
-   * @param args - Has Quota info
-   * @param remove - true if the request is to remove the quota
-   *
-   * @throws IOException
-   * @throws OzoneException
-   */
-  void setVolumeQuota(VolumeArgs args, boolean remove)
-      throws IOException, OzoneException;
-
-  /**
-   * Checks if a Volume exists and the user with a role specified has access
-   * to the Volume.
-   *
-   * @param volume - Volume Name whose access permissions needs to be checked
-   * @param acl - requested acls which needs to be checked for access
-   *
-   * @return - Boolean - True if the user with a role can access the volume.
-   * This is possible for owners of the volume and admin users
-   *
-   * @throws IOException
-   * @throws OzoneException
-   */
-  boolean checkVolumeAccess(String volume, OzoneAcl acl)
-      throws IOException, OzoneException;
-
-
-  /**
-   * Returns the List of Volumes owned by the specific user.
-   *
-   * @param args - ListArgs
-   *
-   * @return - List of Volumes
-   *
-   * @throws IOException
-   * @throws OzoneException
-   */
-  ListVolumes listVolumes(ListArgs args) throws IOException, OzoneException;
-
-  /**
-   * Deletes an Empty Volume.
-   *
-   * @param args - Volume Args
-   *
-   * @throws IOException
-   * @throws OzoneException
-   */
-  void deleteVolume(VolumeArgs args) throws IOException, OzoneException;
-
-
-  /**
-   * Returns Info about the specified Volume.
-   *
-   * @param args - Volume Args
-   *
-   * @return VolumeInfo
-   *
-   * @throws IOException
-   * @throws OzoneException
-   */
-  VolumeInfo getVolumeInfo(VolumeArgs args) throws IOException, OzoneException;
-
-  /**
-   * Creates a Bucket in specified Volume.
-   *
-   * @param args BucketArgs- BucketName, UserName and Acls
-   *
-   * @throws IOException
-   */
-  void createBucket(BucketArgs args) throws IOException, OzoneException;
-
-  /**
-   * Adds or Removes ACLs from a Bucket.
-   *
-   * @param args - BucketArgs
-   *
-   * @throws IOException
-   */
-  void setBucketAcls(BucketArgs args) throws IOException, OzoneException;
-
-  /**
-   * Enables or disables Bucket Versioning.
-   *
-   * @param args - BucketArgs
-   *
-   * @throws IOException
-   */
-  void setBucketVersioning(BucketArgs args) throws IOException, OzoneException;
-
-  /**
-   * Sets the Storage Class of a Bucket.
-   *
-   * @param args - BucketArgs
-   *
-   * @throws IOException
-   */
-  void setBucketStorageClass(BucketArgs args)
-      throws IOException, OzoneException;
-
-  /**
-   * Deletes a bucket if it is empty.
-   *
-   * @param args Bucket args structure
-   *
-   * @throws IOException
-   */
-  void deleteBucket(BucketArgs args) throws IOException, OzoneException;
-
-  /**
-   * true if the bucket exists and user has read access
-   * to the bucket else throws Exception.
-   *
-   * @param args Bucket args structure
-   *
-   * @throws IOException
-   */
-  void checkBucketAccess(BucketArgs args) throws IOException, OzoneException;
-
-
-  /**
-   * Returns all Buckets of a specified Volume.
-   *
-   * @param listArgs -- List Args.
-   *
-   * @return ListAllBuckets
-   *
-   * @throws OzoneException
-   */
-  ListBuckets listBuckets(ListArgs listArgs) throws
-      IOException, OzoneException;
-
-
-  /**
-   * Returns Bucket's Metadata as a String.
-   *
-   * @param args Bucket args structure
-   *
-   * @return Info about the bucket
-   *
-   * @throws IOException
-   */
-  BucketInfo getBucketInfo(BucketArgs args) throws IOException, OzoneException;
-
-  /**
-   * Writes a key in an existing bucket.
-   *
-   * @param args KeyArgs
-   *
-   * @return InputStream
-   *
-   * @throws OzoneException
-   */
-  OutputStream newKeyWriter(KeyArgs args)
-      throws IOException, OzoneException;
-
-
-  /**
-   * Tells the file system that the object has been written out
-   * completely and it can do any house keeping operation that needs
-   * to be done.
-   *
-   * @param args Key Args
-   *
-   * @param stream
-   * @throws IOException
-   */
-  void commitKey(KeyArgs args, OutputStream stream)
-      throws IOException, OzoneException;
-
-
-  /**
-   * Reads a key from an existing bucket.
-   *
-   * @param args KeyArgs
-   *
-   * @return LengthInputStream
-   *
-   * @throws IOException
-   */
-  LengthInputStream newKeyReader(KeyArgs args)
-      throws IOException, OzoneException;
-
-
-  /**
-   * Deletes an existing key.
-   *
-   * @param args KeyArgs
-   *
-   * @throws OzoneException
-   */
-  void deleteKey(KeyArgs args) throws IOException, OzoneException;
-
-
-  /**
-   * Returns a list of Key.
-   *
-   * @param args KeyArgs
-   *
-   * @return BucketList
-   *
-   * @throws IOException
-   */
-  ListKeys listKeys(ListArgs args) throws IOException, OzoneException;
-
-  /**
-   * Get information of the specified Key.
-   *
-   * @param args Key Args
-   *
-   * @return KeyInfo
-   *
-   * @throws IOException
-   * @throws OzoneException
-   */
-  KeyInfo getKeyInfo(KeyArgs args) throws IOException, OzoneException;
-
-  /**
-   * Closes all the opened resources.
-   */
-  void close();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/UserAuth.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/UserAuth.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/UserAuth.java
deleted file mode 100644
index a1d2e7c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/UserAuth.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.interfaces;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-
-/**
- * This interface is used by Ozone to determine user identity.
- *
- * Please see concrete implementations for more information
- */
-@InterfaceAudience.Private
-public interface UserAuth {
-  /**
-   * Returns the user name as a string from the URI and HTTP headers.
-   *
-   * @param userArgs - userArgs
-   *
-   * @return String - User name
-   *
-   * @throws OzoneException
-   */
-  String getUser(UserArgs userArgs) throws OzoneException;
-
-  /**
-   * Returns all the Groups that user is a member of.
-   *
-   * @param userArgs - userArgs
-   *
-   * @return Array of Groups
-   *
-   * @throws OzoneException
-   */
-  String[] getGroups(UserArgs userArgs) throws OzoneException;
-
-  /**
-   * Returns true if a user is a Admin.
-   *
-   * @param userArgs - userArgs
-   *
-   * @return true if Admin , false otherwise
-   *
-   * @throws OzoneException -- Allows the underlying system
-   * to throw, that error will get propagated to clients
-   */
-  boolean isAdmin(UserArgs userArgs) throws OzoneException;
-
-  /**
-   * Returns true if the request is Anonymous.
-   *
-   * @param userArgs - userArgs
-   *
-   * @return true if the request is anonymous, false otherwise.
-   *
-   * @throws OzoneException - Will be propagated back to end user
-   */
-  boolean isAnonymous(UserArgs userArgs) throws OzoneException;
-
-  /**
-   * Returns true if the name is a recognizable user in the system.
-   *
-   * @param userName - User Name to check
-   * @param userArgs - userArgs
-   *
-   * @return true if the username string is the name of a valid user.
-   *
-   * @throws OzoneException - Will be propagated back to end user
-   */
-  boolean isUser(String userName, UserArgs userArgs) throws OzoneException;
-
-  /**
-   * Returns the x-ozone-user or the user on behalf of, This is
-   * used in Volume creation path.
-   *
-   * @param userArgs - userArgs
-   *
-   * @return a user name if it has x-ozone-user args in header.
-   *
-   * @throws OzoneException
-   */
-  String getOzoneUser(UserArgs userArgs) throws OzoneException;
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java
deleted file mode 100644
index 85b2240..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.interfaces;
-
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiImplicitParam;
-import io.swagger.annotations.ApiImplicitParams;
-import io.swagger.annotations.ApiOperation;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-
-import javax.ws.rs.DELETE;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-
-/**
- * Volume Interface acts as the HTTP entry point for
- * volume related functionality.
- */
-@InterfaceAudience.Private
-@Path("/{volume}")
-@Api(tags = "volume")
-public interface Volume {
-
-  /**
-   * Creates a Volume owned by the user.
-   *
-   * Params :
-   * Quota - Specifies the Maximum usable size by the user
-   * the valid parameters for quota are <int>(<BYTES| MB|GB|TB>) | remove.
-   * For example 10GB or "remove".
-   *
-   * @param volume Volume Name, this has to be unique at Ozone Level
-   * @param quota Quota for this Storage Volume - <int>(<MB|GB|TB>) | remove
-   * @param req - Request Object - Request Object
-   * @param uriInfo - Http UriInfo
-   * @param headers Http Headers HttpHeaders
-   *
-   * @return Response
-   *
-   * @throws OzoneException
-   */
-
-  @POST
-  @ApiOperation("Creates a Volume owned by the user")
-  @ApiImplicitParams({
-      @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
-          + "04:23:30 GMT", required = true, paramType = "header"),
-      @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
-          true, paramType = "header")})
-  Response createVolume(@PathParam("volume") String volume,
-      @DefaultValue(Header.OZONE_QUOTA_UNDEFINED)
-      @QueryParam(Header.OZONE_QUOTA_QUERY_TAG) String quota,
-      @Context Request req, @Context UriInfo uriInfo,
-      @Context HttpHeaders headers) throws OzoneException;
-
-  /**
-   * Updates a Volume owned by the user.
-   *
-   * Params :
-   * Owner - Specifies the name of the owner
-   * Quota - Specifies the Maximum usable size by the user
-   * the valid parameters for quota are <int>(<MB|GB|TB>) | remove.
-   * For example 10GB or "remove".
-   *
-   * @param volume Volume Name, this has to be unique at Ozone Level
-   * @param quota Quota for this Storage Volume - <int>(<MB|GB|TB>) | remove
-   * @param req - Request Object - Request Object
-   * @param headers Http Headers HttpHeaders
-   *
-   * @return Response
-   *
-   * @throws OzoneException
-   */
-  @PUT
-  @ApiOperation("Updates a Volume owned by the user")
-  @ApiImplicitParams({
-      @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
-          + "04:23:30 GMT", required = true, paramType = "header"),
-      @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
-          true, paramType = "header")})
-  Response updateVolume(@PathParam("volume") String volume,
-      @DefaultValue(Header.OZONE_QUOTA_UNDEFINED)
-      @QueryParam(Header.OZONE_QUOTA_QUERY_TAG) String quota,
-      @Context Request req, @Context UriInfo uriInfo,
-      @Context HttpHeaders headers) throws OzoneException;
-
-  /**
-   * Deletes a Volume if it is empty.
-   *
-   * @param volume Storage Volume Name
-   *
-   * @return Response Response
-   *
-   * @throws OzoneException
-   */
-  @DELETE
-  @ApiOperation("Deletes a Volume if it is empty")
-  @ApiImplicitParams({
-      @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
-          + "04:23:30 GMT", required = true, paramType = "header"),
-      @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
-          true, paramType = "header")})
-  Response deleteVolume(@PathParam("volume") String volume,
-      @Context Request req, @Context UriInfo uriInfo,
-      @Context HttpHeaders headers) throws OzoneException;
-
-  /**
-   * Returns Volume info. This API can be invoked either
-   * by admin or the owner
-   *
-   * @param volume - Storage Volume Name
-   * @param req - Http Req
-   * @param headers - Http headers
-   *
-   * @return - Response
-   *
-   * @throws OzoneException
-   */
-  @GET
-  @ApiOperation(value = "Returns Volume info", notes = "This API can be "
-      + "invoked either by admin or the owner")
-  @ApiImplicitParams({
-      @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
-          true, paramType = "header"),
-      @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
-          + "04:23:30 GMT", required = true, paramType = "header"),
-      @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
-          true, paramType = "header")})
-  Response getVolumeInfo(@PathParam("volume") String volume,
-      @DefaultValue(Header.OZONE_INFO_QUERY_BUCKET)
-      @QueryParam(Header.OZONE_INFO_QUERY_TAG) String info,
-      @QueryParam(Header.OZONE_LIST_QUERY_PREFIX) String prefix,
-      @DefaultValue(Header.OZONE_DEFAULT_LIST_SIZE)
-      @QueryParam(Header.OZONE_LIST_QUERY_MAXKEYS) int keys,
-      @QueryParam(Header.OZONE_LIST_QUERY_PREVKEY) String prevKey,
-      @QueryParam(Header.OZONE_LIST_QUERY_ROOTSCAN) boolean rootScan,
-      @Context Request req, @Context UriInfo uriInfo,
-      @Context HttpHeaders headers) throws OzoneException;
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/package-info.java
deleted file mode 100644
index 940f179..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.interfaces;
-
-/**
- This package contains ozone client side libraries.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java
deleted file mode 100644
index 9747eac..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.localstorage;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.client.io.LengthInputStream;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.KeyArgs;
-import org.apache.hadoop.ozone.web.handlers.ListArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.web.request.OzoneQuota;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-import org.apache.hadoop.ozone.web.response.ListBuckets;
-import org.apache.hadoop.ozone.web.response.ListKeys;
-import org.apache.hadoop.ozone.web.response.ListVolumes;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * PLEASE NOTE : This file is a dummy backend for test purposes and prototyping
- * effort only. It does not handle any Object semantics correctly, neither does
- * it take care of security.
- */
-@InterfaceAudience.Private
-public class LocalStorageHandler implements StorageHandler {
-  private final Configuration conf;
-
-  /**
-   * Constructs LocalStorageHandler.
-   *
-   * @param conf ozone conf.
-   */
-  public LocalStorageHandler(Configuration conf) {
-    this.conf = conf;
-  }
-
-  /**
-   * Creates Storage Volume.
-   *
-   * @param args - volumeArgs
-   * @throws IOException
-   */
-  @Override
-  public void createVolume(VolumeArgs args) throws IOException, OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    oz.createVolume(args);
-
-  }
-
-  /**
-   * setVolumeOwner - sets the owner of the volume.
-   *
-   * @param args volumeArgs
-   * @throws IOException
-   */
-  @Override
-  public void setVolumeOwner(VolumeArgs args)
-      throws IOException, OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    oz.setVolumeProperty(args, OzoneMetadataManager.VolumeProperty.OWNER);
-  }
-
-  /**
-   * Set Volume Quota Info.
-   *
-   * @param args   - volumeArgs
-   * @param remove - true if the request is to remove the quota
-   * @throws IOException
-   */
-  @Override
-  public void setVolumeQuota(VolumeArgs args, boolean remove)
-      throws IOException, OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-
-    if (remove) {
-      OzoneQuota quota = new OzoneQuota();
-      args.setQuota(quota);
-    }
-    oz.setVolumeProperty(args, OzoneMetadataManager.VolumeProperty.QUOTA);
-  }
-
-  /**
-   * Checks if a Volume exists and the user specified has access to the volume.
-   *
-   * @param volume - Volume Name
-   * @param acl - Ozone acl which needs to be compared for access
-   * @return - Boolean - True if the user can modify the volume. This is
-   * possible for owners of the volume and admin users
-   * @throws IOException
-   */
-  @Override
-  public boolean checkVolumeAccess(String volume, OzoneAcl acl)
-      throws IOException, OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    return oz.checkVolumeAccess(volume, acl);
-  }
-
-  /**
-   * Returns Info about the specified Volume.
-   *
-   * @param args - volumeArgs
-   * @return VolumeInfo
-   * @throws IOException
-   */
-  @Override
-  public VolumeInfo getVolumeInfo(VolumeArgs args)
-      throws IOException, OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    return oz.getVolumeInfo(args);
-  }
-
-  /**
-   * Deletes an Empty Volume.
-   *
-   * @param args - Volume Args
-   * @throws IOException
-   */
-  @Override
-  public void deleteVolume(VolumeArgs args) throws IOException, OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    oz.deleteVolume(args);
-
-  }
-
-  /**
-   * Returns the List of Volumes owned by the specific user.
-   *
-   * @param args - ListArgs
-   * @return - List of Volumes
-   * @throws IOException
-   */
-  @Override
-  public ListVolumes listVolumes(ListArgs args)
-      throws IOException, OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    return oz.listVolumes(args);
-  }
-
-  /**
-   * true if the bucket exists and user has read access to the bucket else
-   * throws Exception.
-   *
-   * @param args Bucket args structure
-   * @throws IOException
-   */
-  @Override
-  public void checkBucketAccess(BucketArgs args)
-      throws IOException, OzoneException {
-
-  }
-
-  /**
-   * Creates a Bucket in specified Volume.
-   *
-   * @param args BucketArgs- BucketName, UserName and Acls
-   * @throws IOException
-   */
-  @Override
-  public void createBucket(BucketArgs args) throws IOException, OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    oz.createBucket(args);
-  }
-
-  /**
-   * Adds or Removes ACLs from a Bucket.
-   *
-   * @param args - BucketArgs
-   * @throws IOException
-   */
-  @Override
-  public void setBucketAcls(BucketArgs args)
-      throws IOException, OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    oz.setBucketProperty(args, OzoneMetadataManager.BucketProperty.ACLS);
-  }
-
-  /**
-   * Enables or disables Bucket Versioning.
-   *
-   * @param args - BucketArgs
-   * @throws IOException
-   */
-  @Override
-  public void setBucketVersioning(BucketArgs args)
-      throws IOException, OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    oz.setBucketProperty(args, OzoneMetadataManager.BucketProperty.VERSIONING);
-
-  }
-
-  /**
-   * Sets the Storage Class of a Bucket.
-   *
-   * @param args - BucketArgs
-   * @throws IOException
-   */
-  @Override
-  public void setBucketStorageClass(BucketArgs args)
-      throws IOException, OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    oz.setBucketProperty(args, OzoneMetadataManager.BucketProperty.STORAGETYPE);
-
-  }
-
-  /**
-   * Deletes a bucket if it is empty.
-   *
-   * @param args Bucket args structure
-   * @throws IOException
-   */
-  @Override
-  public void deleteBucket(BucketArgs args) throws IOException, OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    oz.deleteBucket(args);
-  }
-
-  /**
-   * Returns all Buckets of a specified Volume.
-   *
-   * @param args --User Args
-   * @return ListAllBuckets
-   * @throws OzoneException
-   */
-  @Override
-  public ListBuckets listBuckets(ListArgs args)
-      throws IOException, OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    return oz.listBuckets(args);
-  }
-
-  /**
-   * Returns Bucket's Metadata as a String.
-   *
-   * @param args Bucket args structure
-   * @return Info about the bucket
-   * @throws IOException
-   */
-  @Override
-  public BucketInfo getBucketInfo(BucketArgs args)
-      throws IOException, OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    return oz.getBucketInfo(args);
-  }
-
-  /**
-   * Writes a key in an existing bucket.
-   *
-   * @param args KeyArgs
-   * @return InputStream
-   * @throws OzoneException
-   */
-  @Override
-  public OutputStream newKeyWriter(KeyArgs args) throws IOException,
-      OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    return oz.createKey(args);
-  }
-
-  /**
-   * Tells the file system that the object has been written out completely and
-   * it can do any house keeping operation that needs to be done.
-   *
-   * @param args   Key Args
-   * @param stream
-   * @throws IOException
-   */
-  @Override
-  public void commitKey(KeyArgs args, OutputStream stream) throws
-      IOException, OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    oz.commitKey(args, stream);
-
-  }
-
-  /**
-   * Reads a key from an existing bucket.
-   *
-   * @param args KeyArgs
-   * @return LengthInputStream
-   * @throws IOException
-   */
-  @Override
-  public LengthInputStream newKeyReader(KeyArgs args) throws IOException,
-      OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    return oz.newKeyReader(args);
-  }
-
-  /**
-   * Deletes an existing key.
-   *
-   * @param args KeyArgs
-   * @throws OzoneException
-   */
-  @Override
-  public void deleteKey(KeyArgs args) throws IOException, OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    oz.deleteKey(args);
-  }
-
-  /**
-   * Returns a list of Key.
-   *
-   * @param args KeyArgs
-   * @return BucketList
-   * @throws IOException
-   */
-  @Override
-  public ListKeys listKeys(ListArgs args) throws IOException, OzoneException {
-    OzoneMetadataManager oz =
-        OzoneMetadataManager.getOzoneMetadataManager(conf);
-    return oz.listKeys(args);
-
-  }
-
-  /**
-   * Get information of the specified Key.
-   *
-   * @param args Key Args
-   *
-   * @return KeyInfo
-   *
-   * @throws IOException
-   * @throws OzoneException
-   */
-  @Override
-  public KeyInfo getKeyInfo(KeyArgs args) throws IOException, OzoneException {
-    OzoneMetadataManager oz = OzoneMetadataManager
-        .getOzoneMetadataManager(conf);
-    return oz.getKeyInfo(args);
-  }
-
-  @Override
-  public void close() {
-    //No resource to close, do nothing.
-  }
-
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[06/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dfs-dust.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dfs-dust.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dfs-dust.js
index c7af6a1..1f37d21 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dfs-dust.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dfs-dust.js
@@ -38,10 +38,7 @@
     'fmt_percentage': function (v) {
       return Math.round(v * 100) / 100 + '%';
     },
-    'elapsed': function(v) {
-      //elapsed sec from epoch sec
-      return Date.now() - v * 1000;
-    },
+
     'fmt_time': function (v) {
       var s = Math.floor(v / 1000), h = Math.floor(s / 3600);
       s -= h * 3600;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.css
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.css b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.css
deleted file mode 100644
index b8a5c0f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.css
+++ /dev/null
@@ -1,2 +0,0 @@
-.nvd3 .nv-axis line,.nvd3 .nv-axis path{fill:none;shape-rendering:crispEdges}.nv-brush .extent,.nvd3 .background path,.nvd3 .nv-axis line,.nvd3 .nv-axis path{shape-rendering:crispEdges}.nv-distx,.nv-disty,.nv-noninteractive,.nvd3 .nv-axis,.nvd3.nv-pie .nv-label,.nvd3.nv-sparklineplus g.nv-hoverValue{pointer-events:none}.nvd3 .nv-axis{opacity:1}.nvd3 .nv-axis.nv-disabled,.nvd3 .nv-controlsWrap .nv-legend .nv-check-box .nv-check{opacity:0}.nvd3 .nv-axis path{stroke:#000;stroke-opacity:.75}.nvd3 .nv-axis path.domain{stroke-opacity:.75}.nvd3 .nv-axis.nv-x path.domain{stroke-opacity:0}.nvd3 .nv-axis line{stroke:#e5e5e5}.nvd3 .nv-axis .zero line, .nvd3 .nv-axis line.zero{stroke-opacity:.75}.nvd3 .nv-axis .nv-axisMaxMin text{font-weight:700}.nvd3 .x .nv-axis .nv-axisMaxMin text,.nvd3 .x2 .nv-axis .nv-axisMaxMin text,.nvd3 .x3 .nv-axis .nv-axisMaxMin text{text-anchor:middle}.nvd3 .nv-bars rect{fill-opacity:.75;transition:fill-opacity 250ms linear}.nvd3 .nv-bars rect.hover{fill-opacity:1}.nv
 d3 .nv-bars .hover rect{fill:#add8e6}.nvd3 .nv-bars text{fill:transparent}.nvd3 .nv-bars .hover text{fill:rgba(0,0,0,1)}.nvd3 .nv-discretebar .nv-groups rect,.nvd3 .nv-multibar .nv-groups rect,.nvd3 .nv-multibarHorizontal .nv-groups rect{stroke-opacity:0;transition:fill-opacity 250ms linear}.with-transitions .nv-candlestickBar .nv-ticks .nv-tick,.with-transitions .nvd3 .nv-groups .nv-point{transition:stroke-width 250ms linear,stroke-opacity 250ms linear}.nvd3 .nv-candlestickBar .nv-ticks rect:hover,.nvd3 .nv-discretebar .nv-groups rect:hover,.nvd3 .nv-multibar .nv-groups rect:hover,.nvd3 .nv-multibarHorizontal .nv-groups rect:hover{fill-opacity:1}.nvd3 .nv-discretebar .nv-groups text,.nvd3 .nv-multibarHorizontal .nv-groups text{font-weight:700;fill:rgba(0,0,0,1);stroke:transparent}.nvd3 .nv-boxplot circle{fill-opacity:.5}.nvd3 .nv-boxplot circle:hover,.nvd3 .nv-boxplot rect:hover{fill-opacity:1}.nvd3 line.nv-boxplot-median{stroke:#000}.nv-boxplot-tick:hover{stroke-width:2.5px}.nvd3.
 nv-bullet{font:10px sans-serif}.nvd3.nv-bullet .nv-measure{fill-opacity:.8}.nvd3.nv-bullet .nv-measure:hover{fill-opacity:1}.nvd3.nv-bullet .nv-marker{stroke:#000;stroke-width:2px}.nvd3.nv-bullet .nv-markerTriangle{stroke:#000;fill:#fff;stroke-width:1.5px}.nvd3.nv-bullet .nv-markerLine{stroke:#000;stroke-width:1.5px}.nvd3.nv-bullet .nv-tick line{stroke:#666;stroke-width:.5px}.nvd3.nv-bullet .nv-range.nv-s0{fill:#eee}.nvd3.nv-bullet .nv-range.nv-s1{fill:#ddd}.nvd3.nv-bullet .nv-range.nv-s2{fill:#ccc}.nvd3.nv-bullet .nv-title{font-size:14px;font-weight:700}.nvd3.nv-bullet .nv-subtitle{fill:#999}.nvd3.nv-bullet .nv-range{fill:#bababa;fill-opacity:.4}.nvd3.nv-bullet .nv-range:hover{fill-opacity:.7}.nvd3.nv-candlestickBar .nv-ticks .nv-tick{stroke-width:1px}.nvd3.nv-candlestickBar .nv-ticks .nv-tick.hover{stroke-width:2px}.nvd3.nv-candlestickBar .nv-ticks .nv-tick.positive rect{stroke:#2ca02c;fill:#2ca02c}.nvd3.nv-candlestickBar .nv-ticks .nv-tick.negative rect{stroke:#d62728;fill:#d6272
 8}.nvd3.nv-candlestickBar .nv-ticks line{stroke:#333}.nv-force-node{stroke:#fff;stroke-width:1.5px}.nv-force-link{stroke:#999;stroke-opacity:.6}.nv-force-node text{stroke-width:0}.nvd3 .nv-check-box .nv-box{fill-opacity:0;stroke-width:2}.nvd3 .nv-check-box .nv-check{fill-opacity:0;stroke-width:4}.nvd3 .nv-series.nv-disabled .nv-check-box .nv-check{fill-opacity:0;stroke-opacity:0}.nvd3.nv-linePlusBar .nv-bar rect{fill-opacity:.75}.nvd3.nv-linePlusBar .nv-bar rect:hover{fill-opacity:1}.nvd3 .nv-groups path.nv-line{fill:none}.nvd3 .nv-groups path.nv-area{stroke:none}.nvd3.nv-line .nvd3.nv-scatter .nv-groups .nv-point{fill-opacity:0;stroke-opacity:0}.nvd3.nv-scatter.nv-single-point .nv-groups .nv-point{fill-opacity:.5!important;stroke-opacity:.5!important}.nvd3 .nv-groups .nv-point.hover,.nvd3.nv-scatter .nv-groups .nv-point.hover{stroke-width:7px;fill-opacity:.95!important;stroke-opacity:.95!important}.nvd3 .nv-point-paths path{stroke:#aaa;stroke-opacity:0;fill:#eee;fill-opacity:0}.nvd
 3 .nv-indexLine{cursor:ew-resize}svg.nvd3-svg{-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;display:block;width:100%;height:100%}.nvtooltip.with-3d-shadow,.with-3d-shadow .nvtooltip{box-shadow:0 5px 10px rgba(0,0,0,.2);border-radius:5px}.nvd3 text{font:400 12px Arial,sans-serif}.nvd3 .title{font:700 14px Arial,sans-serif}.nvd3 .nv-background{fill:#fff;fill-opacity:0}.nvd3.nv-noData{font-size:18px;font-weight:700}.nv-brush .extent{fill-opacity:.125}.nv-brush .resize path{fill:#eee;stroke:#666}.nvd3 .nv-legend .nv-series{cursor:pointer}.nvd3 .nv-legend .nv-disabled circle{fill-opacity:0}.nvd3 .nv-brush .extent{fill-opacity:0!important}.nvd3 .nv-brushBackground rect{stroke:#000;stroke-width:.4;fill:#fff;fill-opacity:.7}@media print{.nvd3 text{stroke-width:0;fill-opacity:1}}.nvd3.nv-ohlcBar .nv-ticks .nv-tick{stroke-width:1px}.nvd3.nv-ohlcBar .nv-ticks .nv-tick.hover{stroke-width:2px}.nvd3.nv-ohlcBar .nv-ticks .nv-tick.positive{stroke:#2ca02c}.nvd3
 .nv-ohlcBar .nv-ticks .nv-tick.negative{stroke:#d62728}.nvd3 .background path{fill:none;stroke:#EEE;stroke-opacity:.4}.nvd3 .foreground path{fill:none;stroke-opacity:.7}.nvd3 .nv-parallelCoordinates-brush .extent{fill:#fff;fill-opacity:.6;stroke:gray;shape-rendering:crispEdges}.nvd3 .nv-parallelCoordinates .hover{fill-opacity:1;stroke-width:3px}.nvd3 .missingValuesline line{fill:none;stroke:#000;stroke-width:1;stroke-opacity:1;stroke-dasharray:5,5}.nvd3.nv-pie .nv-pie-title{font-size:24px;fill:rgba(19,196,249,.59)}.nvd3.nv-pie .nv-slice text{stroke:#000;stroke-width:0}.nvd3.nv-pie path{transition:fill-opacity 250ms linear,stroke-width 250ms linear,stroke-opacity 250ms linear;stroke:#fff;stroke-width:1px;stroke-opacity:1;fill-opacity:.7}.nvd3.nv-pie .hover path{fill-opacity:1}.nvd3.nv-pie .nv-label rect{fill-opacity:0;stroke-opacity:0}.nvd3 .nv-groups .nv-point.hover{stroke-width:20px;stroke-opacity:.5}.nvd3 .nv-scatter .nv-point.hover{fill-opacity:1}.nvd3.nv-sparkline path{fill:none
 }.nvd3.nv-sparklineplus .nv-hoverValue line{stroke:#333;stroke-width:1.5px}.nvd3.nv-sparklineplus,.nvd3.nv-sparklineplus g{pointer-events:all}.nvd3 .nv-interactiveGuideLine,.nvtooltip{pointer-events:none}.nvd3 .nv-hoverArea{fill-opacity:0;stroke-opacity:0}.nvd3.nv-sparklineplus .nv-xValue,.nvd3.nv-sparklineplus .nv-yValue{stroke-width:0;font-size:.9em;font-weight:400}.nvd3.nv-sparklineplus .nv-yValue{stroke:#f66}.nvd3.nv-sparklineplus .nv-maxValue{stroke:#2ca02c;fill:#2ca02c}.nvd3.nv-sparklineplus .nv-minValue{stroke:#d62728;fill:#d62728}.nvd3.nv-sparklineplus .nv-currentValue{font-weight:700;font-size:1.1em}.nvtooltip h3,.nvtooltip table td.key{font-weight:400}.nvd3.nv-stackedarea path.nv-area{fill-opacity:.7;stroke-opacity:0;transition:fill-opacity 250ms linear,stroke-opacity 250ms linear}.nvd3.nv-stackedarea path.nv-area.hover{fill-opacity:.9}.nvd3.nv-stackedarea .nv-groups .nv-point{stroke-opacity:0;fill-opacity:0}.nvtooltip{position:absolute;color:rgba(0,0,0,1);padding:1px;z-in
 dex:10000;display:block;font-family:Arial,sans-serif;font-size:13px;text-align:left;white-space:nowrap;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background:rgba(255,255,255,.8);border:1px solid rgba(0,0,0,.5);border-radius:4px}.nvtooltip h3,.nvtooltip p{margin:0;text-align:center}.nvtooltip.with-transitions,.with-transitions .nvtooltip{transition:opacity 50ms linear;transition-delay:200ms}.nvtooltip.x-nvtooltip,.nvtooltip.y-nvtooltip{padding:8px}.nvtooltip h3{padding:4px 14px;line-height:18px;background-color:rgba(247,247,247,.75);color:rgba(0,0,0,1);border-bottom:1px solid #ebebeb;border-radius:5px 5px 0 0}.nvtooltip p{padding:5px 14px}.nvtooltip span{display:inline-block;margin:2px 0}.nvtooltip table{margin:6px;border-spacing:0}.nvtooltip table td{padding:2px 9px 2px 0;vertical-align:middle}.nvtooltip table td.key.total{font-weight:700}.nvtooltip table td.value{text-align:right;font-weight:700}.nvtooltip table td.percent{color:#a9a9a9}.nv
 tooltip table tr.highlight td{padding:1px 9px 1px 0;border-bottom-style:solid;border-bottom-width:1px;border-top-style:solid;border-top-width:1px}.nvtooltip table td.legend-color-guide div{vertical-align:middle;width:12px;height:12px;border:1px solid #999}.nvtooltip .footer{padding:3px;text-align:center}.nvtooltip-pending-removal{pointer-events:none;display:none}.nvd3 line.nv-guideline{stroke:#ccc}
-/*# sourceMappingURL=nv.d3.min.css.map */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.css.map
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.css.map b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.css.map
deleted file mode 100644
index 63380e6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.css.map
+++ /dev/null
@@ -1 +0,0 @@
-{"version":3,"sources":["build/nv.d3.css"],"names":[],"mappings":"AAqBA,oBAfA,oBAgBI,KAAM,KAiWN,gBAAiB,WA/ErB,kBA+DA,uBAlVA,oBAfA,oBAiXI,gBAAiB,WAqErB,UAAW,UAJX,mBAvbA,eAoaA,uBAgCA,uCACI,eAAgB,KArcpB,eAEI,QAAS,EAuCb,2BAsJA,0DACI,QAAS,EA3Lb,oBAEI,OAAQ,KACR,eAAgB,IAIpB,2BACI,eAAgB,IAGpB,gCACI,eAAgB,EAGpB,oBAEI,OAAQ,QAIZ,0BACI,0BACA,eAAgB,IAGpB,mCACI,YAAa,IAGjB,sCACA,uCACA,uCACI,YAAa,OAOjB,oBACI,aAAc,IAEd,WAAY,aAAa,MAAM,OAGnC,0BACI,aAAc,EAGlB,2BACI,KAAM,QAGV,oBACI,KAAM,YAGV,2BACI,KAAM,cAKV,sCAFA,mCACA,6CAEI,eAAgB,EAEhB,WAAY,aAAa,MAAM,OA8EnC,wDAwEA,6CACI,WAAY,aAAa,MAAM,OAAQ,eAAe,MAAM,OAlJhE,8CACA,4CAHA,yCACA,mDAGI,aAAc,EAGlB,sCACA,6CACI,YAAa,IACb,KAAM,cACN,OAAQ,YAIZ,yBACE,aAAc,GAGhB,+BAIA,6BAHE,aAAc,EAOhB,6BACE,OAAQ,KAGV,uBACE,aAAc,MAGhB,gBAAkB,KAAM,KAAK,WAC7B,4BAA8B,aAAc,GAC5C,kCAAoC,aAAc,EAClD,2BAA6B,OAAQ,KAAM,aAAc,IACzD,mCAAqC,OAAQ,KAAM,KAAM,KAAM,aAAc,MAC7E,+BAAiC,OAAQ,KAAM,aAAc,MAC7D,8BAAgC,OAAQ,KAAM,aAAc,KAC5D,gCAAkC,KAAM,KACxC,gCAAkC,KAAM,KACxC,gCAAkC,KAAM,KACxC,0BAA4B,UAAW,KAAM,Y
 AAa,IAC1D,6BAA+B,KAAM,KAErC,0BACI,KAAM,QACN,aAAc,GAGlB,gCACI,aAAc,GAGlB,2CACI,aAAc,IAGlB,iDACI,aAAc,IAGlB,yDACI,OAAQ,QACR,KAAM,QAGV,yDACI,OAAQ,QACR,KAAM,QAOV,uCACI,OAAQ,KAGZ,eACI,OAAQ,KACR,aAAc,MAGlB,eACI,OAAQ,KACR,eAAgB,GAGpB,oBACI,aAAc,EAOlB,4BACI,aAAa,EACb,aAAa,EAGjB,8BACI,aAAa,EACb,aAAa,EAGjB,qDACI,aAAa,EACb,eAAe,EAQnB,kCACI,aAAc,IAGlB,wCACI,aAAc,EAElB,8BACI,KAAM,KAGV,8BACI,OAAQ,KAGZ,oDACI,aAAc,EACd,eAAgB,EAGpB,sDACI,aAAc,aACd,eAAgB,aASpB,iCADA,4CAEI,aAAc,IACd,aAAc,cACd,eAAgB,cAIpB,2BACI,OAAQ,KACR,eAAgB,EAChB,KAAM,KACN,aAAc,EAIlB,oBACI,OAAQ,UAUZ,aACI,oBAAqB,KAClB,iBAAkB,KACjB,gBAAiB,KACb,YAAa,KACrB,QAAS,MACT,MAAM,KACN,OAAO,KAMX,0BAA2B,2BACvB,WAAY,EAAE,IAAI,KAAK,eACvB,cAAe,IAInB,WACI,KAAM,IAAO,KAAK,MAAO,WAG7B,aACI,KAAM,IAAK,KAAK,MAAO,WAG3B,qBACI,KAAM,KACN,aAAc,EAGlB,gBACI,UAAW,KACX,YAAa,IAQjB,kBACI,aAAc,KAIlB,uBACI,KAAM,KACN,OAAQ,KAQZ,4BACI,OAAQ,QAGZ,qCACI,aAAc,EAIlB,wBACI,aAAc,YAGlB,+BACI,OAAQ,KACR,aAAc,GACd,KAAM,KACN,aAAc,GAOlB,aACI,WACI,aAAc,EACd,aAAc,GAItB,oCACI,aAAc,IAGlB,0C
 ACI,aAAc,IAGlB,6CACI,OAAQ,QAGZ,6CACI,OAAQ,QAIZ,uBACI,KAAM,KACN,OAAQ,KACR,eAAgB,GAIpB,uBACI,KAAM,KACN,eAAgB,GAGpB,4CACI,KAAM,KACN,aAAc,GACd,OAAQ,KACR,gBAAiB,WAGrB,qCACI,aAAc,EACjB,aAAc,IAIf,8BACE,KAAM,KACN,OAAQ,KACR,aAAc,EACd,eAAgB,EAChB,iBAAkB,EAAG,EAQvB,2BACI,UAAW,KACX,KAAM,qBAGV,4BACI,OAAQ,KACR,aAAc,EAGlB,kBAbI,WAAY,aAAa,MAAM,OAAQ,aAAa,MAAM,OAAQ,eAAe,MAAM,OAcvF,OAAQ,KACR,aAAc,IACd,eAAgB,EAIhB,aAAc,GAGlB,yBACI,aAAc,EAOlB,4BACI,aAAc,EACd,eAAgB,EAIpB,iCACI,aAAc,KACd,eAAgB,GAGpB,kCACI,aAAc,EAYlB,wBACI,KAAM,KAOV,2CACI,OAAQ,KACR,aAAc,MAGlB,uBACA,yBACI,eAAgB,IAsLpB,+BApIA,WAqII,eAAe,KApLnB,oBACI,aAAc,EACd,eAAgB,EAGpB,kCACA,kCACI,aAAc,EACd,UAAW,KACX,YAAa,IAGjB,kCACI,OAAQ,KAGZ,oCACI,OAAQ,QACR,KAAM,QAGV,oCACI,OAAQ,QACR,KAAM,QAGV,wCACI,YAAa,IACb,UAAW,MAgEf,cAoCA,wBACI,YAAa,IAlGjB,kCACI,aAAc,GACd,eAAgB,EAChB,WAAY,aAAa,MAAM,OAAQ,eAAe,MAAM,OAGhE,wCACI,aAAc,GAIlB,0CACI,eAAgB,EAChB,aAAc,EAGlB,WACI,SAAU,SAEV,MAAO,cACP,QAAS,IAET,QAAS,MACT,QAAS,MAET,YAAa,MAAO,WACpB,UAAW,KACX,WAAY,KAGZ,YAAa,OAEb,oBAA
 qB,KAElB,iBAAkB,KAEjB,gBAAiB,KAEb,YAAa,KAIrB,WAAY,qBACZ,OAAQ,IAAI,MAAM,eAClB,cAAe,IAiBnB,cAcA,aACI,OAAQ,EAER,WAAY,OA5BhB,4BAA6B,6BACzB,WAAY,QAAQ,KAAK,OAEzB,iBAAkB,MAGtB,uBACA,uBACI,QAAS,IAGb,cAEI,QAAS,IAAI,KACb,YAAa,KAEb,iBAAkB,sBAClB,MAAO,cAGP,cAAe,IAAI,MAAM,QAEzB,cAAe,IAAI,IAAI,EAAE,EAG7B,aAEI,QAAS,IAAI,KAIjB,gBACI,QAAS,aACT,OAAQ,IAAI,EAGhB,iBACI,OAAQ,IACR,eAAe,EAInB,oBACI,QAAS,IAAI,IAAI,IAAI,EACrB,eAAgB,OAOpB,8BACI,YAAa,IAGjB,0BACI,WAAY,MACZ,YAAa,IAGjB,4BACI,MAAO,QAGX,iCACI,QAAS,IAAI,IAAI,IAAI,EACrB,oBAAqB,MACrB,oBAAqB,IACrB,iBAAkB,MAClB,iBAAkB,IAGtB,2CAGI,eAAgB,OAIhB,MAAO,KACP,OAAQ,KACR,OAAQ,IAAI,MAAM,KAGtB,mBACI,QAAS,IACT,WAAY,OAGhB,2BACI,eAAgB,KAChB,QAAS,KAWb,wBACI,OAAQ"}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[28/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.

Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
deleted file mode 100644
index 95bf107..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.states.endpoint;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.container.common.statemachine
-    .EndpointStateMachine;
-
-import org.apache.hadoop.ozone.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerNodeIDProto;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.Future;
-
-/**
- * Register a container with SCM.
- */
-public final class RegisterEndpointTask implements
-    Callable<EndpointStateMachine.EndPointStates> {
-  static final Logger LOG = LoggerFactory.getLogger(RegisterEndpointTask.class);
-
-  private final EndpointStateMachine rpcEndPoint;
-  private final Configuration conf;
-  private Future<EndpointStateMachine.EndPointStates> result;
-  private ContainerNodeIDProto containerNodeIDProto;
-
-  /**
-   * Creates a register endpoint task.
-   *
-   * @param rpcEndPoint - endpoint
-   * @param conf - conf
-   */
-  @VisibleForTesting
-  public RegisterEndpointTask(EndpointStateMachine rpcEndPoint,
-      Configuration conf) {
-    this.rpcEndPoint = rpcEndPoint;
-    this.conf = conf;
-
-  }
-
-  /**
-   * Get the ContainerNodeID Proto.
-   *
-   * @return ContainerNodeIDProto
-   */
-  public ContainerNodeIDProto getContainerNodeIDProto() {
-    return containerNodeIDProto;
-  }
-
-  /**
-   * Set the contiainerNodeID Proto.
-   *
-   * @param containerNodeIDProto - Container Node ID.
-   */
-  public void setContainerNodeIDProto(ContainerNodeIDProto
-      containerNodeIDProto) {
-    this.containerNodeIDProto = containerNodeIDProto;
-  }
-
-  /**
-   * Computes a result, or throws an exception if unable to do so.
-   *
-   * @return computed result
-   * @throws Exception if unable to compute a result
-   */
-  @Override
-  public EndpointStateMachine.EndPointStates call() throws Exception {
-
-    if (getContainerNodeIDProto() == null) {
-      LOG.error("Container ID proto cannot be null in RegisterEndpoint task, " +
-          "shutting down the endpoint.");
-      return rpcEndPoint.setState(EndpointStateMachine.EndPointStates.SHUTDOWN);
-    }
-
-    rpcEndPoint.lock();
-    try {
-      DatanodeID dnNodeID = DatanodeID.getFromProtoBuf(
-          getContainerNodeIDProto().getDatanodeID());
-
-      // TODO : Add responses to the command Queue.
-      rpcEndPoint.getEndPoint().register(dnNodeID,
-          conf.getStrings(ScmConfigKeys.OZONE_SCM_NAMES));
-      EndpointStateMachine.EndPointStates nextState =
-          rpcEndPoint.getState().getNextState();
-      rpcEndPoint.setState(nextState);
-      rpcEndPoint.zeroMissedCount();
-    } catch (IOException ex) {
-      rpcEndPoint.logIfNeeded(ex
-      );
-    } finally {
-      rpcEndPoint.unlock();
-    }
-
-    return rpcEndPoint.getState();
-  }
-
-  /**
-   * Returns a builder class for RegisterEndPoint task.
-   *
-   * @return Builder.
-   */
-  public static Builder newBuilder() {
-    return new Builder();
-  }
-
-  /**
-   * Builder class for RegisterEndPoint task.
-   */
-  public static class Builder {
-    private EndpointStateMachine endPointStateMachine;
-    private Configuration conf;
-    private ContainerNodeIDProto containerNodeIDProto;
-
-    /**
-     * Constructs the builder class.
-     */
-    public Builder() {
-    }
-
-    /**
-     * Sets the endpoint state machine.
-     *
-     * @param rpcEndPoint - Endpoint state machine.
-     * @return Builder
-     */
-    public Builder setEndpointStateMachine(EndpointStateMachine rpcEndPoint) {
-      this.endPointStateMachine = rpcEndPoint;
-      return this;
-    }
-
-    /**
-     * Sets the Config.
-     *
-     * @param config - config
-     * @return Builder.
-     */
-    public Builder setConfig(Configuration config) {
-      this.conf = config;
-      return this;
-    }
-
-    /**
-     * Sets the NodeID.
-     *
-     * @param nodeID - NodeID proto
-     * @return Builder
-     */
-    public Builder setNodeID(ContainerNodeIDProto nodeID) {
-      this.containerNodeIDProto = nodeID;
-      return this;
-    }
-
-    public RegisterEndpointTask build() {
-      if (endPointStateMachine == null) {
-        LOG.error("No endpoint specified.");
-        throw new IllegalArgumentException("A valid endpoint state machine is" +
-            " needed to construct RegisterEndPoint task");
-      }
-
-      if (conf == null) {
-        LOG.error("No config specified.");
-        throw new IllegalArgumentException("A valid configration is needed to" +
-            " construct RegisterEndpoint task");
-      }
-
-      if (containerNodeIDProto == null) {
-        LOG.error("No nodeID specified.");
-        throw new IllegalArgumentException("A vaild Node ID is needed to " +
-            "construct RegisterEndpoint task");
-      }
-
-      RegisterEndpointTask task = new RegisterEndpointTask(this
-          .endPointStateMachine, this.conf);
-      task.setContainerNodeIDProto(containerNodeIDProto);
-      return task;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java
deleted file mode 100644
index fa59234..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.states.endpoint;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
-import org.apache.hadoop.ozone.protocol.VersionResponse;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
-
-import java.io.IOException;
-import java.util.concurrent.Callable;
-
-/**
- * Task that returns version.
- */
-public class VersionEndpointTask implements
-    Callable<EndpointStateMachine.EndPointStates> {
-  private final EndpointStateMachine rpcEndPoint;
-  private final Configuration configuration;
-
-  public VersionEndpointTask(EndpointStateMachine rpcEndPoint,
-      Configuration conf) {
-    this.rpcEndPoint = rpcEndPoint;
-    this.configuration = conf;
-  }
-
-  /**
-   * Computes a result, or throws an exception if unable to do so.
-   *
-   * @return computed result
-   * @throws Exception if unable to compute a result
-   */
-  @Override
-  public EndpointStateMachine.EndPointStates call() throws Exception {
-    rpcEndPoint.lock();
-    try{
-      SCMVersionResponseProto versionResponse =
-          rpcEndPoint.getEndPoint().getVersion(null);
-      rpcEndPoint.setVersion(VersionResponse.getFromProtobuf(versionResponse));
-
-      EndpointStateMachine.EndPointStates nextState =
-          rpcEndPoint.getState().getNextState();
-      rpcEndPoint.setState(nextState);
-      rpcEndPoint.zeroMissedCount();
-    } catch (IOException ex) {
-      rpcEndPoint.logIfNeeded(ex);
-    } finally {
-      rpcEndPoint.unlock();
-    }
-    return rpcEndPoint.getState();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/package-info.java
deleted file mode 100644
index 1122598..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/package-info.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.states.endpoint;
-/**
- This package contains code for RPC endpoints transitions.
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/package-info.java
deleted file mode 100644
index 92c953f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.states;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java
deleted file mode 100644
index 3a5ce18..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.transport.server;
-
-import com.google.common.base.Preconditions;
-import io.netty.bootstrap.ServerBootstrap;
-import io.netty.channel.Channel;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
-import io.netty.handler.logging.LogLevel;
-import io.netty.handler.logging.LoggingHandler;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.net.SocketAddress;
-
-/**
- * Creates a netty server endpoint that acts as the communication layer for
- * Ozone containers.
- */
-public final class XceiverServer implements XceiverServerSpi {
-  private static final Logger
-      LOG = LoggerFactory.getLogger(XceiverServer.class);
-  private int port;
-  private final ContainerDispatcher storageContainer;
-
-  private EventLoopGroup bossGroup;
-  private EventLoopGroup workerGroup;
-  private Channel channel;
-
-  /**
-   * Constructs a netty server class.
-   *
-   * @param conf - Configuration
-   */
-  public XceiverServer(Configuration conf,
-                       ContainerDispatcher dispatcher) {
-    Preconditions.checkNotNull(conf);
-
-    this.port = conf.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
-        OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
-    // Get an available port on current node and
-    // use that as the container port
-    if (conf.getBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT,
-        OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT_DEFAULT)) {
-      try (ServerSocket socket = new ServerSocket()) {
-        socket.setReuseAddress(true);
-        SocketAddress address = new InetSocketAddress(0);
-        socket.bind(address);
-        this.port = socket.getLocalPort();
-        LOG.info("Found a free port for the server : {}", this.port);
-      } catch (IOException e) {
-        LOG.error("Unable find a random free port for the server, "
-            + "fallback to use default port {}", this.port, e);
-      }
-    }
-    this.storageContainer = dispatcher;
-  }
-
-  @Override
-  public int getIPCPort() {
-    return this.port;
-  }
-
-  /**
-   * Returns the Replication type supported by this end-point.
-   *
-   * @return enum -- {Stand_Alone, Ratis, Chained}
-   */
-  @Override
-  public OzoneProtos.ReplicationType getServerType() {
-    return OzoneProtos.ReplicationType.STAND_ALONE;
-  }
-
-  @Override
-  public void start() throws IOException {
-    bossGroup = new NioEventLoopGroup();
-    workerGroup = new NioEventLoopGroup();
-    channel = new ServerBootstrap()
-        .group(bossGroup, workerGroup)
-        .channel(NioServerSocketChannel.class)
-        .handler(new LoggingHandler(LogLevel.INFO))
-        .childHandler(new XceiverServerInitializer(storageContainer))
-        .bind(port)
-        .syncUninterruptibly()
-        .channel();
-  }
-
-  @Override
-  public void stop() {
-    if (storageContainer != null) {
-      storageContainer.shutdown();
-    }
-    if (bossGroup != null) {
-      bossGroup.shutdownGracefully();
-    }
-    if (workerGroup != null) {
-      workerGroup.shutdownGracefully();
-    }
-    if (channel != null) {
-      channel.close().awaitUninterruptibly();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerHandler.java
deleted file mode 100644
index c4a8f53..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerHandler.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.transport.server;
-
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.SimpleChannelInboundHandler;
-import org.slf4j.LoggerFactory;
-import org.slf4j.Logger;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandRequestProto;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-
-/**
- * Netty server handlers that respond to Network events.
- */
-public class XceiverServerHandler extends
-    SimpleChannelInboundHandler<ContainerCommandRequestProto> {
-
-  static final Logger LOG = LoggerFactory.getLogger(XceiverServerHandler.class);
-  private final ContainerDispatcher dispatcher;
-
-  /**
-   * Constructor for server handler.
-   * @param dispatcher - Dispatcher interface
-   */
-  public XceiverServerHandler(ContainerDispatcher dispatcher) {
-    this.dispatcher = dispatcher;
-  }
-
-  /**
-   * <strong>Please keep in mind that this method will be renamed to {@code
-   * messageReceived(ChannelHandlerContext, I)} in 5.0.</strong>
-   * <p>
-   * Is called for each message of type {@link ContainerCommandRequestProto}.
-   *
-   * @param ctx the {@link ChannelHandlerContext} which this {@link
-   *            SimpleChannelInboundHandler} belongs to
-   * @param msg the message to handle
-   * @throws Exception is thrown if an error occurred
-   */
-  @Override
-  public void channelRead0(ChannelHandlerContext ctx,
-                           ContainerCommandRequestProto msg) throws
-      Exception {
-    ContainerCommandResponseProto response = this.dispatcher.dispatch(msg);
-    LOG.debug("Writing the reponse back to client.");
-    ctx.writeAndFlush(response);
-
-  }
-
-  /**
-   * Calls {@link ChannelHandlerContext#fireExceptionCaught(Throwable)}
-   * Sub-classes may override this method to change behavior.
-   *
-   * @param ctx   - Channel Handler Context
-   * @param cause - Exception
-   */
-  @Override
-  public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause)
-      throws Exception {
-    LOG.error("An exception caught in the pipeline : " + cause.toString());
-    super.exceptionCaught(ctx, cause);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerInitializer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerInitializer.java
deleted file mode 100644
index 4d32d86..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerInitializer.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.transport.server;
-
-import com.google.common.base.Preconditions;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.ChannelPipeline;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.handler.codec.protobuf.ProtobufDecoder;
-import io.netty.handler.codec.protobuf.ProtobufEncoder;
-import io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder;
-import io.netty.handler.codec.protobuf.ProtobufVarint32LengthFieldPrepender;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandRequestProto;
-
-/**
- * Creates a channel for the XceiverServer.
- */
-public class XceiverServerInitializer extends ChannelInitializer<SocketChannel>{
-  private final ContainerDispatcher dispatcher;
-  public XceiverServerInitializer(ContainerDispatcher dispatcher) {
-    Preconditions.checkNotNull(dispatcher);
-    this.dispatcher = dispatcher;
-  }
-
-  /**
-   * This method will be called once the Channel is registered. After
-   * the method returns this instance will be removed from the {@link
-   * ChannelPipeline}
-   *
-   * @param ch the  which was registered.
-   * @throws Exception is thrown if an error occurs. In that case the channel
-   * will be closed.
-   */
-  @Override
-  protected void initChannel(SocketChannel ch) throws Exception {
-    ChannelPipeline pipeline = ch.pipeline();
-    pipeline.addLast(new ProtobufVarint32FrameDecoder());
-    pipeline.addLast(new ProtobufDecoder(ContainerCommandRequestProto
-        .getDefaultInstance()));
-    pipeline.addLast(new ProtobufVarint32LengthFieldPrepender());
-    pipeline.addLast(new ProtobufEncoder());
-    pipeline.addLast(new XceiverServerHandler(dispatcher));
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
deleted file mode 100644
index b61d7fd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.transport.server;
-
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-import java.io.IOException;
-
-/** A server endpoint that acts as the communication layer for Ozone
- * containers. */
-public interface XceiverServerSpi {
-  /** Starts the server. */
-  void start() throws IOException;
-
-  /** Stops a running server. */
-  void stop();
-
-  /** Get server IPC port. */
-  int getIPCPort();
-
-  /**
-   * Returns the Replication type supported by this end-point.
-   * @return enum -- {Stand_Alone, Ratis, Chained}
-   */
-  OzoneProtos.ReplicationType getServerType();
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/package-info.java
deleted file mode 100644
index 59c96f1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.transport.server;
-
-/**
- * This package contains classes for the server of the storage container
- * protocol.
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
deleted file mode 100644
index 569fb23..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.transport.server.ratis;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.WriteChunkRequestProto;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.protocol.Message;
-import org.apache.ratis.protocol.RaftClientRequest;
-import org.apache.ratis.protocol.RaftPeerId;
-import org.apache.ratis.server.storage.RaftStorage;
-import org.apache.ratis.shaded.com.google.protobuf.ShadedProtoUtil;
-import org.apache.ratis.shaded.proto.RaftProtos.LogEntryProto;
-import org.apache.ratis.shaded.proto.RaftProtos.SMLogEntryProto;
-import org.apache.ratis.statemachine.impl.BaseStateMachine;
-import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
-import org.apache.ratis.statemachine.StateMachineStorage;
-import org.apache.ratis.statemachine.TransactionContext;
-import org.apache.ratis.statemachine.impl.TransactionContextImpl;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
-
-import java.io.IOException;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.ConcurrentHashMap;
-
-/** A {@link org.apache.ratis.statemachine.StateMachine} for containers.
- *
- * The stateMachine is responsible for handling different types of container
- * requests. The container requests can be divided into readonly and write
- * requests.
- *
- * Read only requests are classified in
- * {@link org.apache.hadoop.scm.XceiverClientRatis#isReadOnly}
- * and these readonly requests are replied from the {@link #query(Message)}.
- *
- * The write requests can be divided into requests with user data
- * (WriteChunkRequest) and other request without user data.
- *
- * Inorder to optimize the write throughput, the writeChunk request is
- * processed in 2 phases. The 2 phases are divided in
- * {@link #startTransaction(RaftClientRequest)}, in the first phase the user
- * data is written directly into the state machine via
- * {@link #writeStateMachineData} and in the second phase the
- * transaction is committed via {@link #applyTransaction(TransactionContext)}
- *
- * For the requests with no stateMachine data, the transaction is directly
- * committed through
- * {@link #applyTransaction(TransactionContext)}
- *
- * There are 2 ordering operation which are enforced right now in the code,
- * 1) Write chunk operation are executed after the create container operation,
- * the write chunk operation will fail otherwise as the container still hasn't
- * been created. Hence the create container operation has been split in the
- * {@link #startTransaction(RaftClientRequest)}, this will help in synchronizing
- * the calls in {@link #writeStateMachineData}
- *
- * 2) Write chunk commit operation is executed after write chunk state machine
- * operation. This will ensure that commit operation is sync'd with the state
- * machine operation.
- * */
-public class ContainerStateMachine extends BaseStateMachine {
-  static final Logger LOG = LoggerFactory.getLogger(
-      ContainerStateMachine.class);
-  private final SimpleStateMachineStorage storage
-      = new SimpleStateMachineStorage();
-  private final ContainerDispatcher dispatcher;
-  private ThreadPoolExecutor writeChunkExecutor;
-  private final ConcurrentHashMap<Long, CompletableFuture<Message>>
-      writeChunkFutureMap;
-  private final ConcurrentHashMap<String, CompletableFuture<Message>>
-      createContainerFutureMap;
-
-  ContainerStateMachine(ContainerDispatcher dispatcher,
-      ThreadPoolExecutor writeChunkExecutor) {
-    this.dispatcher = dispatcher;
-    this.writeChunkExecutor = writeChunkExecutor;
-    this.writeChunkFutureMap = new ConcurrentHashMap<>();
-    this.createContainerFutureMap = new ConcurrentHashMap<>();
-  }
-
-  @Override
-  public StateMachineStorage getStateMachineStorage() {
-    return storage;
-  }
-
-  @Override
-  public void initialize(
-      RaftPeerId id, RaftProperties properties, RaftStorage raftStorage)
-      throws IOException {
-    super.initialize(id, properties, raftStorage);
-    storage.init(raftStorage);
-    //  TODO handle snapshots
-
-    // TODO: Add a flag that tells you that initialize has been called.
-    // Check with Ratis if this feature is done in Ratis.
-  }
-
-  @Override
-  public TransactionContext startTransaction(RaftClientRequest request)
-      throws IOException {
-    final ContainerCommandRequestProto proto =
-        getRequestProto(request.getMessage().getContent());
-
-    final SMLogEntryProto log;
-    if (proto.getCmdType() == ContainerProtos.Type.WriteChunk) {
-      final WriteChunkRequestProto write = proto.getWriteChunk();
-      // create the state machine data proto
-      final WriteChunkRequestProto dataWriteChunkProto =
-          WriteChunkRequestProto
-              .newBuilder(write)
-              .setStage(ContainerProtos.Stage.WRITE_DATA)
-              .build();
-      ContainerCommandRequestProto dataContainerCommandProto =
-          ContainerCommandRequestProto
-              .newBuilder(proto)
-              .setWriteChunk(dataWriteChunkProto)
-              .build();
-
-      // create the log entry proto
-      final WriteChunkRequestProto commitWriteChunkProto =
-          WriteChunkRequestProto.newBuilder()
-              .setPipeline(write.getPipeline())
-              .setKeyName(write.getKeyName())
-              .setChunkData(write.getChunkData())
-              // skipping the data field as it is
-              // already set in statemachine data proto
-              .setStage(ContainerProtos.Stage.COMMIT_DATA)
-              .build();
-      ContainerCommandRequestProto commitContainerCommandProto =
-          ContainerCommandRequestProto
-              .newBuilder(proto)
-              .setWriteChunk(commitWriteChunkProto)
-              .build();
-
-      log = SMLogEntryProto.newBuilder()
-          .setData(getShadedByteString(commitContainerCommandProto))
-          .setStateMachineData(getShadedByteString(dataContainerCommandProto))
-          .build();
-    } else if (proto.getCmdType() == ContainerProtos.Type.CreateContainer) {
-      log = SMLogEntryProto.newBuilder()
-          .setData(request.getMessage().getContent())
-          .setStateMachineData(request.getMessage().getContent())
-          .build();
-    } else {
-      log = SMLogEntryProto.newBuilder()
-          .setData(request.getMessage().getContent())
-          .build();
-    }
-    return new TransactionContextImpl(this, request, log);
-  }
-
-  private ByteString getShadedByteString(ContainerCommandRequestProto proto) {
-    return ShadedProtoUtil.asShadedByteString(proto.toByteArray());
-  }
-
-  private ContainerCommandRequestProto getRequestProto(ByteString request)
-      throws InvalidProtocolBufferException {
-    return ContainerCommandRequestProto.parseFrom(
-        ShadedProtoUtil.asByteString(request));
-  }
-
-  private Message runCommand(ContainerCommandRequestProto requestProto) {
-    LOG.trace("dispatch {}", requestProto);
-    ContainerCommandResponseProto response = dispatcher.dispatch(requestProto);
-    LOG.trace("response {}", response);
-    return () -> ShadedProtoUtil.asShadedByteString(response.toByteArray());
-  }
-
-  private CompletableFuture<Message> handleWriteChunk(
-      ContainerCommandRequestProto requestProto, long entryIndex) {
-    final WriteChunkRequestProto write = requestProto.getWriteChunk();
-    String containerName = write.getPipeline().getContainerName();
-    CompletableFuture<Message> future =
-        createContainerFutureMap.get(containerName);
-    CompletableFuture<Message> writeChunkFuture;
-    if (future != null) {
-      writeChunkFuture = future.thenApplyAsync(
-          v -> runCommand(requestProto), writeChunkExecutor);
-    } else {
-      writeChunkFuture = CompletableFuture.supplyAsync(
-          () -> runCommand(requestProto), writeChunkExecutor);
-    }
-    writeChunkFutureMap.put(entryIndex, writeChunkFuture);
-    return writeChunkFuture;
-  }
-
-  private CompletableFuture<Message> handleCreateContainer(
-      ContainerCommandRequestProto requestProto) {
-    String containerName =
-        requestProto.getCreateContainer().getContainerData().getName();
-    createContainerFutureMap.
-        computeIfAbsent(containerName, k -> new CompletableFuture<>());
-    return CompletableFuture.completedFuture(() -> ByteString.EMPTY);
-  }
-
-  @Override
-  public CompletableFuture<Message> writeStateMachineData(LogEntryProto entry) {
-    try {
-      final ContainerCommandRequestProto requestProto =
-          getRequestProto(entry.getSmLogEntry().getStateMachineData());
-      ContainerProtos.Type cmdType = requestProto.getCmdType();
-      switch (cmdType) {
-      case CreateContainer:
-        return handleCreateContainer(requestProto);
-      case WriteChunk:
-        return handleWriteChunk(requestProto, entry.getIndex());
-      default:
-        throw new IllegalStateException("Cmd Type:" + cmdType
-            + " should not have state machine data");
-      }
-    } catch (IOException e) {
-      return completeExceptionally(e);
-    }
-  }
-
-  @Override
-  public CompletableFuture<Message> query(Message request) {
-    try {
-      final ContainerCommandRequestProto requestProto =
-          getRequestProto(request.getContent());
-      return CompletableFuture.completedFuture(runCommand(requestProto));
-    } catch (IOException e) {
-      return completeExceptionally(e);
-    }
-  }
-
-  @Override
-  public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
-    try {
-      ContainerCommandRequestProto requestProto =
-          getRequestProto(trx.getSMLogEntry().getData());
-      ContainerProtos.Type cmdType = requestProto.getCmdType();
-
-      if (cmdType == ContainerProtos.Type.WriteChunk) {
-        WriteChunkRequestProto write = requestProto.getWriteChunk();
-        // the data field has already been removed in start Transaction
-        Preconditions.checkArgument(!write.hasData());
-        CompletableFuture<Message> stateMachineFuture =
-            writeChunkFutureMap.remove(trx.getLogEntry().getIndex());
-        return stateMachineFuture
-            .thenComposeAsync(v ->
-                CompletableFuture.completedFuture(runCommand(requestProto)));
-      } else {
-        Message message = runCommand(requestProto);
-        if (cmdType == ContainerProtos.Type.CreateContainer) {
-          String containerName =
-              requestProto.getCreateContainer().getContainerData().getName();
-          createContainerFutureMap.remove(containerName).complete(message);
-        }
-        return CompletableFuture.completedFuture(message);
-      }
-    } catch (IOException e) {
-      return completeExceptionally(e);
-    }
-  }
-
-  private static <T> CompletableFuture<T> completeExceptionally(Exception e) {
-    final CompletableFuture<T> future = new CompletableFuture<>();
-    future.completeExceptionally(e);
-    return future;
-  }
-
-  @Override
-  public void close() throws IOException {
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
deleted file mode 100644
index d0ff094..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.transport.server.ratis;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.container.common.transport.server
-    .XceiverServerSpi;
-
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.ratis.RaftConfigKeys;
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.grpc.GrpcConfigKeys;
-import org.apache.ratis.netty.NettyConfigKeys;
-import org.apache.ratis.RatisHelper;
-import org.apache.ratis.rpc.RpcType;
-import org.apache.ratis.rpc.SupportedRpcType;
-import org.apache.ratis.server.RaftServer;
-import org.apache.ratis.server.RaftServerConfigKeys;
-import org.apache.ratis.util.SizeInBytes;
-import org.apache.ratis.util.TimeDuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.net.SocketAddress;
-import java.util.Objects;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Creates a ratis server endpoint that acts as the communication layer for
- * Ozone containers.
- */
-public final class XceiverServerRatis implements XceiverServerSpi {
-  static final Logger LOG = LoggerFactory.getLogger(XceiverServerRatis.class);
-  private final int port;
-  private final RaftServer server;
-  private ThreadPoolExecutor writeChunkExecutor;
-
-  private XceiverServerRatis(DatanodeID id, int port, String storageDir,
-      ContainerDispatcher dispatcher, Configuration conf) throws IOException {
-
-    final String rpcType = conf.get(
-        OzoneConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
-        OzoneConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT);
-    final RpcType rpc = SupportedRpcType.valueOfIgnoreCase(rpcType);
-    final int raftSegmentSize = conf.getInt(
-        OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY,
-        OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT);
-    final int raftSegmentPreallocatedSize = conf.getInt(
-        OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY,
-        OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT);
-    final int maxChunkSize = OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE;
-    final int numWriteChunkThreads = conf.getInt(
-        OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY,
-        OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT);
-
-    Objects.requireNonNull(id, "id == null");
-    this.port = port;
-    RaftProperties serverProperties = newRaftProperties(rpc, port,
-        storageDir, maxChunkSize, raftSegmentSize, raftSegmentPreallocatedSize);
-
-    writeChunkExecutor =
-        new ThreadPoolExecutor(numWriteChunkThreads, numWriteChunkThreads,
-            100, TimeUnit.SECONDS,
-            new ArrayBlockingQueue<>(1024),
-            new ThreadPoolExecutor.CallerRunsPolicy());
-    ContainerStateMachine stateMachine =
-        new ContainerStateMachine(dispatcher, writeChunkExecutor);
-    this.server = RaftServer.newBuilder()
-        .setServerId(RatisHelper.toRaftPeerId(id))
-        .setGroup(RatisHelper.emptyRaftGroup())
-        .setProperties(serverProperties)
-        .setStateMachine(stateMachine)
-        .build();
-  }
-
-  private static RaftProperties newRaftProperties(
-      RpcType rpc, int port, String storageDir, int scmChunkSize,
-      int raftSegmentSize, int raftSegmentPreallocatedSize) {
-    final RaftProperties properties = new RaftProperties();
-    RaftServerConfigKeys.Log.Appender.setBatchEnabled(properties, true);
-    RaftServerConfigKeys.Log.Appender.setBufferCapacity(properties,
-        SizeInBytes.valueOf(raftSegmentPreallocatedSize));
-    RaftServerConfigKeys.Log.setWriteBufferSize(properties,
-        SizeInBytes.valueOf(scmChunkSize));
-    RaftServerConfigKeys.Log.setPreallocatedSize(properties,
-        SizeInBytes.valueOf(raftSegmentPreallocatedSize));
-    RaftServerConfigKeys.Log.setSegmentSizeMax(properties,
-        SizeInBytes.valueOf(raftSegmentSize));
-    RaftServerConfigKeys.setStorageDir(properties, new File(storageDir));
-    RaftConfigKeys.Rpc.setType(properties, rpc);
-
-    RaftServerConfigKeys.Log.setMaxCachedSegmentNum(properties, 2);
-    GrpcConfigKeys.setMessageSizeMax(properties,
-        SizeInBytes.valueOf(scmChunkSize + raftSegmentPreallocatedSize));
-    RaftServerConfigKeys.Rpc.setTimeoutMin(properties,
-        TimeDuration.valueOf(800, TimeUnit.MILLISECONDS));
-    RaftServerConfigKeys.Rpc.setTimeoutMax(properties,
-        TimeDuration.valueOf(1000, TimeUnit.MILLISECONDS));
-    if (rpc == SupportedRpcType.GRPC) {
-      GrpcConfigKeys.Server.setPort(properties, port);
-    } else if (rpc == SupportedRpcType.NETTY) {
-      NettyConfigKeys.Server.setPort(properties, port);
-    }
-    return properties;
-  }
-
-  public static XceiverServerRatis newXceiverServerRatis(DatanodeID datanodeID,
-      Configuration ozoneConf, ContainerDispatcher dispatcher)
-      throws IOException {
-    final String ratisDir = File.separator + "ratis";
-    int localPort = ozoneConf.getInt(
-        OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT,
-        OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT_DEFAULT);
-    String storageDir = ozoneConf.get(
-        OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR);
-
-    if (Strings.isNullOrEmpty(storageDir)) {
-      storageDir = ozoneConf.get(OzoneConfigKeys
-          .OZONE_METADATA_DIRS);
-      Preconditions.checkNotNull(storageDir, "ozone.metadata.dirs " +
-          "cannot be null, Please check your configs.");
-      storageDir = storageDir.concat(ratisDir);
-      LOG.warn("Storage directory for Ratis is not configured. Mapping Ratis " +
-              "storage under {}. It is a good idea to map this to an SSD disk.",
-          storageDir);
-    }
-
-    // Get an available port on current node and
-    // use that as the container port
-    if (ozoneConf.getBoolean(OzoneConfigKeys
-            .DFS_CONTAINER_RATIS_IPC_RANDOM_PORT,
-        OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT_DEFAULT)) {
-      try (ServerSocket socket = new ServerSocket()) {
-        socket.setReuseAddress(true);
-        SocketAddress address = new InetSocketAddress(0);
-        socket.bind(address);
-        localPort = socket.getLocalPort();
-        LOG.info("Found a free port for the server : {}", localPort);
-        // If we have random local ports configured this means that it
-        // probably running under MiniOzoneCluster. Ratis locks the storage
-        // directories, so we need to pass different local directory for each
-        // local instance. So we map ratis directories under datanode ID.
-        storageDir =
-            storageDir.concat(File.separator + datanodeID.getDatanodeUuid());
-      } catch (IOException e) {
-        LOG.error("Unable find a random free port for the server, "
-            + "fallback to use default port {}", localPort, e);
-      }
-    }
-    datanodeID.setRatisPort(localPort);
-    return new XceiverServerRatis(datanodeID, localPort, storageDir,
-        dispatcher, ozoneConf);
-  }
-
-  @Override
-  public void start() throws IOException {
-    LOG.info("Starting {} {} at port {}", getClass().getSimpleName(),
-        server.getId(), getIPCPort());
-    writeChunkExecutor.prestartAllCoreThreads();
-    server.start();
-  }
-
-  @Override
-  public void stop() {
-    try {
-      writeChunkExecutor.shutdown();
-      server.close();
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  @Override
-  public int getIPCPort() {
-    return port;
-  }
-
-  /**
-   * Returns the Replication type supported by this end-point.
-   *
-   * @return enum -- {Stand_Alone, Ratis, Chained}
-   */
-  @Override
-  public OzoneProtos.ReplicationType getServerType() {
-    return OzoneProtos.ReplicationType.RATIS;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/package-info.java
deleted file mode 100644
index 8debfe0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.transport.server.ratis;
-
-/**
- * This package contains classes for the server implementation
- * using Apache Ratis
- */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
deleted file mode 100644
index 6ae45b6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.collections.MapIterator;
-import org.apache.commons.collections.map.LRUMap;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-/**
- * container cache is a LRUMap that maintains the DB handles.
- */
-public final class ContainerCache extends LRUMap {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ContainerCache.class);
-  private final Lock lock = new ReentrantLock();
-  private static ContainerCache cache;
-  private static final float LOAD_FACTOR = 0.75f;
-  /**
-   * Constructs a cache that holds DBHandle references.
-   */
-  private ContainerCache(int maxSize, float loadFactor, boolean
-      scanUntilRemovable) {
-    super(maxSize, loadFactor, scanUntilRemovable);
-  }
-
-  /**
-   * Return a singleton instance of {@link ContainerCache}
-   * that holds the DB handlers.
-   *
-   * @param conf - Configuration.
-   * @return A instance of {@link ContainerCache}.
-   */
-  public synchronized static ContainerCache getInstance(Configuration conf) {
-    if (cache == null) {
-      int cacheSize = conf.getInt(OzoneConfigKeys.OZONE_CONTAINER_CACHE_SIZE,
-          OzoneConfigKeys.OZONE_CONTAINER_CACHE_DEFAULT);
-      cache = new ContainerCache(cacheSize, LOAD_FACTOR, true);
-    }
-    return cache;
-  }
-
-  /**
-   * Closes a db instance.
-   *
-   * @param container - name of the container to be closed.
-   * @param db - db instance to close.
-   */
-  private void closeDB(String container, MetadataStore db) {
-    if (db != null) {
-      try {
-        db.close();
-      } catch (IOException e) {
-        LOG.error("Error closing DB. Container: " + container, e);
-      }
-    }
-  }
-
-  /**
-   * Closes all the db instances and resets the cache.
-   */
-  public void shutdownCache() {
-    lock.lock();
-    try {
-      // iterate the cache and close each db
-      MapIterator iterator = cache.mapIterator();
-      while (iterator.hasNext()) {
-        iterator.next();
-        MetadataStore db = (MetadataStore) iterator.getValue();
-        closeDB(iterator.getKey().toString(), db);
-      }
-      // reset the cache
-      cache.clear();
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  protected boolean removeLRU(LinkEntry entry) {
-    lock.lock();
-    try {
-      MetadataStore db = (MetadataStore) entry.getValue();
-      closeDB(entry.getKey().toString(), db);
-    } finally {
-      lock.unlock();
-    }
-    return true;
-  }
-
-  /**
-   * Returns a DB handle if available, create the handler otherwise.
-   *
-   * @param containerName - Name of the container.
-   * @return MetadataStore.
-   */
-  public MetadataStore getDB(String containerName, String containerDBPath)
-      throws IOException {
-    Preconditions.checkNotNull(containerName);
-    Preconditions.checkState(!containerName.isEmpty());
-    lock.lock();
-    try {
-      MetadataStore db = (MetadataStore) this.get(containerName);
-
-      if (db == null) {
-        db = MetadataStoreBuilder.newBuilder()
-            .setDbFile(new File(containerDBPath))
-            .setCreateIfMissing(false)
-            .build();
-        this.put(containerName, db);
-      }
-      return db;
-    } catch (Exception e) {
-      LOG.error("Error opening DB. Container:{} ContainerPath:{}",
-          containerName, containerDBPath, e);
-      throw e;
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  /**
-   * Remove a DB handler from cache.
-   *
-   * @param containerName - Name of the container.
-   */
-  public void removeDB(String containerName) {
-    Preconditions.checkNotNull(containerName);
-    Preconditions.checkState(!containerName.isEmpty());
-    lock.lock();
-    try {
-      MetadataStore db = (MetadataStore)this.get(containerName);
-      closeDB(containerName, db);
-      this.remove(containerName);
-    } finally {
-      lock.unlock();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/package-info.java
deleted file mode 100644
index 08264f0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.utils;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
deleted file mode 100644
index b143f22..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.container.ozoneimpl;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.impl.ChunkManagerImpl;
-import org.apache.hadoop.ozone.container.common.impl.ContainerManagerImpl;
-import org.apache.hadoop.ozone.container.common.impl.Dispatcher;
-import org.apache.hadoop.ozone.container.common.impl.KeyManagerImpl;
-import org.apache.hadoop.ozone.container.common.interfaces.ChunkManager;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
-import org.apache.hadoop.ozone.container.common.interfaces.KeyManager;
-import org.apache.hadoop.ozone.container.common.statemachine.background.BlockDeletingService;
-import org.apache.hadoop.ozone.container.common.transport.server.XceiverServer;
-import org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis;
-
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerSpi;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_ROOT_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConsts.INVALID_PORT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
-
-/**
- * Ozone main class sets up the network server and initializes the container
- * layer.
- */
-public class OzoneContainer {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(OzoneContainer.class);
-
-  private final Configuration ozoneConfig;
-  private final ContainerDispatcher dispatcher;
-  private final ContainerManager manager;
-  private final XceiverServerSpi[] server;
-  private final ChunkManager chunkManager;
-  private final KeyManager keyManager;
-  private final BlockDeletingService blockDeletingService;
-
-  /**
-   * Creates a network endpoint and enables Ozone container.
-   *
-   * @param ozoneConfig - Config
-   * @throws IOException
-   */
-  public OzoneContainer(DatanodeID datanodeID, Configuration ozoneConfig) throws
-      IOException {
-    this.ozoneConfig = ozoneConfig;
-    List<StorageLocation> locations = new LinkedList<>();
-    String[] paths = ozoneConfig.getStrings(
-        OzoneConfigKeys.OZONE_METADATA_DIRS);
-    if (paths != null && paths.length > 0) {
-      for (String p : paths) {
-        locations.add(StorageLocation.parse(
-            Paths.get(p).resolve(CONTAINER_ROOT_PREFIX).toString()));
-      }
-    } else {
-      getDataDir(locations);
-    }
-
-    manager = new ContainerManagerImpl();
-    manager.init(this.ozoneConfig, locations, datanodeID);
-    this.chunkManager = new ChunkManagerImpl(manager);
-    manager.setChunkManager(this.chunkManager);
-
-    this.keyManager = new KeyManagerImpl(manager, ozoneConfig);
-    manager.setKeyManager(this.keyManager);
-
-    long svcInterval =
-        ozoneConfig.getTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL,
-        OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS);
-    long serviceTimeout = ozoneConfig.getTimeDuration(
-        OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
-        OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
-    this.blockDeletingService = new BlockDeletingService(manager,
-        svcInterval, serviceTimeout, ozoneConfig);
-
-    this.dispatcher = new Dispatcher(manager, this.ozoneConfig);
-
-    server = new XceiverServerSpi[]{
-        new XceiverServer(this.ozoneConfig, this.dispatcher),
-      XceiverServerRatis
-          .newXceiverServerRatis(datanodeID, ozoneConfig, dispatcher)
-    };
-  }
-
-  /**
-   * Starts serving requests to ozone container.
-   *
-   * @throws IOException
-   */
-  public void start() throws IOException {
-    for (XceiverServerSpi serverinstance : server) {
-      serverinstance.start();
-    }
-    blockDeletingService.start();
-    dispatcher.init();
-  }
-
-  /**
-   * Stops the ozone container.
-   * <p>
-   * Shutdown logic is not very obvious from the following code. if you need to
-   * modify the logic, please keep these comments in mind. Here is the shutdown
-   * sequence.
-   * <p>
-   * 1. We shutdown the network ports.
-   * <p>
-   * 2. Now we need to wait for all requests in-flight to finish.
-   * <p>
-   * 3. The container manager lock is a read-write lock with "Fairness"
-   * enabled.
-   * <p>
-   * 4. This means that the waiting threads are served in a "first-come-first
-   * -served" manner. Please note that this applies to waiting threads only.
-   * <p>
-   * 5. Since write locks are exclusive, if we are waiting to get a lock it
-   * implies that we are waiting for in-flight operations to complete.
-   * <p>
-   * 6. if there are other write operations waiting on the reader-writer lock,
-   * fairness guarantees that they will proceed before the shutdown lock
-   * request.
-   * <p>
-   * 7. Since all operations either take a reader or writer lock of container
-   * manager, we are guaranteed that we are the last operation since we have
-   * closed the network port, and we wait until close is successful.
-   * <p>
-   * 8. We take the writer lock and call shutdown on each of the managers in
-   * reverse order. That is chunkManager, keyManager and containerManager is
-   * shutdown.
-   */
-  public void stop() {
-    LOG.info("Attempting to stop container services.");
-    for(XceiverServerSpi serverinstance: server) {
-      serverinstance.stop();
-    }
-    dispatcher.shutdown();
-
-    try {
-      this.manager.writeLock();
-      this.chunkManager.shutdown();
-      this.keyManager.shutdown();
-      this.manager.shutdown();
-      this.blockDeletingService.shutdown();
-      LOG.info("container services shutdown complete.");
-    } catch (IOException ex) {
-      LOG.warn("container service shutdown error:", ex);
-    } finally {
-      this.manager.writeUnlock();
-    }
-  }
-
-  /**
-   * Returns a paths to data dirs.
-   *
-   * @param pathList - List of paths.
-   * @throws IOException
-   */
-  private void getDataDir(List<StorageLocation> pathList) throws IOException {
-    for (String dir : ozoneConfig.getStrings(DFS_DATANODE_DATA_DIR_KEY)) {
-      StorageLocation location = StorageLocation.parse(dir);
-      pathList.add(location);
-    }
-  }
-
-  /**
-   * Returns node report of container storage usage.
-   */
-  public SCMNodeReport getNodeReport() throws IOException {
-    return this.manager.getNodeReport();
-  }
-
-  private int getPortbyType(OzoneProtos.ReplicationType replicationType) {
-    for (XceiverServerSpi serverinstance : server) {
-      if (serverinstance.getServerType() == replicationType) {
-        return serverinstance.getIPCPort();
-      }
-    }
-    return INVALID_PORT;
-  }
-
-  /**
-   * Returns the container server IPC port.
-   *
-   * @return Container server IPC port.
-   */
-  public int getContainerServerPort() {
-    return getPortbyType(OzoneProtos.ReplicationType.STAND_ALONE);
-  }
-
-  /**
-   * Returns the Ratis container Server IPC port.
-   *
-   * @return Ratis port.
-   */
-  public int getRatisContainerServerPort() {
-    return getPortbyType(OzoneProtos.ReplicationType.RATIS);
-  }
-
-  /**
-   * Returns container report.
-   * @return - container report.
-   * @throws IOException
-   */
-  public ContainerReportsRequestProto getContainerReport() throws IOException {
-    return this.manager.getContainerReport();
-  }
-
-// TODO: remove getContainerReports
-  /**
-   * Returns the list of closed containers.
-   * @return - List of closed containers.
-   * @throws IOException
-   */
-  public List<ContainerData> getContainerReports() throws IOException {
-    return this.manager.getContainerReports();
-  }
-
-  @VisibleForTesting
-  public ContainerManager getContainerManager() {
-    return this.manager;
-  }
-
-  /**
-   * Get the container report state to send via HB to SCM.
-   * @return the container report state.
-   */
-  public ReportState getContainerReportState() {
-    return this.manager.getContainerReportState();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/package-info.java
deleted file mode 100644
index c99c038..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.ozoneimpl;
-/**
- Ozone main that calls into the container layer
-**/
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManager.java
deleted file mode 100644
index 6c75691..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManager.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * BucketManager handles all the bucket level operations.
- */
-public interface BucketManager {
-  /**
-   * Creates a bucket.
-   * @param bucketInfo - KsmBucketInfo for creating bucket.
-   */
-  void createBucket(KsmBucketInfo bucketInfo) throws IOException;
-  /**
-   * Returns Bucket Information.
-   * @param volumeName - Name of the Volume.
-   * @param bucketName - Name of the Bucket.
-   */
-  KsmBucketInfo getBucketInfo(String volumeName, String bucketName)
-      throws IOException;
-
-  /**
-   * Sets bucket property from args.
-   * @param args - BucketArgs.
-   * @throws IOException
-   */
-  void setBucketProperty(KsmBucketArgs args) throws IOException;
-
-  /**
-   * Deletes an existing empty bucket from volume.
-   * @param volumeName - Name of the volume.
-   * @param bucketName - Name of the bucket.
-   * @throws IOException
-   */
-  void deleteBucket(String volumeName, String bucketName) throws IOException;
-
-  /**
-   * Returns a list of buckets represented by {@link KsmBucketInfo}
-   * in the given volume.
-   *
-   * @param volumeName
-   *   Required parameter volume name determines buckets in which volume
-   *   to return.
-   * @param startBucket
-   *   Optional start bucket name parameter indicating where to start
-   *   the bucket listing from, this key is excluded from the result.
-   * @param bucketPrefix
-   *   Optional start key parameter, restricting the response to buckets
-   *   that begin with the specified name.
-   * @param maxNumOfBuckets
-   *   The maximum number of buckets to return. It ensures
-   *   the size of the result will not exceed this limit.
-   * @return a list of buckets.
-   * @throws IOException
-   */
-  List<KsmBucketInfo> listBuckets(String volumeName,
-      String startBucket, String bucketPrefix, int maxNumOfBuckets)
-      throws IOException;
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org